From a3e9e51801ece22bf6a16b624877fe6209bbdaab Mon Sep 17 00:00:00 2001 From: Nikhil Pareek <8347993+nik13@users.noreply.github.com> Date: Wed, 11 Nov 2026 09:33:49 -0800 Subject: [PATCH 01/17] feat(python): capture typed framework media with one upload per trace LangChain and LlamaIndex message content blocks, Agno Image/Audio/File and Pydantic AI BinaryContent reach the shared attachment manager. The shared helper gains a per-trace claim table, so a framework span and the provider span beneath it (traceai_openai, traceai_anthropic) upload identical bytes once. A later observer must pass its own policy before reusing a claim, and only admitted bytes are hashed. URLs go to the core RemoteUrl policy and are never fetched; paths, streams, provider files and unknown objects are never read. Agno capture uses the intersection of the FI and OpenInference TraceConfig. Co-Authored-By: Claude Opus 5.5 --- .../attachments-python-frameworks-tests.yml | 84 ++++++ .../attachments/media_capture.py | 251 +++++++++++++++++- .../agno/traceai_agno/_attachments.py | 150 +++++++++++ .../agno/traceai_agno/_instrumentor.py | 10 + .../agno/traceai_agno/_wrappers/_install.py | 18 +- .../traceai_agno/_wrappers/_model_wrapper.py | 45 +++- .../langchain/traceai_langchain/__init__.py | 23 +- .../traceai_langchain/_attachments.py | 182 +++++++++++++ .../langchain/traceai_langchain/_tracer.py | 34 ++- .../traceai_llamaindex/__init__.py | 21 +- .../traceai_llamaindex/_attachments.py | 123 +++++++++ .../traceai_llamaindex/_handler.py | 34 ++- .../traceai_pydantic_ai/_attachments.py | 158 +++++++++++ .../_execution_wrappers.py | 28 +- .../traceai_pydantic_ai/_instrumentor.py | 37 ++- .../tests/attachments_frameworks/conftest.py | 56 ++++ .../test_agno_attachments.py | 136 ++++++++++ .../attachments_frameworks/test_claims.py | 236 ++++++++++++++++ .../attachments_frameworks/test_exit_probe.py | 93 +++++++ .../test_langchain_attachments.py | 224 ++++++++++++++++ .../test_llamaindex_attachments.py | 146 ++++++++++ .../test_provider_dedup.py | 155 +++++++++++ .../test_pydantic_ai_attachments.py | 160 +++++++++++ 23 files changed, 2379 insertions(+), 25 deletions(-) create mode 100644 .github/workflows/attachments-python-frameworks-tests.yml create mode 100644 python/frameworks/agno/traceai_agno/_attachments.py create mode 100644 python/frameworks/langchain/traceai_langchain/_attachments.py create mode 100644 python/frameworks/llama_index/traceai_llamaindex/_attachments.py create mode 100644 python/frameworks/pydantic-ai/traceai_pydantic_ai/_attachments.py create mode 100644 python/tests/attachments_frameworks/conftest.py create mode 100644 python/tests/attachments_frameworks/test_agno_attachments.py create mode 100644 python/tests/attachments_frameworks/test_claims.py create mode 100644 python/tests/attachments_frameworks/test_exit_probe.py create mode 100644 python/tests/attachments_frameworks/test_langchain_attachments.py create mode 100644 python/tests/attachments_frameworks/test_llamaindex_attachments.py create mode 100644 python/tests/attachments_frameworks/test_provider_dedup.py create mode 100644 python/tests/attachments_frameworks/test_pydantic_ai_attachments.py diff --git a/.github/workflows/attachments-python-frameworks-tests.yml b/.github/workflows/attachments-python-frameworks-tests.yml new file mode 100644 index 000000000..9225990f7 --- /dev/null +++ b/.github/workflows/attachments-python-frameworks-tests.yml @@ -0,0 +1,84 @@ +name: Python framework attachments + +# Real LangChain, LlamaIndex, Agno and Pydantic AI (declared floor and latest) through +# the Python framework instrumentors to a TLS loopback F01 attachment server, with and +# without traceai_openai on the same manager. Outbound network is disabled while the +# tests run. + +on: + pull_request: + paths: + - 'python/fi_instrumentation/attachments/**' + - 'python/frameworks/langchain/**' + - 'python/frameworks/llama_index/**' + - 'python/frameworks/agno/**' + - 'python/frameworks/pydantic-ai/**' + - 'python/frameworks/openai/**' + - 'python/tests/attachments_frameworks/**' + - 'python/tests/attachments_providers/conftest.py' + - '.github/workflows/attachments-python-frameworks-tests.yml' + push: + branches: [dev] + paths: + - 'python/fi_instrumentation/attachments/**' + - 'python/frameworks/langchain/**' + - 'python/frameworks/llama_index/**' + - 'python/frameworks/agno/**' + - 'python/frameworks/pydantic-ai/**' + - 'python/frameworks/openai/**' + - 'python/tests/attachments_frameworks/**' + - 'python/tests/attachments_providers/conftest.py' + - '.github/workflows/attachments-python-frameworks-tests.yml' + workflow_dispatch: + +permissions: + contents: read + +jobs: + frameworks: + name: Framework attachments (${{ matrix.name }}) + runs-on: ubuntu-24.04 + timeout-minutes: 25 + strategy: + fail-fast: false + matrix: + include: + # Declared floors (ChatMessage blocks need llama-index-core 0.12.3+). + - name: floor + python: '3.10' + sdks: >- + langchain-core==0.2.43 llama-index-core==0.12.41 agno==2.7.2 + openinference-instrumentation-agno==1.1.0 pydantic-ai-slim[openai]==0.1.0 openai==1.69.0 + 'httpx<0.28' + # Latest releases, including the framework OpenAI integrations for the dedup tests. + - name: latest + python: '3.12' + sdks: >- + langchain-core==1.8.0 langchain-openai==1.7.4 llama-index-core==0.14.25 + llama-index-llms-openai==0.8.4 agno==3.1.12 openinference-instrumentation-agno==1.1.0 + pydantic-ai-slim[openai]==2.67.0 + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: false + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: ${{ matrix.python }} + - name: Install source integrations and the selected frameworks + env: + SDKS: ${{ matrix.sdks }} + run: | + python -m venv .venv + # shellcheck disable=SC2086 + eval .venv/bin/pip install -e python -e python/frameworks/openai -e python/frameworks/langchain \ + -e python/frameworks/llama_index -e python/frameworks/agno -e python/frameworks/pydantic-ai \ + opentelemetry-sdk "'pytest>=7,<9'" $SDKS + .venv/bin/pip freeze | grep -iE '^(langchain|llama-index-core|llama-index-llms|agno|pydantic-ai|openai|openinference)' || true + - name: Run with outbound network disabled + run: | + sudo unshare --net -- bash -e -c ' + ip link set lo up + cd "$2/python" + AGNO_TELEMETRY=false PYTHONPATH=. "$1" -m pytest tests/attachments_frameworks -q -p no:cacheprovider \ + -o addopts="" -o faulthandler_timeout=120 + ' _ "$PWD/.venv/bin/python" "$PWD" diff --git a/python/fi_instrumentation/attachments/media_capture.py b/python/fi_instrumentation/attachments/media_capture.py index a2fd637c4..8f70627eb 100644 --- a/python/fi_instrumentation/attachments/media_capture.py +++ b/python/fi_instrumentation/attachments/media_capture.py @@ -13,6 +13,11 @@ - One registry per manager remembers admitted (span, position) pairs, so a second observer of the same span and position gets the FIRST occurrence back and uploads nothing. +- One claim table per manager remembers the bytes each trace already uploaded, + so a framework span and the provider span beneath it (or any later observer in + the same trace) upload identical bytes once. A later observer must be admitted + under its OWN policy before it may reuse a claim, and only admitted bytes are + hashed. Every public function is non-throwing and returns ``None``/empty on denial. """ @@ -20,15 +25,17 @@ import base64 import binascii import dataclasses +import hashlib import re import threading import weakref from collections import OrderedDict -from typing import Any, Callable, Iterable, Mapping, NamedTuple, Optional, Tuple +from types import MappingProxyType +from typing import Any, Callable, Iterable, Mapping, NamedTuple, Optional, Sequence, Tuple from .policy import CapturePolicy from .stream import StreamOptions -from .types import Bytes, CorrelationHandle +from .types import Bytes, CorrelationHandle, RemoteUrl KiB = 1024 MiB = 1024 * KiB @@ -41,6 +48,11 @@ STREAM_RESERVATIONS = (4 * MiB, 2 * MiB, 1 * MiB, 512 * KiB, 256 * KiB) IMAGE_MIMES = ("image/png", "image/jpeg", "image/webp") FILE_MIMES = ("application/pdf", "application/octet-stream") +DOCUMENT_MIMES = ("application/pdf",) +AUDIO_MIMES = ("audio/wav", "audio/mpeg", "audio/ogg", "audio/flac", "audio/webm") +# Framework boundaries hand over complete inline parts (mirrors @traceai/framework-attachments). +MAX_FRAMEWORK_BYTES = 1 * MiB +MAX_FRAMEWORK_PARTS = 64 TOOL_ID = re.compile(r"[a-zA-Z0-9_-]{1,128}") _DATA_URI = re.compile(r"data:([a-z]+/[a-z0-9.+-]+);base64") _BASE64 = re.compile(r"[A-Za-z0-9+/]*={0,2}") @@ -159,19 +171,92 @@ def put(self, manager, key: str, position: str, occurrence: Occurrence) -> None: REGISTRY = _Registry() +class _Claims: + """Uploaded bytes per manager and trace: (mime, sha256) -> occurrence. Digests and IDs only. + + The lock is held across the lookup and ``ticket.attach`` (which only queues), + so two concurrent observers of identical bytes cannot both upload. + """ + + def __init__(self) -> None: + self.lock = threading.Lock() + self._managers: "weakref.WeakKeyDictionary" = weakref.WeakKeyDictionary() + + def get(self, manager, trace: str, key: Tuple[str, str]) -> Optional[Occurrence]: + traces = self._managers.get(manager) + claims = traces.get(trace) if traces is not None else None + return claims.get(key) if claims is not None else None + + def put(self, manager, trace: str, key: Tuple[str, str], occurrence: Occurrence) -> None: + traces = self._managers.get(manager) + if traces is None: + traces = OrderedDict() + self._managers[manager] = traces + claims = traces.get(trace) + if claims is None: + if len(traces) >= MAX_SPANS: + traces.popitem(last=False) + claims = traces[trace] = {} + else: + traces.move_to_end(trace) + if len(claims) < MAX_PARTS: + claims[key] = occurrence + + +CLAIMS = _Claims() + + +def attach_claimed(manager, ticket, data: bytes, mime: str, fields: dict) -> Optional[Occurrence]: + """Attach ADMITTED bytes at most once per trace and manager. Never raises. + + The caller's ``ticket`` was already admitted under its own policy, so reusing an + earlier occurrence never widens what this observer may capture. A reuse + releases the ticket and uploads nothing. + """ + try: + correlation = ticket.correlation + trace = correlation.trace_id if correlation is not None else None + key = (mime, hashlib.sha256(data).hexdigest()) if trace is not None else None + with CLAIMS.lock: + previous = CLAIMS.get(manager, trace, key) if key is not None else None + if previous is not None: + ticket.release() + return previous + handle = ticket.attach(Bytes(data), **fields) + if handle.admission != "queued": + return None + occurrence = Occurrence(handle.occurrence_id, mime) + if key is not None: + CLAIMS.put(manager, trace, key, occurrence) + return occurrence + except Exception: + try: + ticket.release() + except Exception: + pass + return None + + def encoded_length(encoded: Any, max_bytes: int = MAX_INLINE_BYTES) -> int: """Exact decoded length from the string's shape only (no decode); 0 when out of bounds.""" - if not isinstance(encoded, str): + if not isinstance(encoded, (str, bytes)): return 0 length = len(encoded) if not length or length % 4 or length > 4 * -(-max_bytes // 3): return 0 - return (length // 4) * 3 - (2 if encoded.endswith("==") else 1 if encoded.endswith("=") else 0) + pad = "=" if isinstance(encoded, str) else b"=" + return (length // 4) * 3 - (2 if encoded.endswith(pad * 2) else 1 if encoded.endswith(pad) else 0) def split_inline(value: Any, mime_types: Tuple[str, ...], raw: bool, max_bytes: int = MAX_INLINE_BYTES) -> Optional[Tuple[str, str, int]]: """Structural parse: (mime, encoded, exact decoded length) or None. Bounded before slicing.""" + if raw and isinstance(value, bytes) and 0 < len(value) <= 4 * -(-max_bytes // 3): + # Base64 held as ASCII bytes (LlamaIndex blocks). Bounded before the copy. + try: + value = value.decode("ascii") + except UnicodeDecodeError: + return None if not isinstance(value, str): return None mime, start = mime_types[0], 0 @@ -212,8 +297,13 @@ def attach_inline(manager, span, read: Callable[[], Any], mime_types: Tuple[str, direction: str, purpose: Optional[str], capture: CapturePolicy, message_index: Optional[int] = None, part_index: Optional[int] = None, role: Optional[str] = None, tool_call_id: Optional[str] = None, - max_bytes: int = MAX_INLINE_BYTES) -> Optional[Occurrence]: - """Preflight, then read once, decode strictly and attach. Never raises.""" + max_bytes: int = MAX_INLINE_BYTES, verify: bool = False) -> Optional[Occurrence]: + """Preflight, then read once, decode strictly and attach. Never raises. + + ``verify`` also requires the decoded container (magic bytes) to match the + MIME it was admitted under. Identical bytes already uploaded in this trace + through the same manager are reused (``attach_claimed``). + """ try: state = {} @@ -248,11 +338,60 @@ def preflight(mime): if not ticket.admitted: return None data = strict_decode(encoded, ticket.max_bytes, max_bytes) - if data is None: + if data is None or (verify and sniff_media(data[:16]) != mime): + ticket.release() + return None + return attach_claimed(manager, ticket, data, mime, _fields(message_index, part_index, role, tool_call_id)) + except Exception: + return None + + +def sniff_media(head: Any) -> Optional[str]: + """Image, PDF or audio MIME from magic bytes the application already holds; None when unknown.""" + if not isinstance(head, (bytes, bytearray, memoryview)): + return None + head = bytes(head[:16]) + if head[:8] == b"\x89PNG\r\n\x1a\n": + return "image/png" + if head[:3] == b"\xff\xd8\xff": + return "image/jpeg" + if len(head) >= 12 and head[:4] == b"RIFF" and head[8:12] == b"WEBP": + return "image/webp" + if head[:5] == b"%PDF-": + return "application/pdf" + return sniff_audio(head) + + +def attach_bytes(manager, span, data: Any, mime_types: Tuple[str, ...], *, direction: str, + purpose: Optional[str], capture: CapturePolicy, message_index: Optional[int] = None, + part_index: Optional[int] = None, role: Optional[str] = None, + tool_call_id: Optional[str] = None, max_bytes: int = MAX_FRAMEWORK_BYTES) -> Optional[Occurrence]: + """Complete in-memory bytes a framework already holds (never a stream, path or URL). Never raises. + + Admitted at the exact length under the declared type first; the magic bytes + must then name one of ``mime_types``, re-admitted under that exact type. + """ + try: + if not isinstance(data, (bytes, bytearray, memoryview)) or not mime_types: + return None + length = data.nbytes if isinstance(data, memoryview) else len(data) + if not 0 < length <= max_bytes: + return None + options = dict(direction=direction, purpose=purpose, capture_policy=capture, max_bytes=length) + ticket = manager.preflight(span, mime_type=mime_types[0], **options) + if not ticket.admitted: + return None + mime = sniff_media(data[:16]) + if mime is None or mime not in mime_types: ticket.release() return None - handle = ticket.attach(Bytes(data), **_fields(message_index, part_index, role, tool_call_id)) - return Occurrence(handle.occurrence_id, mime) if handle.admission == "queued" else None + if mime != mime_types[0]: + ticket.release() + ticket = manager.preflight(span, mime_type=mime, **options) + if not ticket.admitted: + return None + payload = data if isinstance(data, bytes) else bytes(data) + return attach_claimed(manager, ticket, payload, mime, _fields(message_index, part_index, role, tool_call_id)) except Exception: return None @@ -343,9 +482,8 @@ def attach_audio(manager, span, data: Any, *, direction: str, purpose: Optional[ ticket = manager.preflight(span, mime_type=mime, **options) if not ticket.admitted: return None - handle = ticket.attach(Bytes(bytes(data) if not isinstance(data, bytes) else data), - **_fields(message_index, part_index, role)) - return Occurrence(handle.occurrence_id, mime) if handle.admission == "queued" else None + return attach_claimed(manager, ticket, bytes(data) if not isinstance(data, bytes) else data, mime, + _fields(message_index, part_index, role)) except Exception: return None @@ -455,3 +593,92 @@ def request_files(files: Any) -> Iterable[Tuple[int, Any]]: yield index, content except Exception: return + + +# ------------------------------------------------------------------ framework parts +class MediaPart(NamedTuple): + """One typed media part a framework binding found at an original-value boundary. + + ``kind`` is ``"inline"`` (base64 str/ASCII bytes, or a data URI when ``raw`` is + False), ``"bytes"`` (complete raw bytes) or ``"url"`` (a remote reference that + is handed to the core's ``RemoteUrl`` policy and never fetched here). + ``read`` is called at most once, and only after admission. + """ + + kind: str + read: Callable[[], Any] + mime_types: Tuple[str, ...] + purpose: str + message_index: int + part_index: int + role: str = "unknown" + raw: bool = True + tool_call_id: Optional[str] = None + + +def intersect(*policies: CapturePolicy) -> CapturePolicy: + """Deny-wins intersection: any layer that hides content, mutes or marks provenance wins.""" + policies = tuple(p for p in policies if isinstance(p, CapturePolicy)) or (CapturePolicy(),) + allow = ("capture_content", "record_inputs", "record_outputs", "requested") + mark = ("muted", "temporary_chat", "model_input", "contains_input", "contains_output", "error_detail", + "new_recording") + values = {name: all(getattr(p, name) for p in policies) for name in allow} + values.update({name: any(getattr(p, name) for p in policies) for name in mark}) + return CapturePolicy(**values) + + +def capture_parts(manager, span, namespace: str, parts: Iterable[MediaPart], *, direction: str, + trace_configs: Sequence[Any] = (), capture: Optional[CapturePolicy] = None, + max_bytes: int = MAX_FRAMEWORK_BYTES) -> Mapping[str, Occurrence]: + """Capture typed framework parts on ``span``; returns admitted ``"{dir}/{msg}/{part}"`` positions. + + Every local ``trace_configs`` entry (FI TraceConfig, OpenInference TraceConfig, ...) + folds in deny-only; the strictest wins. At most ``MAX_FRAMEWORK_PARTS`` positions per + span. Never raises, never walks arbitrary objects, never fetches or opens anything. + """ + observation = Observation(manager, span, namespace) + if not observation.usable: + return EMPTY + try: + visited = 0 + for part in parts: + visited += 1 + if visited > MAX_FRAMEWORK_PARTS or len(observation.captured) >= MAX_FRAMEWORK_PARTS: + break + if not isinstance(part, MediaPart) or not part.mime_types: + continue + image = part.mime_types[0].startswith("image/") + policy = intersect(*(local_capture_policy(config, direction=direction, base=capture, image=image) + for config in (tuple(trace_configs) or (None,)))) + fields = dict(direction=direction, purpose=part.purpose, capture=policy, + message_index=part.message_index, part_index=part.part_index, role=part.role, + tool_call_id=part.tool_call_id) + position = "%s/%d/%d" % (direction, part.message_index, part.part_index) + if part.kind == "inline": + observation.visit(position, lambda p=part, f=fields: attach_inline( + manager, span, p.read, p.mime_types, raw=p.raw, max_bytes=max_bytes, verify=True, **f)) + elif part.kind == "bytes": + observation.visit(position, lambda p=part, f=fields: attach_bytes( + manager, span, p.read(), p.mime_types, max_bytes=max_bytes, **f)) + elif part.kind == "url": + observation.visit(position, lambda p=part, f=fields: _remote(manager, span, p, f)) + except Exception: + pass # Instrumentation must not throw or log source-bearing errors. + return MappingProxyType(dict(observation.captured)) if observation.captured else EMPTY + + +def looks_base64(data: Any) -> bool: + """Bytes a framework holds as base64 ASCII (vs raw container bytes); decided from the head only.""" + if not isinstance(data, (bytes, bytearray)) or not data or len(data) % 4: + return False + return all(48 <= c <= 57 or 65 <= c <= 90 or 97 <= c <= 122 or c in (43, 47, 61) for c in bytes(data[:64])) + + +def _remote(manager, span, part: MediaPart, fields: dict) -> Optional[Occurrence]: + """A remote reference goes to the core's RemoteUrl policy. Nothing here resolves or fetches it.""" + url = part.read() + if not isinstance(url, str) or not url.startswith(("https://", "http://")) or len(url) > 2048: + return None + handle = manager.attach(span, RemoteUrl(url), mime_type=part.mime_types[0], direction=fields["direction"], + purpose=fields["purpose"], capture_policy=fields["capture"]) + return Occurrence(handle.occurrence_id, part.mime_types[0]) if handle.admission == "queued" else None diff --git a/python/frameworks/agno/traceai_agno/_attachments.py b/python/frameworks/agno/traceai_agno/_attachments.py new file mode 100644 index 000000000..c4766a3d4 --- /dev/null +++ b/python/frameworks/agno/traceai_agno/_attachments.py @@ -0,0 +1,150 @@ +"""Agno typed media through the shared, provider-owned attachment manager. + +Observed at the model boundary only (``Model.invoke`` and friends): + +- input: ``Message.images`` / ``audio`` / ``files`` of every message (tool + messages keep their ``tool_call_id``); +- output: ``ModelResponse.images`` / ``audio`` / ``audios`` / ``files`` of a + non-streaming call. + +Only ``content`` held as bytes is captured, and its magic bytes must match an +admitted type. ``filepath`` is never opened, ``url`` goes to the core's +``RemoteUrl`` policy without being fetched, ``external`` provider objects and +media references are ignored. Videos are not captured. + +Privacy is the INTERSECTION of the FI TraceConfig and the OpenInference +TraceConfig the instrumentor runs with (either one hiding wins), the manager's +own policy, and the per-call capture policy. Bytes identical to ones a provider +instrumentor already uploaded in the same trace are not uploaded again. +""" + +from types import MappingProxyType +from typing import Any, Iterator, Mapping, Optional, Sequence + +from fi_instrumentation.attachments.media_capture import ( + AUDIO_MIMES, + DOCUMENT_MIMES, + EMPTY, + IMAGE_MIMES, + MAX_FRAMEWORK_BYTES, + MAX_FRAMEWORK_PARTS, + MediaPart, + Occurrence, + capture_parts, + looks_base64, +) + +__all__ = ("capture_agno_input", "capture_agno_output", "agno_attachment_capabilities") + +agno_attachment_capabilities = MappingProxyType({ + "schema_version": 1, + "operations": ("model.invoke.input", "model.invoke.output", "model.invoke_stream.input"), + "sources": ("content_bytes",), + "unsupported": ("filepath", "remote_url_fetch", "external_file", "media_reference", "video", + "streamed_output"), + "max_part_bytes": MAX_FRAMEWORK_BYTES, + "max_parts_per_span": MAX_FRAMEWORK_PARTS, +}) + +_ROLES = ("user", "assistant", "tool", "system") +_AUDIO_FORMATS = {"wav": "audio/wav", "mp3": "audio/mpeg", "mpeg": "audio/mpeg", "ogg": "audio/ogg", + "flac": "audio/flac", "webm": "audio/webm"} +_IMAGE_FORMATS = {"png": "image/png", "jpeg": "image/jpeg", "jpg": "image/jpeg", "webp": "image/webp"} +# Message/ModelResponse field -> MIME family. Order fixes part indexes. +_INPUT_FIELDS = (("images", IMAGE_MIMES), ("audio", AUDIO_MIMES), ("files", DOCUMENT_MIMES)) +_OUTPUT_FIELDS = (("images", IMAGE_MIMES), ("audio", AUDIO_MIMES), ("audios", AUDIO_MIMES), + ("files", DOCUMENT_MIMES)) + + +def _field(value: Any, name: str) -> Any: + """A declared model field only: never a property, resolver or ``__getattr__``.""" + try: + return vars(value).get(name) + except TypeError: + return None + + +def _ordered(mime: Any, family: tuple) -> Optional[tuple]: + if mime is None: + return family + if isinstance(mime, str) and mime in family: + return (mime,) + tuple(m for m in family if m != mime) + return None + + +def _declared(media: Any, family: tuple) -> Optional[tuple]: + mime = _field(media, "mime_type") + if mime is None: + fmt = _field(media, "format") + if isinstance(fmt, str): + mime = (_IMAGE_FORMATS if family is IMAGE_MIMES else _AUDIO_FORMATS if family is AUDIO_MIMES + else {"pdf": "application/pdf"}).get(fmt.lower(), "unsupported/format") + return _ordered(mime, family) + + +def _media_parts(values: Any, family: tuple, purpose: str, message_index: int, start: int, role: str, + tool_call_id: Optional[str]) -> Iterator[MediaPart]: + values = [values] if values is not None and not isinstance(values, (list, tuple)) else list(values or ()) + for offset, media in enumerate(values[:MAX_FRAMEWORK_PARTS]): + mimes = _declared(media, family) + if not mimes: + continue + content = _field(media, "content") + url = _field(media, "url") + index = start + offset + if isinstance(content, (bytes, bytearray)): + kind = "inline" if looks_base64(content) else "bytes" + yield MediaPart(kind, lambda c=content: c, mimes, purpose, message_index, index, role=role, + tool_call_id=tool_call_id) + elif content is None and isinstance(url, str) and _field(media, "filepath") is None: + yield MediaPart("url", lambda u=url: u, mimes, purpose, message_index, index, role=role, + tool_call_id=tool_call_id) + # filepath, external objects, media references, str content: never read or resolved. + + +def _input_parts(messages: Sequence[Any]) -> Iterator[MediaPart]: + for message_index, message in enumerate(list(messages)[:MAX_FRAMEWORK_PARTS]): + role = _field(message, "role") + role = role if role in _ROLES else "unknown" + tool_call_id = _field(message, "tool_call_id") if role == "tool" else None + tool_call_id = tool_call_id if isinstance(tool_call_id, str) else None + start = 0 + for name, family in _INPUT_FIELDS: + values = _field(message, name) + purpose = ("tool_result" if role == "tool" else "document" if family is DOCUMENT_MIMES + else "model_input") + parts = list(_media_parts(values, family, purpose, message_index, start, role, tool_call_id)) + yield from parts + start += len(values) if isinstance(values, (list, tuple)) else (1 if values is not None else 0) + + +def _output_parts(response: Any) -> Iterator[MediaPart]: + start = 0 + for name, family in _OUTPUT_FIELDS: + values = _field(response, name) + yield from _media_parts(values, family, "model_output", 0, start, "assistant", None) + start += len(values) if isinstance(values, (list, tuple)) else (1 if values is not None else 0) + + +def capture_agno_input(manager, span, messages: Any, *, trace_configs: Sequence[Any] = (), + capture=None) -> Mapping[str, Occurrence]: + """Capture typed media of the model call's input messages. Never raises.""" + try: + if not isinstance(messages, (list, tuple)): + return EMPTY + return capture_parts(manager, span, "agno", _input_parts(messages), direction="input", + trace_configs=trace_configs, capture=capture) + except Exception: + return EMPTY + + +def capture_agno_output(manager, span, response: Any, *, trace_configs: Sequence[Any] = (), + capture=None) -> Mapping[str, Occurrence]: + """Capture generated media of a non-streaming ``ModelResponse``. Never raises.""" + try: + if response is None: + return EMPTY + return capture_parts(manager, span, "agno", _output_parts(response), direction="output", + trace_configs=trace_configs, capture=capture) + except Exception: + return EMPTY diff --git a/python/frameworks/agno/traceai_agno/_instrumentor.py b/python/frameworks/agno/traceai_agno/_instrumentor.py index 266f922a7..673b903c0 100644 --- a/python/frameworks/agno/traceai_agno/_instrumentor.py +++ b/python/frameworks/agno/traceai_agno/_instrumentor.py @@ -36,6 +36,7 @@ class AgnoInstrumentorWrapper: dict[str, str], str | None, TraceConfig | None, + object | None, ] | None ) @@ -59,6 +60,7 @@ def instrument( *, project_name: str | None = None, config: TraceConfig | None = None, + attachments: object | None = None, ) -> AgnoInstrumentorWrapper: """Use an existing provider OR create an OTLP/HTTP protobuf exporter. @@ -66,6 +68,9 @@ def instrument( globally. Endpoint configuration owns a private provider. With no arguments, an already configured global provider is required. ``config`` is OpenInference's TraceConfig, not fi_instrumentation's. + ``attachments`` is the provider-owned attachment manager + (``register(...).attachment_manager``); media is captured only when it is + enabled, under both the FI and the OpenInference content policy. """ with self._lock: configuration = ( @@ -74,6 +79,7 @@ def instrument( dict(otlp_headers or {}), project_name, config, + attachments, ) if self._agno_instrumentor is not None and not self.is_instrumented: raise RuntimeError( @@ -88,6 +94,7 @@ def instrument( otlp_headers, project_name, config, + attachments, ) ): return self @@ -151,6 +158,7 @@ def instrument( delegate, tracer_provider=provider, config=config, + attachments=attachments, raise_exception_on_conflict=True, ) if not delegate.is_instrumented_by_opentelemetry: @@ -204,6 +212,7 @@ def configure_agno_tracing( project_name: str | None = None, *, config: TraceConfig | None = None, + attachments: object | None = None, ) -> AgnoInstrumentorWrapper: """Configure Agno using an existing provider or a complete OTLP traces URL. @@ -219,6 +228,7 @@ def configure_agno_tracing( otlp_headers=otlp_headers, project_name=project_name, config=config, + attachments=attachments, ) diff --git a/python/frameworks/agno/traceai_agno/_wrappers/_install.py b/python/frameworks/agno/traceai_agno/_wrappers/_install.py index 42678cd36..cd0ed6987 100644 --- a/python/frameworks/agno/traceai_agno/_wrappers/_install.py +++ b/python/frameworks/agno/traceai_agno/_wrappers/_install.py @@ -39,6 +39,17 @@ def _wrap_methods( wrap_function_wrapper(target, name, wrapper) +def _attachment_manager(manager: Any) -> Any: + """An enabled shared manager, else None (a disabled or pre-attachments core is a no-op).""" + if manager is None: + return None + try: + from fi_instrumentation.attachments.media_capture import active + except ImportError: + return None + return manager if active(manager) else None + + def _instrument(self: Any, **kwargs: Any) -> None: from agno.agent import _run as agent_run from agno.knowledge.knowledge import Knowledge @@ -75,7 +86,12 @@ def _instrument(self: Any, **kwargs: Any) -> None: ), ) run = _RunWrapper(self._tracer) - model = _ModelWrapper(self._tracer, hide_inputs=bool(config.hide_inputs)) + model = _ModelWrapper( + self._tracer, + hide_inputs=bool(config.hide_inputs), + attachments=_attachment_manager(kwargs.get("attachments")), + trace_configs=(fi_config, config), + ) tool = _FunctionCallWrapper(self._tracer, hide_inputs=bool(config.hide_inputs)) workflow = _WorkflowWrapper(self._tracer) execute = _WorkflowExecuteWrapper(self._tracer) diff --git a/python/frameworks/agno/traceai_agno/_wrappers/_model_wrapper.py b/python/frameworks/agno/traceai_agno/_wrappers/_model_wrapper.py index f69d923eb..422260d03 100644 --- a/python/frameworks/agno/traceai_agno/_wrappers/_model_wrapper.py +++ b/python/frameworks/agno/traceai_agno/_wrappers/_model_wrapper.py @@ -617,9 +617,46 @@ def _usage_attributes(metrics: Any) -> Iterator[Tuple[str, Any]]: class _ModelWrapper: - def __init__(self, tracer: trace_api.Tracer, *, hide_inputs: bool = False) -> None: + def __init__( + self, + tracer: trace_api.Tracer, + *, + hide_inputs: bool = False, + attachments: Any = None, + trace_configs: Tuple[Any, ...] = (), + ) -> None: self._tracer = tracer self._hide_inputs = hide_inputs + # The shared, provider-owned attachment manager (already checked active), or None. + # trace_configs are the FI and OpenInference configs: either one hiding wins. + self._attachments = attachments + self._trace_configs = trace_configs + + def _capture_input(self, span: Any, arguments: Mapping[str, Any]) -> None: + # Before the provider call, so a provider instrumentor sharing the manager reuses it. + if self._attachments is None: + return + try: + from traceai_agno._attachments import capture_agno_input + + capture_agno_input( + self._attachments, span, arguments.get("messages"), + trace_configs=self._trace_configs, + ) + except Exception: + pass + + def _capture_output(self, span: Any, response: Any) -> None: + if self._attachments is None: + return + try: + from traceai_agno._attachments import capture_agno_output + + capture_agno_output( + self._attachments, span, response, trace_configs=self._trace_configs + ) + except Exception: + pass def run( self, @@ -657,10 +694,12 @@ def run( span.set_attribute(LLM_PROVIDER, model.provider) if llm_system := _get_llm_system(model): span.set_attribute(LLM_SYSTEM, llm_system) + self._capture_input(span, arguments) with span_scope(span, _invocation_context(instance, "invoke", wrapped)): response = wrapped(*args, **kwargs) output_message = _parse_model_output(response) + self._capture_output(span, response) span.set_attributes(dict(_output_value_and_mime_type(output_message))) span.set_attributes( @@ -708,6 +747,7 @@ def run_stream( span.set_attribute(LLM_PROVIDER, model.provider) if llm_system := _get_llm_system(model): span.set_attribute(LLM_SYSTEM, llm_system) + self._capture_input(span, arguments) # Token usage will be set after streaming completes based on final response responses = [] @@ -769,10 +809,12 @@ async def arun( span.set_attribute(LLM_PROVIDER, model.provider) if llm_system := _get_llm_system(model): span.set_attribute(LLM_SYSTEM, llm_system) + self._capture_input(span, arguments) with span_scope(span, _invocation_context(instance, "ainvoke", wrapped)): response = await wrapped(*args, **kwargs) output_message = _parse_model_output(response) + self._capture_output(span, response) span.set_attributes( dict(_usage_attributes(getattr(response, "response_usage", None))) @@ -822,6 +864,7 @@ async def arun_stream( span.set_attribute(LLM_PROVIDER, model.provider) if llm_system := _get_llm_system(model): span.set_attribute(LLM_SYSTEM, llm_system) + self._capture_input(span, arguments) # Token usage will be set after streaming completes based on final response responses = [] diff --git a/python/frameworks/langchain/traceai_langchain/__init__.py b/python/frameworks/langchain/traceai_langchain/__init__.py index df30b52be..61e8be978 100644 --- a/python/frameworks/langchain/traceai_langchain/__init__.py +++ b/python/frameworks/langchain/traceai_langchain/__init__.py @@ -35,6 +35,23 @@ _MODULE = "langchain_core" +def _attachment_manager(kwargs: Any) -> Any: + """The shared manager passed as ``attachments`` (or auto's ``attachment_manager``), if it can admit. + + A missing, disabled or pre-attachments core leaves tracing exactly as before. + """ + manager = kwargs.get("attachments") + if manager is None: + manager = kwargs.get("attachment_manager") + if manager is None: + return None + try: + from fi_instrumentation.attachments.media_capture import active + except ImportError: + return None + return manager if active(manager) else None + + class LangChainInstrumentor(BaseInstrumentor): # type: ignore """ An instrumentor for LangChain @@ -63,7 +80,11 @@ def _instrument(self, **kwargs: Any) -> None: trace_api.get_tracer(__name__, __version__, tracer_provider), config=config, ) - self._tracer: Optional[LangChainFiTracer] = LangChainFiTracer(self.fi_tracer) + self._tracer: Optional[LangChainFiTracer] = LangChainFiTracer( + self.fi_tracer, + attachments=_attachment_manager(kwargs), + trace_config=config, + ) self._original_callback_manager_init = BaseCallbackManager.__init__ wrap_function_wrapper( module="langchain_core.callbacks", diff --git a/python/frameworks/langchain/traceai_langchain/_attachments.py b/python/frameworks/langchain/traceai_langchain/_attachments.py new file mode 100644 index 000000000..87ac18004 --- /dev/null +++ b/python/frameworks/langchain/traceai_langchain/_attachments.py @@ -0,0 +1,182 @@ +"""LangChain typed message media through the shared, provider-owned attachment manager. + +Mirrors ``@traceai/framework-attachments`` (dialect ``langchain``). Only the typed +content arrays of chat-model messages are observed, in the serialized form the +tracer receives (``dumpd`` messages under ``kwargs.content``): + +- OpenAI-style ``image_url`` / ``file`` / ``input_audio`` parts with inline data; +- LangChain 0.3 standard blocks (``source_type: "base64"``); +- LangChain 1.x content blocks (``base64`` + ``mime_type``); +- Anthropic-style ``source: {"type": "base64"}`` blocks. + +Remote URLs go to the core's ``RemoteUrl`` policy and are never fetched; provider +file IDs and anything else are ignored. Bytes identical to ones a provider +instrumentor (or another observer) already uploaded in the same trace are not +uploaded again (``media_capture`` claim table). + +This module imports ``fi_instrumentation.attachments``; the tracer imports it only +when an enabled attachment manager is passed. +""" + +from types import MappingProxyType +from typing import Any, Iterator, List, Mapping, Optional + +from fi_instrumentation.attachments.media_capture import ( + AUDIO_MIMES, + DOCUMENT_MIMES, + EMPTY, + IMAGE_MIMES, + MAX_FRAMEWORK_BYTES, + MAX_FRAMEWORK_PARTS, + MediaPart, + Occurrence, + capture_parts, + get, + items, +) + +__all__ = ("capture_langchain_messages", "langchain_attachment_capabilities") + +langchain_attachment_capabilities = MappingProxyType({ + "schema_version": 1, + "operations": ("chat_model.input", "chat_model.output", "tool_message.input"), + "sources": ("inline_base64", "data_uri"), + "unsupported": ("remote_url_fetch", "provider_file", "local_path", "streamed_output", "audio_output"), + "max_part_bytes": MAX_FRAMEWORK_BYTES, + "max_parts_per_span": MAX_FRAMEWORK_PARTS, +}) + +_ROLES = {"HumanMessage": "user", "AIMessage": "assistant", "SystemMessage": "system", "ToolMessage": "tool", + "HumanMessageChunk": "user", "AIMessageChunk": "assistant", "ToolMessageChunk": "tool"} +_AUDIO_FORMATS = {"wav": "audio/wav", "mp3": "audio/mpeg"} + + +def _family(mime: Any, default: tuple) -> Optional[tuple]: + """The admitted MIME list for a declared type, the declared type first; None when unsupported.""" + if mime is None: + return default + if not isinstance(mime, str): + return None + for family in (IMAGE_MIMES, DOCUMENT_MIMES, AUDIO_MIMES): + if mime in family: + return (mime,) + tuple(m for m in family if m != mime) + return None + + +def _purpose(mimes: tuple, direction: str, role: str) -> str: + if direction == "output": + return "model_output" + if role == "tool": + return "tool_result" + return "document" if mimes[0] == "application/pdf" else "model_input" + + +def _part(block: Any, message_index: int, part_index: int, role: str, direction: str, + tool_call_id: Optional[str]) -> Optional[MediaPart]: + kind = get(block, "type") + found = None # (source kind, reader, mimes, raw) + if kind == "image_url": + image = get(block, "image_url") + url = image if isinstance(image, str) else get(image, "url") + if isinstance(url, str) and url.startswith("data:"): + found = ("inline", lambda: url, IMAGE_MIMES, False) + elif isinstance(url, str): + found = ("url", lambda: url, IMAGE_MIMES, True) + elif kind == "input_audio": + audio = get(block, "input_audio") + mimes = _family(_AUDIO_FORMATS.get(get(audio, "format")), ()) + if mimes: + found = ("inline", lambda: get(audio, "data"), mimes, True) + elif kind == "file" and isinstance(get(block, "file"), Mapping): + data = get(get(block, "file"), "file_data") + if isinstance(data, str) and data.startswith("data:"): + found = ("inline", lambda: data, DOCUMENT_MIMES, False) + elif kind in ("image", "audio", "file"): + default = {"image": IMAGE_MIMES, "audio": AUDIO_MIMES, "file": DOCUMENT_MIMES}[kind] + source = get(block, "source") + if isinstance(source, Mapping): + # Anthropic-style block. + if get(source, "type") == "base64": + mimes = _family(get(source, "media_type"), default) + if mimes: + found = ("inline", lambda: get(source, "data"), mimes, True) + elif get(source, "type") == "url" and isinstance(get(source, "url"), str): + found = ("url", lambda: get(source, "url"), default, True) + elif get(block, "source_type") == "base64": + mimes = _family(get(block, "mime_type"), default) # LangChain 0.3 standard block + if mimes: + found = ("inline", lambda: get(block, "data"), mimes, True) + elif get(block, "source_type") == "url" and isinstance(get(block, "url"), str): + found = ("url", lambda: get(block, "url"), default, True) + elif get(block, "base64") is not None: + mimes = _family(get(block, "mime_type"), default) # LangChain 1.x content block + if mimes: + found = ("inline", lambda: get(block, "base64"), mimes, True) + elif isinstance(get(block, "url"), str): + found = ("url", lambda: get(block, "url"), default, True) + if found is None: + return None + source_kind, read, mimes, raw = found + return MediaPart(source_kind, read, mimes, _purpose(mimes, direction, role), message_index, part_index, + role=role, raw=raw, tool_call_id=tool_call_id) + + +def _message(message: Any) -> tuple: + """(role, content, tool_call_id) of a serialized (``dumpd``) or plain message.""" + kwargs = get(message, "kwargs") + if isinstance(kwargs, Mapping): + ident = get(message, "id") + name = ident[-1] if isinstance(ident, list) and ident and isinstance(ident[-1], str) else "" + role = _ROLES.get(name, "unknown") + if name.startswith("ChatMessage"): + native = get(kwargs, "role") + role = native if native in ("user", "assistant", "tool", "system") else "unknown" + return role, get(kwargs, "content"), get(kwargs, "tool_call_id") + role = get(message, "type") + role = {"human": "user", "ai": "assistant", "system": "system", "tool": "tool"}.get(role, "unknown") + return role, get(message, "content"), get(message, "tool_call_id") + + +def _parts(messages: List[Any], direction: str) -> Iterator[MediaPart]: + for message_index, message in enumerate(items(messages)): + role, content, tool_call_id = _message(message) + tool_call_id = tool_call_id if role == "tool" and isinstance(tool_call_id, str) else None + for part_index, block in enumerate(items(content)): + if isinstance(block, Mapping): + part = _part(block, message_index, part_index, role, direction, tool_call_id) + if part is not None: + yield part + + +def capture_langchain_messages(manager, span, messages: Any, *, direction: str, trace_config=None, + capture=None) -> Mapping[str, Occurrence]: + """Capture typed media of one chat-model run's first message batch or generations. + + ``messages`` is a list of serialized messages. Returns admitted positions keyed + ``"{direction}/{message_index}/{part_index}"``; empty when no manager, disabled, + unsampled or denied. Never raises and never changes ``messages``. + """ + try: + if not isinstance(messages, list): + return EMPTY + return capture_parts(manager, span, "langchain", _parts(messages, direction), direction=direction, + trace_configs=(trace_config,), capture=capture) + except Exception: + return EMPTY + + +def run_messages(run: Any, direction: str) -> Optional[list]: + """The first message batch (input) or the first generation's messages (output) of a run.""" + try: + if direction == "input": + batches = get(get(run, "inputs"), "messages") + first = batches[0] if isinstance(batches, list) and batches else None + return first if isinstance(first, list) else None + generations = get(get(run, "outputs"), "generations") + first = generations[0] if isinstance(generations, list) and generations else None + if not isinstance(first, list): + return None + return [message for message in (get(g, "message") for g in first[:MAX_FRAMEWORK_PARTS]) + if message is not None] + except Exception: + return None diff --git a/python/frameworks/langchain/traceai_langchain/_tracer.py b/python/frameworks/langchain/traceai_langchain/_tracer.py index e7d4a7781..d3c99ae1a 100644 --- a/python/frameworks/langchain/traceai_langchain/_tracer.py +++ b/python/frameworks/langchain/traceai_langchain/_tracer.py @@ -178,8 +178,18 @@ def _send_span_data_to_api( except Exception as e: logger.exception(f"Error sending span data to API: {e}") - def __init__(self, fi_tracer: "FITracer", *args: Any, **kwargs: Any) -> None: + def __init__( + self, + fi_tracer: "FITracer", + *args: Any, + attachments: Any = None, + trace_config: Optional[TraceConfig] = None, + **kwargs: Any, + ) -> None: super().__init__(*args, **kwargs) + # The shared, provider-owned attachment manager (already checked active), or None. + self._attachments = attachments + self._attachment_config = trace_config if TYPE_CHECKING: # check that `run_map` still exists in parent class assert self.run_map @@ -239,6 +249,9 @@ def _start_trace(self, run: Run) -> None: # token = context_api.attach(context) with self._lock: self._spans_by_run[run.id] = span + # Chat-model input media is captured BEFORE the provider call, so a provider + # instrumentor sharing the manager reuses this upload instead of a second one. + self._capture_media(span, run, "input") # Send initial span data with input input_data = None @@ -279,6 +292,9 @@ def _end_trace(self, run: Run) -> None: is_interrupt=is_interrupt, error=error, ) + # Never from a failed run: its outputs can carry provider error bodies. + if run.error is None: + self._capture_media(span, run, "output") # Send final span data with both input and output output_data = None @@ -306,6 +322,22 @@ def _end_trace(self, run: Run) -> None: def _persist_run(self, run: Run) -> None: pass + def _capture_media(self, span: Span, run: Run, direction: str) -> None: + if self._attachments is None or run.run_type != "llm": + return + try: + from traceai_langchain._attachments import capture_langchain_messages, run_messages + + messages = run_messages(run, direction) + if messages: + capture_langchain_messages( + self._attachments, span, messages, direction=direction, + trace_config=self._attachment_config, + ) + except Exception: + # Media observation must never affect the run or log source-bearing data. + pass + def _handle_error( self, error: BaseException, diff --git a/python/frameworks/llama_index/traceai_llamaindex/__init__.py b/python/frameworks/llama_index/traceai_llamaindex/__init__.py index 1ca5eb3f7..ddd59556f 100644 --- a/python/frameworks/llama_index/traceai_llamaindex/__init__.py +++ b/python/frameworks/llama_index/traceai_llamaindex/__init__.py @@ -20,6 +20,23 @@ logger.addHandler(logging.NullHandler()) +def _attachment_manager(kwargs: Any) -> Any: + """The shared manager passed as ``attachments`` (or auto's ``attachment_manager``), if it can admit. + + A missing, disabled or pre-attachments core leaves tracing exactly as before. + """ + manager = kwargs.get("attachments") + if manager is None: + manager = kwargs.get("attachment_manager") + if manager is None: + return None + try: + from fi_instrumentation.attachments.media_capture import active + except ImportError: + return None + return manager if active(manager) else None + + class LlamaIndexInstrumentor(BaseInstrumentor): # type: ignore """ An instrumentor for LlamaIndex @@ -83,7 +100,9 @@ def _instrument(self, **kwargs: Any) -> None: from ._handler import EventHandler, instrument_workflow_runs from ._stream import StreamInstrumentation - self._event_handler = EventHandler(tracer=tracer, config=config) + self._event_handler = EventHandler( + tracer=tracer, config=config, attachments=_attachment_manager(kwargs) + ) self._span_handler = self._event_handler._span_handler dispatcher = get_dispatcher() for span_handler in dispatcher.span_handlers: diff --git a/python/frameworks/llama_index/traceai_llamaindex/_attachments.py b/python/frameworks/llama_index/traceai_llamaindex/_attachments.py new file mode 100644 index 000000000..ae1a9054b --- /dev/null +++ b/python/frameworks/llama_index/traceai_llamaindex/_attachments.py @@ -0,0 +1,123 @@ +"""LlamaIndex typed message media through the shared, provider-owned attachment manager. + +Mirrors ``@traceai/framework-attachments`` (dialect ``llamaindex``). Only +``ChatMessage.blocks`` (llama-index-core 0.12.3+) of chat-model input and of the +non-streaming chat response are observed: + +- ``ImageBlock.image``, ``AudioBlock.audio`` and ``DocumentBlock.data`` held as + bytes (LlamaIndex stores them base64-encoded; raw bytes are accepted too); +- the magic bytes must match the admitted type. + +An ``IOBase`` source is never read, a ``path`` is never opened, and a ``url`` goes +to the core's ``RemoteUrl`` policy without being fetched. Bytes identical to ones +a provider instrumentor already uploaded in the same trace are not uploaded again. + +This module imports ``fi_instrumentation.attachments``; the handler imports it only +when an enabled attachment manager is passed. +""" + +from types import MappingProxyType +from typing import Any, Iterator, Mapping, Optional, Sequence + +from fi_instrumentation.attachments.media_capture import ( + AUDIO_MIMES, + DOCUMENT_MIMES, + EMPTY, + IMAGE_MIMES, + MAX_FRAMEWORK_BYTES, + MAX_FRAMEWORK_PARTS, + MediaPart, + Occurrence, + capture_parts, + looks_base64, +) + +__all__ = ("capture_llamaindex_messages", "llamaindex_attachment_capabilities") + +llamaindex_attachment_capabilities = MappingProxyType({ + "schema_version": 1, + "operations": ("llm.chat.input", "llm.chat.output"), + "sources": ("block_bytes",), + "unsupported": ("io_source", "local_path", "remote_url_fetch", "video"), + "min_core": "0.12.3", + "max_part_bytes": MAX_FRAMEWORK_BYTES, + "max_parts_per_span": MAX_FRAMEWORK_PARTS, +}) + +_ROLES = ("user", "assistant", "tool", "system") +_AUDIO_FORMATS = {"wav": "audio/wav", "mp3": "audio/mpeg", "ogg": "audio/ogg", "flac": "audio/flac", + "webm": "audio/webm"} +# block_type -> (data field, MIME field, default family) +_BLOCKS = {"image": ("image", "image_mimetype", IMAGE_MIMES), "audio": ("audio", None, AUDIO_MIMES), + "document": ("data", "document_mimetype", DOCUMENT_MIMES)} + + +def _field(block: Any, name: str) -> Any: + """A declared model field only: never a property, resolver or ``__getattr__``.""" + try: + values = vars(block) + except TypeError: + return None + return values.get(name) + + +def _ordered(mime: Any, family: tuple) -> Optional[tuple]: + if mime is None: + return family + if isinstance(mime, str) and mime in family: + return (mime,) + tuple(m for m in family if m != mime) + return None + + +def _role(message: Any) -> str: + role = _field(message, "role") + role = getattr(role, "value", role) + return role if role in _ROLES else "unknown" + + +def _parts(messages: Sequence[Any], direction: str) -> Iterator[MediaPart]: + for message_index, message in enumerate(list(messages)[:MAX_FRAMEWORK_PARTS]): + role = _role(message) + blocks = _field(message, "blocks") + if not isinstance(blocks, list): + continue + for part_index, block in enumerate(blocks[:MAX_FRAMEWORK_PARTS]): + spec = _BLOCKS.get(_field(block, "block_type")) + if spec is None: + continue + data_field, mime_field, family = spec + if mime_field is not None: + mimes = _ordered(_field(block, mime_field), family) + else: + fmt = _field(block, "format") + mimes = _ordered(_AUDIO_FORMATS.get(fmt), family) if fmt is not None else family + if not mimes: + continue + purpose = ("model_output" if direction == "output" else "tool_result" if role == "tool" + else "document" if family is DOCUMENT_MIMES else "model_input") + data = _field(block, data_field) + url = _field(block, "url") + if isinstance(data, (bytes, bytearray)): + # Base64 ASCII when it parses strictly; otherwise complete raw bytes. + kind = "inline" if looks_base64(data) else "bytes" + yield MediaPart(kind, lambda d=data: d, mimes, purpose, message_index, part_index, role=role, + raw=True) + elif data is None and url is not None and _field(block, "path") is None: + yield MediaPart("url", lambda u=url: str(u), mimes, purpose, message_index, part_index, + role=role) + # IOBase sources and paths: never read or opened. + + +def capture_llamaindex_messages(manager, span, messages: Any, *, direction: str, trace_config=None, + capture=None) -> Mapping[str, Occurrence]: + """Capture typed block media of chat messages; returns ``"{dir}/{msg}/{part}"`` positions. + + Never raises and never changes ``messages``. + """ + try: + if not isinstance(messages, (list, tuple)): + return EMPTY + return capture_parts(manager, span, "llamaindex", _parts(messages, direction), direction=direction, + trace_configs=(trace_config,), capture=capture) + except Exception: + return EMPTY diff --git a/python/frameworks/llama_index/traceai_llamaindex/_handler.py b/python/frameworks/llama_index/traceai_llamaindex/_handler.py index 24e624a7e..b9b2a92e3 100644 --- a/python/frameworks/llama_index/traceai_llamaindex/_handler.py +++ b/python/frameworks/llama_index/traceai_llamaindex/_handler.py @@ -241,6 +241,7 @@ class Config: _cancel_reason: Optional[str] = PrivateAttr() _workflow_run: bool = PrivateAttr() _run_handler: Any = PrivateAttr() + _attachments: Any = PrivateAttr() def __init__( self, @@ -248,9 +249,12 @@ def __init__( config: Optional[TraceConfig] = None, span_kind: Optional[str] = None, parent: Optional["_Span"] = None, + attachments: Any = None, **kwargs: Any, ) -> None: super().__init__(**kwargs) + # The shared, provider-owned attachment manager (already checked active), or None. + self._attachments = attachments self._config = config if config is not None else TraceConfig() self._otel_span = otel_span self._active = otel_span.is_recording() @@ -650,6 +654,8 @@ def _(self, event: LLMChatStartEvent) -> None: GEN_AI_INPUT_MESSAGES, *event.messages, ) + # Before the provider call, so a provider instrumentor sharing the manager reuses it. + self._capture_media(event.messages, "input") self[INPUT_VALUE] = safe_json_dumps(event.messages) self[INPUT_VALUE] = safe_json_dumps(_to_dict(event)) @@ -667,6 +673,7 @@ def _(self, event: LLMChatEndEvent) -> None: GEN_AI_OUTPUT_MESSAGES, response.message, ) + self._capture_media([response.message], "output") self[OUTPUT_VALUE] = safe_json_dumps( { "message": _to_dict(response.message), @@ -823,6 +830,20 @@ def _process_messages( f"{prefix}.{i}.{MESSAGE_CONTENTS}.{index}.{MESSAGE_CONTENT_AUDIO}" ] = (block.audio or block.path or block.url) + def _capture_media(self, messages: Any, direction: str) -> None: + if self._attachments is None or not self._active: + return + try: + from ._attachments import capture_llamaindex_messages + + capture_llamaindex_messages( + self._attachments, self._otel_span, list(messages or ()), + direction=direction, trace_config=self._config, + ) + except Exception: + # Media observation must never affect the call or log source-bearing data. + pass + def _process_query_type(self, query: Optional[QueryType]) -> None: if query is None: return @@ -956,9 +977,13 @@ class _SpanHandler(BaseSpanHandler[_Span], extra="allow"): _otel_tracer: Tracer = PrivateAttr() _config: TraceConfig = PrivateAttr() _export_queue: _ExportQueue = PrivateAttr() + _attachments: Any = PrivateAttr() - def __init__(self, tracer: Tracer, config: Optional[TraceConfig] = None) -> None: + def __init__( + self, tracer: Tracer, config: Optional[TraceConfig] = None, attachments: Any = None + ) -> None: super().__init__() + self._attachments = attachments self._config = config if config is not None else TraceConfig() self._otel_tracer = tracer self._export_queue = _ExportQueue() @@ -994,6 +1019,7 @@ def new_span( config=self._config, span_kind=_init_span_kind(instance), parent=parent, + attachments=self._attachments, id_=id_, parent_id=parent_span_id, ) @@ -1234,9 +1260,11 @@ def uninstrument_workflow_runs(patched: Iterable[Tuple[type, str]]) -> None: class EventHandler(BaseEventHandler, extra="allow"): _span_handler: _SpanHandler = PrivateAttr() - def __init__(self, tracer: Tracer, config: Optional[TraceConfig] = None) -> None: + def __init__( + self, tracer: Tracer, config: Optional[TraceConfig] = None, attachments: Any = None + ) -> None: super().__init__() - self._span_handler = _SpanHandler(tracer=tracer, config=config) + self._span_handler = _SpanHandler(tracer=tracer, config=config, attachments=attachments) def handle(self, event: BaseEvent, **kwargs: Any) -> Any: if context_api.get_value(_SUPPRESS_INSTRUMENTATION_KEY): diff --git a/python/frameworks/pydantic-ai/traceai_pydantic_ai/_attachments.py b/python/frameworks/pydantic-ai/traceai_pydantic_ai/_attachments.py new file mode 100644 index 000000000..a6a3813dd --- /dev/null +++ b/python/frameworks/pydantic-ai/traceai_pydantic_ai/_attachments.py @@ -0,0 +1,158 @@ +"""Pydantic AI typed media through the shared, provider-owned attachment manager. + +Observed at SDK execution boundaries only: + +- model request span, input: ``BinaryContent`` in ``UserPromptPart.content``, + in ``ToolReturnPart.content`` (with ``tool_call_id``) and in re-sent + ``FilePart`` history; ``ImageUrl`` / ``AudioUrl`` / ``DocumentUrl`` go to the + core's ``RemoteUrl`` policy and are never fetched or resolved here; +- model request span, output: ``FilePart`` content of a successful, + non-streaming response (never an error body); +- tool call span: ``BinaryContent`` a tool returned. + +``BinaryContent.data`` must be complete bytes whose magic bytes match its +declared ``media_type``. Videos, uploaded provider files and unknown content +objects are not captured, and nothing is serialized to look for media. Bytes +identical to ones a provider instrumentor (or the tool span) already uploaded in +the same trace are not uploaded again. +""" + +import dataclasses +from types import MappingProxyType +from typing import Any, Iterator, Mapping, Optional + +from fi_instrumentation.attachments.media_capture import ( + AUDIO_MIMES, + DOCUMENT_MIMES, + EMPTY, + IMAGE_MIMES, + MAX_FRAMEWORK_BYTES, + MAX_FRAMEWORK_PARTS, + MediaPart, + Occurrence, + capture_parts, +) + +__all__ = ("PydanticAIMedia", "pydantic_ai_attachment_capabilities") + +pydantic_ai_attachment_capabilities = MappingProxyType({ + "schema_version": 1, + "operations": ("model.request.input", "model.request.output", "tool.call.result", + "model.request_stream.input"), + "sources": ("binary_content",), + "unsupported": ("remote_url_fetch", "uploaded_file", "video", "streamed_output", "error_body"), + "max_part_bytes": MAX_FRAMEWORK_BYTES, + "max_parts_per_span": MAX_FRAMEWORK_PARTS, +}) + +_URL_FAMILIES = {"image-url": IMAGE_MIMES, "audio-url": AUDIO_MIMES, "document-url": DOCUMENT_MIMES} + + +def _field(value: Any, name: str) -> Any: + """A declared field only: never a property, resolver or ``__getattr__``.""" + try: + return vars(value).get(name) + except TypeError: + pass + try: + if dataclasses.is_dataclass(value) and not isinstance(value, type) and \ + any(f.name == name for f in dataclasses.fields(value)): + return getattr(value, name) + except Exception: + pass + return None + + +def _family(mime: Any) -> Optional[tuple]: + for family in (IMAGE_MIMES, AUDIO_MIMES, DOCUMENT_MIMES): + if mime in family: + return (mime,) + tuple(m for m in family if m != mime) + return None + + +def _content_parts(content: Any, message_index: int, start: int, role: str, purpose: str, + tool_call_id: Optional[str]) -> Iterator[MediaPart]: + values = content if isinstance(content, (list, tuple)) else [content] + index = start + for value in values[:MAX_FRAMEWORK_PARTS]: + kind = _field(value, "kind") + if kind == "binary": + data = _field(value, "data") + mimes = _family(_field(value, "media_type")) + if mimes and isinstance(data, (bytes, bytearray)): + part_purpose = "document" if purpose == "model_input" and mimes[0] in DOCUMENT_MIMES else purpose + yield MediaPart("bytes", lambda d=data: d, mimes, part_purpose, message_index, index, role=role, + tool_call_id=tool_call_id) + index += 1 + elif kind in _URL_FAMILIES: + url = _field(value, "url") + if isinstance(url, str): + yield MediaPart("url", lambda u=url: u, _URL_FAMILIES[kind], purpose, message_index, index, + role=role, tool_call_id=tool_call_id) + index += 1 + elif kind is not None: + index += 1 # video-url, uploaded files, ...: never captured + + +def _request_parts(messages: Any) -> Iterator[MediaPart]: + for message_index, message in enumerate(list(messages)[:MAX_FRAMEWORK_PARTS]): + index = 0 + for part in list(_field(message, "parts") or ())[:MAX_FRAMEWORK_PARTS]: + part_kind = _field(part, "part_kind") + if part_kind == "user-prompt": + content = _field(part, "content") + if isinstance(content, str): + continue + parts = list(_content_parts(content, message_index, index, "user", "model_input", None)) + elif part_kind == "tool-return": + tool_call_id = _field(part, "tool_call_id") + parts = list(_content_parts(_field(part, "content"), message_index, index, "tool", "tool_result", + tool_call_id if isinstance(tool_call_id, str) else None)) + elif part_kind == "file": + # A generated file re-sent as history is model input now. + parts = list(_content_parts(_field(part, "content"), message_index, index, "assistant", + "model_input", None)) + else: + continue + yield from parts + index += max(len(parts), 1) + + +def _response_parts(response: Any) -> Iterator[MediaPart]: + index = 0 + for part in list(_field(response, "parts") or ())[:MAX_FRAMEWORK_PARTS]: + if _field(part, "part_kind") == "file": + yield from _content_parts(_field(part, "content"), 0, index, "assistant", "model_output", None) + index += 1 + + +class PydanticAIMedia: + """The instrumentor's binding: manager plus the caller's TraceConfig (deny-only).""" + + __slots__ = ("manager", "trace_config") + + def __init__(self, manager: Any, trace_config: Any) -> None: + self.manager, self.trace_config = manager, trace_config + + def _capture(self, span, parts, direction: str) -> Mapping[str, Occurrence]: + if span is None: + return EMPTY + try: + return capture_parts(self.manager, span, "pydantic_ai", parts, direction=direction, + trace_configs=(self.trace_config,)) + except Exception: + return EMPTY + + def request(self, span, messages: Any) -> Mapping[str, Occurrence]: + if not isinstance(messages, (list, tuple)): + return EMPTY + return self._capture(span, _request_parts(messages), "input") + + def response(self, span, response: Any) -> Mapping[str, Occurrence]: + # Historical Model.request returns (ModelResponse, Usage). + response = response[0] if isinstance(response, tuple) and response else response + return self._capture(span, _response_parts(response), "output") + + def tool_result(self, span, result: Any, tool_call_id: Any) -> Mapping[str, Occurrence]: + tool_call_id = tool_call_id if isinstance(tool_call_id, str) else None + return self._capture(span, _content_parts(result, 0, 0, "tool", "tool_result", tool_call_id), "input") diff --git a/python/frameworks/pydantic-ai/traceai_pydantic_ai/_execution_wrappers.py b/python/frameworks/pydantic-ai/traceai_pydantic_ai/_execution_wrappers.py index e39bdded9..bde7a0bf6 100644 --- a/python/frameworks/pydantic-ai/traceai_pydantic_ai/_execution_wrappers.py +++ b/python/frameworks/pydantic-ai/traceai_pydantic_ai/_execution_wrappers.py @@ -126,12 +126,29 @@ async def __aexit__(self, exc_type, exc_value, traceback): return await self.manager.__aexit__(exc_type, exc_value, traceback) -def wrap_model_request(original, tracer, streaming=False): +def _request_messages(args, kwargs): + return args[0] if args else kwargs.get("messages") + + +def _capture(media, method, span, *values): + # Media observation must never affect the call or log source-bearing data. + if media is None or span is None: + return + try: + getattr(media, method)(span, *values) + except Exception: + pass + + +def wrap_model_request(original, tracer, streaming=False, media=None): + """``media`` is the instrumentor's attachment binding (``PydanticAIMedia``) or None.""" if streaming: @functools.wraps(original) @asynccontextmanager async def stream_wrapper(self, *args, **kwargs): with _model_request_span(self, tracer, current=False) as span: + # Input only: streamed output is not captured. + _capture(media, "request", span, _request_messages(args, kwargs)) manager = original(self, *args, **kwargs) async with _ModelStreamContext(manager, self, span) as response: yield response @@ -141,13 +158,17 @@ async def stream_wrapper(self, *args, **kwargs): @functools.wraps(original) async def wrapper(self, *args, **kwargs): with _model_request_span(self, tracer) as span: + # Before the provider call, so a provider instrumentor sharing the manager reuses it. + _capture(media, "request", span, _request_messages(args, kwargs)) response = await original(self, *args, **kwargs) _record_model_usage(span, response) + # Only a successful response: an error never reaches this line. + _capture(media, "response", span, response) return response return wrapper -def wrap_tool_call(original, tracer, legacy=False): +def wrap_tool_call(original, tracer, legacy=False, media=None): signature = inspect.signature(original) @functools.wraps(original) @@ -162,6 +183,7 @@ async def wrapper(self, *args, **kwargs): description = getattr(self, "description", None) arguments = getattr(bound.get("message"), "args", None) retry = getattr(self, "current_retry", None) + tool_call_id = getattr(bound.get("message"), "tool_call_id", None) else: name = bound.get("name", "unknown") tool = bound.get("tool") @@ -169,6 +191,7 @@ async def wrapper(self, *args, **kwargs): description = getattr(definition, "description", None) arguments = bound.get("tool_args") retry = getattr(bound.get("ctx"), "retry", None) + tool_call_id = getattr(bound.get("ctx"), "tool_call_id", None) attributes = { A.SPAN_KIND: PydanticAISpanKind.TOOL_CALL.value, A.GEN_AI_TOOL_NAME: name, @@ -207,6 +230,7 @@ async def wrapper(self, *args, **kwargs): span.set_status(Status(StatusCode.ERROR, message)) else: span.set_status(Status(StatusCode.OK)) + _capture(media, "tool_result", span, content, tool_call_id) return result finally: span.set_attribute(A.TOOL_DURATION_MS, (time.perf_counter() - start) * 1000) diff --git a/python/frameworks/pydantic-ai/traceai_pydantic_ai/_instrumentor.py b/python/frameworks/pydantic-ai/traceai_pydantic_ai/_instrumentor.py index 5be76fb43..6facf1a1f 100644 --- a/python/frameworks/pydantic-ai/traceai_pydantic_ai/_instrumentor.py +++ b/python/frameworks/pydantic-ai/traceai_pydantic_ai/_instrumentor.py @@ -22,6 +22,27 @@ _instruments = ("pydantic-ai >= 0.0.1",) +def _attachment_binding(kwargs, config): + """The shared manager passed as ``attachments`` (or auto's ``attachment_manager``), if it can admit. + + A missing, disabled or pre-attachments core leaves tracing exactly as before. + """ + manager = kwargs.get("attachments") + if manager is None: + manager = kwargs.get("attachment_manager") + if manager is None: + return None + try: + from fi_instrumentation.attachments.media_capture import active + except ImportError: + return None + if not active(manager): + return None + from ._attachments import PydanticAIMedia + + return PydanticAIMedia(manager, config) + + class PydanticAIInstrumentor(BaseInstrumentor): """OpenTelemetry Instrumentor for Pydantic AI. @@ -67,6 +88,7 @@ def __init__(self): self._execution_patches = [] self._legacy_usage_original = None self._use_builtin: bool = False + self._media = None def instrumentation_dependencies(self) -> Collection[str]: """Return the instrumentation dependencies.""" @@ -105,6 +127,7 @@ def _instrument(self, **kwargs: Any) -> None: # Retain the option for compatibility, but never bypass FI's policy. self._use_builtin = kwargs.get("use_builtin", False) + self._media = _attachment_binding(kwargs, config) # Try to import and patch Pydantic AI try: @@ -198,7 +221,8 @@ def patch_model(cls): if callable(original) and not getattr(original, "__isabstractmethod__", False): self._install_execution_patch( cls, name, wrap_model_request( - original, self._tracer, streaming=name == "request_stream")) + original, self._tracer, streaming=name == "request_stream", + media=self._media)) for subclass in cls.__subclasses__(): patch_model(subclass) @@ -223,11 +247,12 @@ def init_subclass(cls, **kwargs): from pydantic_ai.tools import Tool if isinstance(Tool, type) and hasattr(Tool, "run"): self._install_execution_patch( - Tool, "run", wrap_tool_call(Tool.run, self._tracer, legacy=True)) + Tool, "run", wrap_tool_call(Tool.run, self._tracer, legacy=True, media=self._media)) else: if isinstance(FunctionToolset, type): self._install_execution_patch( - FunctionToolset, "call_tool", wrap_tool_call(FunctionToolset.call_tool, self._tracer)) + FunctionToolset, "call_tool", + wrap_tool_call(FunctionToolset.call_tool, self._tracer, media=self._media)) def _install_execution_patch(self, cls, name, replacement): original = cls.__dict__.get(name) @@ -394,6 +419,7 @@ def _uninstrument(self, **kwargs: Any) -> None: self._restore_execution_methods() self._is_instrumented = False self._tracer = None + self._media = None logger.info("Pydantic AI instrumentation disabled") @@ -476,6 +502,7 @@ def instrument_pydantic_ai( tracer_provider: Optional[Any] = None, use_builtin: bool = False, config: Optional[TraceConfig] = None, + attachments: Optional[Any] = None, ) -> PydanticAIInstrumentor: """Convenience function to instrument Pydantic AI. @@ -483,6 +510,8 @@ def instrument_pydantic_ai( tracer_provider: Optional OpenTelemetry tracer provider use_builtin: Deprecated compatibility flag; uses FI-aware wrappers config: Content privacy policy; defaults resolve from the environment + attachments: The provider-owned attachment manager + (``register(...).attachment_manager``); media is captured only when enabled Returns: The instrumentor instance @@ -503,6 +532,8 @@ def instrument_pydantic_ai( kwargs["use_builtin"] = use_builtin if config is not None: kwargs["config"] = config + if attachments is not None: + kwargs["attachments"] = attachments instrumentor.instrument(**kwargs) return instrumentor diff --git a/python/tests/attachments_frameworks/conftest.py b/python/tests/attachments_frameworks/conftest.py new file mode 100644 index 000000000..2a340382a --- /dev/null +++ b/python/tests/attachments_frameworks/conftest.py @@ -0,0 +1,56 @@ +"""Framework attachment tests reuse the provider lane's loopback harness (synthetic data only). + +- ``f01``: TLS loopback F01 attachment server; ``provider``: plain-HTTP loopback that + answers like the OpenAI API, so real framework + real provider SDK paths run end to end; +- ``spans``: an OpenTelemetry provider with an in-memory exporter. +""" + +import importlib.util +import sys +from pathlib import Path + +import pytest + +pytest.importorskip("fi_instrumentation.attachments.media_capture") + +sys.path.insert(0, str(Path(__file__).parent)) + +_spec = importlib.util.spec_from_file_location( + "_attachments_provider_harness", Path(__file__).parent.parent / "attachments_providers" / "conftest.py") +_harness = importlib.util.module_from_spec(_spec) +sys.modules[_spec.name] = _harness +_spec.loader.exec_module(_harness) + +# Fixtures (pytest registers them from this module's namespace) and helpers. +clean_env = _harness.clean_env +f01 = _harness.f01 +provider = _harness.provider +spans = _harness.spans +PDF = _harness.PDF +b64 = _harness.b64 +flush = _harness.flush +png = _harness.png +refs = _harness.refs +wait_uploads = _harness.wait_uploads +wav = _harness.wav + + +def jpeg(seed=0) -> bytes: + """Bytes with a JPEG signature; only the container magic is checked by the SDK side.""" + return b"\xff\xd8\xff\xe0" + bytes((seed + i) % 256 for i in range(60)) + b"\xff\xd9" + + +def content_puts(f01): + """Content PUT bodies (actual uploaded bytes) in arrival order.""" + return f01.payloads + + +def by_name(exporter, fragment): + return [span for span in exporter.get_finished_spans() if fragment in span.name] + + +def all_refs(exporter): + found = [] + for span in exporter.get_finished_spans(): + found.extend((span.name, ref) for ref in refs(span)) + return found diff --git a/python/tests/attachments_frameworks/test_agno_attachments.py b/python/tests/attachments_frameworks/test_agno_attachments.py new file mode 100644 index 000000000..8de479a14 --- /dev/null +++ b/python/tests/attachments_frameworks/test_agno_attachments.py @@ -0,0 +1,136 @@ +"""Real Agno agent + real OpenAI SDK (loopback) -> traceai_agno -> loopback F01. + +Privacy is the intersection of the FI policy and the OpenInference TraceConfig. +""" + +import pytest + +pytest.importorskip("agno") +pytest.importorskip("openinference.instrumentation.agno") +pytest.importorskip("traceai_agno") + +from conftest import PDF, b64, flush, png, refs, wav # noqa: E402 +from agno.agent import Agent # noqa: E402 +from agno.media import Audio, File, Image # noqa: E402 +from agno.models.openai import OpenAIChat # noqa: E402 +from fi_instrumentation.attachments import AttachmentConfig, create_attachment_manager # noqa: E402 +from openinference.instrumentation import TraceConfig as OITraceConfig # noqa: E402 +from traceai_agno import AgnoInstrumentorWrapper, configure_agno_tracing # noqa: E402 + +IMAGE = png(seed=51) + + +@pytest.fixture(autouse=True) +def no_telemetry(monkeypatch): + monkeypatch.setenv("AGNO_TELEMETRY", "false") + for key in ("OPENINFERENCE_HIDE_INPUTS", "OPENINFERENCE_HIDE_INPUT_IMAGES", "OPENINFERENCE_HIDE_OUTPUTS"): + monkeypatch.delenv(key, raising=False) + + +@pytest.fixture +def instrument(spans): + tracer_provider, exporter = spans + + def install(manager, config=None): + configure_agno_tracing(tracer_provider=tracer_provider, attachments=manager, config=config) + return exporter + + yield install + AgnoInstrumentorWrapper().uninstrument() + + +def agent(provider): + return Agent(model=OpenAIChat(id="gpt-4o-mini", api_key="sk-synthetic", base_url=provider.url + "/v1", + max_retries=0), telemetry=False) + + +def model_spans(exporter): + return [s for s in exporter.get_finished_spans() if s.name.endswith((".invoke", ".ainvoke"))] + + +def test_image_bytes_reach_f01_on_the_model_span(f01, provider, instrument): + manager = f01.manager() + exporter = instrument(manager) + agent(provider).run("what is this", images=[Image(content=IMAGE, format="png")]) + flush(manager) + assert f01.payloads == [IMAGE] + (body,) = f01.uploads + assert body["mime_type"] == "image/png" and body["direction"] == "input" and body["purpose"] == "model_input" + (span,) = model_spans(exporter) + assert [r["occurrence_id"] for r in refs(span)] == [body["occurrence_id"]] + + +def test_async_run_and_audio(f01, provider, instrument): + import asyncio + + manager = f01.manager() + instrument(manager) + asyncio.run(agent(provider).arun("listen", audio=[Audio(content=wav(), format="wav")])) + flush(manager) + assert f01.payloads == [wav()] + + +def test_file_bytes_pdf(f01, provider, instrument): + manager = f01.manager() + instrument(manager) + try: + agent(provider).run("read", files=[File(content=PDF, mime_type="application/pdf", filename="a.pdf")]) + except Exception: + pass # the OpenAI chat formatter may reject files on some versions; capture happens before the call + flush(manager) + assert f01.payloads == [PDF] + + +@pytest.mark.parametrize("oi", [OITraceConfig(hide_input_images=True), OITraceConfig(hide_inputs=True), + OITraceConfig(hide_input_messages=True)]) +def test_openinference_restriction_alone_denies(f01, provider, instrument, oi): + manager = f01.manager() + instrument(manager, config=oi) + agent(provider).run("what is this", images=[Image(content=IMAGE, format="png")]) + flush(manager) + assert f01.after_bootstrap() == [] + + +@pytest.mark.parametrize("env", ["FI_HIDE_INPUT_IMAGES", "FI_HIDE_INPUTS", "FI_PII_REDACTION"]) +def test_fi_restriction_alone_denies_even_with_permissive_oi_config(f01, provider, instrument, monkeypatch, env): + monkeypatch.setenv(env, "true") + manager = f01.manager() + instrument(manager, config=OITraceConfig(hide_input_images=False, hide_inputs=False)) + agent(provider).run("what is this", images=[Image(content=IMAGE, format="png")]) + flush(manager) + assert f01.after_bootstrap() == [] + + +def test_openinference_env_restriction_denies(f01, provider, instrument, monkeypatch): + monkeypatch.setenv("OPENINFERENCE_HIDE_INPUT_IMAGES", "true") + manager = f01.manager() + instrument(manager) + agent(provider).run("what is this", images=[Image(content=IMAGE, format="png")]) + flush(manager) + assert f01.after_bootstrap() == [] + + +def test_url_and_filepath_are_never_read(f01, provider, instrument, tmp_path): + manager = f01.manager() + instrument(manager) + path = tmp_path / "secret.png" + path.write_bytes(IMAGE) + run = agent(provider) + for image in (Image(url="http://127.0.0.1:9/never.png"), Image(filepath=str(path))): + try: + run.run("what is this", images=[image]) + except Exception: + pass # Agno itself may try to load the source for the provider; the instrumentation never does + flush(manager) + assert f01.payloads == [] + + +def test_mismatched_bytes_and_disabled_manager(f01, provider, instrument): + instrument(create_attachment_manager(config=AttachmentConfig(enabled=False))) + agent(provider).run("x", images=[Image(content=IMAGE, format="png")]) + AgnoInstrumentorWrapper().uninstrument() + manager = f01.manager() + instrument(manager) + agent(provider).run("x", images=[Image(content=b"GIF89a not a png at all", format="png")]) + flush(manager) + assert f01.payloads == [] diff --git a/python/tests/attachments_frameworks/test_claims.py b/python/tests/attachments_frameworks/test_claims.py new file mode 100644 index 000000000..12fa347c9 --- /dev/null +++ b/python/tests/attachments_frameworks/test_claims.py @@ -0,0 +1,236 @@ +"""Shared helper: cross-span claims, raw bytes, URL policy and part limits (no framework installed).""" + +import hashlib +import http.server +import threading + +import pytest +from conftest import b64, flush, jpeg, png, refs, wav +from fi_instrumentation import TraceConfig +from fi_instrumentation.attachments import CapturePolicy +from fi_instrumentation.attachments import media_capture as mc +from opentelemetry.sdk.trace.sampling import ALWAYS_OFF +from opentelemetry.sdk.trace import TracerProvider + +IMAGE = png(seed=11) +URI = "data:image/png;base64," + b64(IMAGE) + + +def _inline(manager, span, value=URI, policy=None, raw=False): + return mc.attach_inline(manager, span, lambda: value, mc.IMAGE_MIMES, raw=raw, direction="input", + purpose="model_input", capture=policy or mc.local_capture_policy(None, direction="input"), + message_index=0, part_index=0, role="user") + + +def test_framework_then_provider_span_uploads_identical_bytes_once(f01, spans): + tracer_provider, exporter = spans + manager = f01.manager() + tracer = tracer_provider.get_tracer("t") + with tracer.start_as_current_span("framework") as parent: + first = _inline(manager, parent) + with tracer.start_as_current_span("provider") as child: + second = _inline(manager, child) + flush(manager) + assert first is not None and second == first + assert f01.payloads == [IMAGE] + names = {span.name: refs(span) for span in exporter.get_finished_spans()} + assert [r["occurrence_id"] for r in names["framework"]] == [first.occurrence_id] + assert names["provider"] == [] # the occurrence stays bound to the first span + + +def test_raw_bytes_and_inline_base64_of_the_same_image_share_one_claim(f01, spans): + tracer_provider, _ = spans + manager = f01.manager() + with tracer_provider.get_tracer("t").start_as_current_span("s") as span: + a = mc.attach_bytes(manager, span, IMAGE, mc.IMAGE_MIMES, direction="input", purpose="model_input", + capture=CapturePolicy(model_input=True), message_index=0, part_index=0) + b = _inline(manager, span) + flush(manager) + assert a == b and f01.payloads == [IMAGE] + + +def test_another_trace_uploads_again(f01, spans): + tracer_provider, _ = spans + manager = f01.manager() + tracer = tracer_provider.get_tracer("t") + with tracer.start_as_current_span("one") as one: + a = _inline(manager, one) + with tracer.start_as_current_span("two") as two: + b = _inline(manager, two) + flush(manager) + assert a != b and f01.payloads == [IMAGE, IMAGE] + + +def test_a_later_observer_denied_by_its_own_policy_gets_nothing(f01, spans): + tracer_provider, _ = spans + manager = f01.manager() + tracer = tracer_provider.get_tracer("t") + hidden = mc.local_capture_policy(TraceConfig(hide_input_images=True), direction="input", image=True) + with tracer.start_as_current_span("framework") as parent: + assert _inline(manager, parent) is not None + with tracer.start_as_current_span("provider") as child: + assert _inline(manager, child, policy=hidden) is None # reuse never widens permission + flush(manager) + assert f01.payloads == [IMAGE] + + +def test_denied_bytes_are_never_hashed(f01, spans, monkeypatch): + tracer_provider, _ = spans + manager = f01.manager() + calls = [] + real = hashlib.sha256 + monkeypatch.setattr(mc.hashlib, "sha256", lambda data=b"": calls.append(1) or real(data)) + hidden = mc.local_capture_policy(TraceConfig(hide_inputs=True), direction="input") + with tracer_provider.get_tracer("t").start_as_current_span("s") as span: + assert _inline(manager, span, policy=hidden) is None + assert mc.attach_bytes(manager, span, IMAGE, mc.IMAGE_MIMES, direction="input", purpose="model_input", + capture=hidden) is None + assert calls == [] and f01.after_bootstrap() == [] + + +def test_concurrent_observers_of_identical_bytes_upload_once(f01, spans): + tracer_provider, _ = spans + manager = f01.manager() + tracer = tracer_provider.get_tracer("t") + results = [] + with tracer.start_as_current_span("root") as root: + def observe(): + with tracer.start_as_current_span("child", context=None) as child: + results.append(_inline(manager, child)) + from opentelemetry import context as otel_context + ctx = otel_context.get_current() + threads = [threading.Thread(target=lambda: (otel_context.attach(ctx), observe())) for _ in range(8)] + [t.start() for t in threads] + [t.join() for t in threads] + flush(manager) + assert len(set(results)) == 1 and None not in results + assert f01.payloads == [IMAGE] + + +def test_magic_bytes_must_match_and_select_the_exact_type(f01, spans): + tracer_provider, _ = spans + manager = f01.manager() + policy = CapturePolicy(model_input=True) + with tracer_provider.get_tracer("t").start_as_current_span("s") as span: + assert mc.attach_bytes(manager, span, b"not an image at all", mc.IMAGE_MIMES, direction="input", + purpose="model_input", capture=policy) is None + assert mc.attach_bytes(manager, span, wav(), mc.IMAGE_MIMES, direction="input", + purpose="model_input", capture=policy) is None # audio is not an image + found = mc.attach_bytes(manager, span, jpeg(), mc.IMAGE_MIMES, direction="input", purpose="model_input", + capture=policy, part_index=1) + assert found is not None and found.mime_type == "image/jpeg" + # verify=True on inline data: a PNG data URI holding JPEG bytes is rejected. + lying = "data:image/png;base64," + b64(jpeg(seed=3)) + assert mc.attach_inline(manager, span, lambda: lying, mc.IMAGE_MIMES, raw=False, direction="input", + purpose="model_input", capture=policy, verify=True) is None + flush(manager) + assert f01.payloads == [jpeg()] + + +def test_over_cap_part_is_denied_before_any_read(f01, spans): + tracer_provider, _ = spans + manager = f01.manager() + reads = [] + big = b"\x89PNG\r\n\x1a\n" + b"\x00" * (mc.MAX_FRAMEWORK_BYTES + 1) + part = mc.MediaPart("bytes", lambda: reads.append(1) or big, mc.IMAGE_MIMES, "model_input", 0, 0) + with tracer_provider.get_tracer("t").start_as_current_span("s") as span: + assert mc.capture_parts(manager, span, "test", [part], direction="input") == {} + flush(manager) + assert f01.payloads == [] + + +def test_url_parts_go_to_core_policy_and_are_never_fetched(f01, spans): + hits = [] + + class Handler(http.server.BaseHTTPRequestHandler): + def do_GET(self): # noqa: N802 + hits.append(self.path) + self.send_response(200) + self.end_headers() + + def log_message(self, *args): + pass + + server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), Handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + tracer_provider, _ = spans + manager = f01.manager() + urls = ["http://127.0.0.1:%d/private.png" % server.server_port, "http://169.254.169.254/latest", + "file:///etc/passwd"] + parts = [mc.MediaPart("url", lambda u=u: u, mc.IMAGE_MIMES, "model_input", 0, i) for i, u in enumerate(urls)] + with tracer_provider.get_tracer("t").start_as_current_span("s") as span: + assert mc.capture_parts(manager, span, "test", parts, direction="input") == {} + flush(manager) + assert hits == [] and f01.after_bootstrap() == [] + finally: + server.shutdown() + server.server_close() + + +def test_part_limit_per_span(f01, spans): + tracer_provider, _ = spans + manager = f01.manager() + parts = [mc.MediaPart("inline", lambda i=i: b64(png(seed=i)), mc.IMAGE_MIMES, "model_input", 0, i) + for i in range(mc.MAX_FRAMEWORK_PARTS + 6)] + with tracer_provider.get_tracer("t").start_as_current_span("s") as span: + captured = mc.capture_parts(manager, span, "test", parts, direction="input") + flush(manager) + assert len(captured) == mc.MAX_FRAMEWORK_PARTS + assert len(f01.payloads) == mc.MAX_FRAMEWORK_PARTS + + +def test_unsampled_span_captures_nothing(f01): + manager = f01.manager() + tracer = TracerProvider(sampler=ALWAYS_OFF).get_tracer("t") + with tracer.start_as_current_span("s") as span: + assert _inline(manager, span) is None + assert f01.after_bootstrap() == [] + + +def test_intersection_is_deny_wins(): + a = CapturePolicy() + b = CapturePolicy(capture_content=False, muted=True) + both = mc.intersect(a, b) + assert both.capture_content is False and both.muted is True and both.record_inputs is True + assert mc.intersect().capture_content is True + # Any configuration in the list that hides wins (FI and OpenInference configs, for example). + class OIConfig: + hide_inputs = False + hide_input_images = True + folded = mc.intersect(*(mc.local_capture_policy(c, direction="input", image=True) + for c in (TraceConfig(), OIConfig()))) + assert folded.capture_content is False and folded.model_input is True + + +def test_capture_parts_folds_every_config(f01, spans): + class OIConfig: + hide_input_images = True + tracer_provider, _ = spans + manager = f01.manager() + parts = [mc.MediaPart("inline", lambda: URI, mc.IMAGE_MIMES, "model_input", 0, 0, raw=False)] + with tracer_provider.get_tracer("t").start_as_current_span("s") as span: + assert mc.capture_parts(manager, span, "test", parts, direction="input", + trace_configs=(TraceConfig(), OIConfig())) == {} + assert mc.capture_parts(manager, span, "test", parts, direction="input", + trace_configs=(TraceConfig(),)) + flush(manager) + assert f01.payloads == [IMAGE] + + +def test_looks_base64(): + assert mc.looks_base64(b64(IMAGE).encode()) + assert not mc.looks_base64(IMAGE) + assert not mc.looks_base64(b"%PDF-1.4 abc") + assert not mc.looks_base64("not bytes") + + +@pytest.mark.parametrize("value", [None, 7, b"", "data:image/png;base64,@@@@"]) +def test_malformed_sources_capture_nothing(f01, spans, value): + tracer_provider, _ = spans + manager = f01.manager() + with tracer_provider.get_tracer("t").start_as_current_span("s") as span: + assert _inline(manager, span, value=value) is None + flush(manager) + assert f01.payloads == [] diff --git a/python/tests/attachments_frameworks/test_exit_probe.py b/python/tests/attachments_frameworks/test_exit_probe.py new file mode 100644 index 000000000..a16407fe4 --- /dev/null +++ b/python/tests/attachments_frameworks/test_exit_probe.py @@ -0,0 +1,93 @@ +"""Lifecycle: a framework-instrumented process exits promptly with attachments enabled, disabled, after fork.""" + +import os +import subprocess +import sys +import textwrap +import time + +import pytest + +PROBE = textwrap.dedent(""" + import base64, os, ssl, sys, threading + sys.path.insert(0, os.environ["PROBE_DIR"]) + from conftest import png + from fi_instrumentation.attachments import AttachmentConfig, AttachmentManager, create_attachment_manager + from fi_instrumentation.attachments.protocol import HTTPTransport + from opentelemetry.sdk.trace import TracerProvider + + enabled = os.environ["PROBE_ENABLED"] == "1" + framework = os.environ["PROBE_FRAMEWORK"] + before = threading.active_count() + if enabled: + transport = HTTPTransport(os.environ["PROBE_F01"], {"X-API-Key": "SYNTHETIC_KEY"}, + ssl_context=ssl.create_default_context(cafile=os.environ["PROBE_CERT"])) + manager = AttachmentManager(config=AttachmentConfig(enabled=True), project_name="synthetic", + transport=transport) + assert manager.ready(5) + else: + manager = create_attachment_manager(config=AttachmentConfig(enabled=False)) + image = png(seed=71) + + if framework == "langchain": + from langchain_core.language_models.fake_chat_models import FakeListChatModel + from langchain_core.messages import HumanMessage + from traceai_langchain import LangChainInstrumentor + LangChainInstrumentor().instrument(tracer_provider=TracerProvider(), attachments=manager) + model = FakeListChatModel(responses=["ok", "ok"]) + def call(): + model.invoke([HumanMessage(content=[{"type": "image", "source_type": "base64", + "data": base64.b64encode(image).decode(), + "mime_type": "image/png"}])]) + else: + from pydantic_ai import Agent, BinaryContent + from pydantic_ai.messages import ModelResponse, TextPart + from pydantic_ai.models.function import FunctionModel + from traceai_pydantic_ai import PydanticAIInstrumentor + PydanticAIInstrumentor().instrument(tracer_provider=TracerProvider(), attachments=manager) + agent = Agent(FunctionModel(lambda m, i: ModelResponse(parts=[TextPart("ok")]))) + def call(): + agent.run_sync(["x", BinaryContent(data=image, media_type="image/png")]) + + call() + if not enabled: + assert threading.active_count() == before, "disabled attachments started a thread" + if os.environ.get("PROBE_FORK") == "1" and hasattr(os, "fork"): + pid = os.fork() + if pid == 0: + call() + os._exit(0) + os.waitpid(pid, 0) + print("PROBE_DONE", flush=True) + # No explicit flush or shutdown: interpreter exit must not hang. +""") + + +def _available(framework): + module = {"langchain": "traceai_langchain", "pydantic_ai": "traceai_pydantic_ai"}[framework] + try: + __import__(module) + __import__("langchain_core" if framework == "langchain" else "pydantic_ai") + return True + except ImportError: + return False + + +@pytest.mark.parametrize("framework", ["langchain", "pydantic_ai"]) +@pytest.mark.parametrize("enabled,fork", [(True, False), (False, False), (True, True)]) +def test_process_exits_promptly(f01, tmp_path, framework, enabled, fork): + if not _available(framework): + pytest.skip(framework + " not installed") + script = tmp_path / "probe.py" + script.write_text(PROBE) + env = dict(os.environ, PROBE_DIR=os.path.dirname(__file__), PROBE_ENABLED="1" if enabled else "0", + PROBE_F01=f01.endpoint, PROBE_CERT=str(f01.cert), PROBE_FORK="1" if fork else "0", + PROBE_FRAMEWORK=framework) + started = time.monotonic() + result = subprocess.run([sys.executable, str(script)], env=env, capture_output=True, text=True, timeout=60) + elapsed = time.monotonic() - started + assert result.returncode == 0, result.stderr[-2000:] + assert "PROBE_DONE" in result.stdout + assert elapsed < 30, elapsed + if not enabled: + assert f01.state["requests"] == [] diff --git a/python/tests/attachments_frameworks/test_langchain_attachments.py b/python/tests/attachments_frameworks/test_langchain_attachments.py new file mode 100644 index 000000000..32efdeae3 --- /dev/null +++ b/python/tests/attachments_frameworks/test_langchain_attachments.py @@ -0,0 +1,224 @@ +"""Real langchain-core callbacks -> traceai_langchain -> loopback F01 (typed message media).""" + +from typing import Any, List, Optional + +import pytest + +pytest.importorskip("langchain_core") +pytest.importorskip("traceai_langchain") + +from conftest import PDF, b64, flush, png, refs, wav # noqa: E402 +from fi_instrumentation import TraceConfig # noqa: E402 +from fi_instrumentation.attachments import AttachmentConfig, create_attachment_manager # noqa: E402 +from langchain_core.language_models.chat_models import BaseChatModel # noqa: E402 +from langchain_core.messages import AIMessage, BaseMessage, HumanMessage, ToolMessage # noqa: E402 +from langchain_core.outputs import ChatGeneration, ChatResult # noqa: E402 +from opentelemetry.sdk.trace import TracerProvider # noqa: E402 +from opentelemetry.sdk.trace.export import SimpleSpanProcessor # noqa: E402 +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter # noqa: E402 +from opentelemetry.sdk.trace.sampling import ALWAYS_OFF # noqa: E402 +from traceai_langchain import LangChainInstrumentor # noqa: E402 + +IMAGE = png(seed=21) +OUTPUT = png(seed=22) +TOOL = png(seed=23) + + +class FakeChat(BaseChatModel): + """A real BaseChatModel: LangChain's own callback manager drives the tracer.""" + + reply: Any = "ok" + fail: bool = False + + @property + def _llm_type(self) -> str: + return "fake-media" + + def _generate(self, messages: List[BaseMessage], stop: Optional[List[str]] = None, run_manager=None, + **kwargs: Any) -> ChatResult: + if self.fail: + raise RuntimeError("provider said: secret error body") + return ChatResult(generations=[ChatGeneration(message=AIMessage(content=self.reply))]) + + +@pytest.fixture +def instrument(spans): + tracer_provider, exporter = spans + state = {} + + def install(manager, config=None, provider=None): + LangChainInstrumentor().instrument(tracer_provider=provider or tracer_provider, attachments=manager, + config=config or TraceConfig()) + state["on"] = True + return exporter + + yield install + if state.get("on"): + LangChainInstrumentor().uninstrument() + + +def llm_span(exporter): + found = [s for s in exporter.get_finished_spans() if s.attributes.get("gen_ai.span.kind") == "LLM" + or s.name == "FakeChat"] + assert found, [s.name for s in exporter.get_finished_spans()] + return found[-1] + + +def human(*blocks): + return HumanMessage(content=[{"type": "text", "text": "describe"}, *blocks]) + + +BLOCK_STYLES = { + "openai_image_url": ({"type": "image_url", "image_url": {"url": "data:image/png;base64," + b64(IMAGE)}}, + IMAGE, "image/png", "model_input"), + "lc03_standard": ({"type": "image", "source_type": "base64", "data": b64(IMAGE), "mime_type": "image/png"}, + IMAGE, "image/png", "model_input"), + "lc1_content_block": ({"type": "image", "base64": b64(IMAGE), "mime_type": "image/png"}, + IMAGE, "image/png", "model_input"), + "anthropic_source": ({"type": "image", "source": {"type": "base64", "media_type": "image/png", + "data": b64(IMAGE)}}, IMAGE, "image/png", "model_input"), + "openai_file_pdf": ({"type": "file", "file": {"filename": "a.pdf", + "file_data": "data:application/pdf;base64," + b64(PDF)}}, + PDF, "application/pdf", "document"), + "openai_input_audio": ({"type": "input_audio", "input_audio": {"data": b64(wav()), "format": "wav"}}, + wav(), "audio/wav", "model_input"), +} + + +@pytest.mark.parametrize("style", sorted(BLOCK_STYLES)) +def test_input_block_styles_upload_original_bytes(f01, instrument, style): + block, data, mime, purpose = BLOCK_STYLES[style] + manager = f01.manager() + exporter = instrument(manager) + FakeChat().invoke([human(block)]) + flush(manager) + assert f01.payloads == [data] + (body,) = f01.uploads + assert body["mime_type"] == mime and body["direction"] == "input" and body["purpose"] == purpose + span_refs = refs(llm_span(exporter)) + assert [(r["occurrence_id"], r["mime_type"]) for r in span_refs] == [(body["occurrence_id"], mime)] + + +def test_output_media_and_tool_results(f01, instrument): + manager = f01.manager() + exporter = instrument(manager) + reply = [{"type": "text", "text": "here"}, {"type": "image", "base64": b64(OUTPUT), "mime_type": "image/png"}] + tool = ToolMessage(content=[{"type": "image_url", "image_url": {"url": "data:image/png;base64," + b64(TOOL)}}], + tool_call_id="call_screen_1") + ask = AIMessage(content="", tool_calls=[{"id": "call_screen_1", "name": "screenshot", "args": {}}]) + FakeChat(reply=reply).invoke([human(), ask, tool]) + flush(manager) + assert sorted(f01.payloads) == sorted([TOOL, OUTPUT]) + bodies = {body["direction"]: body for body in f01.uploads} + assert bodies["input"]["purpose"] == "tool_result" and bodies["input"].get("tool_call_id") == "call_screen_1" + assert bodies["output"]["purpose"] == "model_output" + positions = {(r["direction"], r.get("message_index"), r.get("part_index"), r.get("role")) + for r in refs(llm_span(exporter))} + assert ("input", 2, 0, "tool") in positions and ("output", 0, 1, "assistant") in positions + + +def test_failed_run_never_captures_output_and_input_still_follows_policy(f01, instrument): + manager = f01.manager() + instrument(manager) + with pytest.raises(RuntimeError): + FakeChat(fail=True, reply=[{"type": "image", "base64": b64(OUTPUT), "mime_type": "image/png"}]).invoke( + [human({"type": "image_url", "image_url": {"url": "data:image/png;base64," + b64(IMAGE)}})]) + flush(manager) + assert f01.payloads == [IMAGE] + + +@pytest.mark.parametrize("config", [ + TraceConfig(hide_inputs=True), TraceConfig(hide_input_messages=True), TraceConfig(hide_input_images=True), + TraceConfig(hide_input_text=True), TraceConfig(pii_redaction=True), +]) +def test_local_input_restrictions_deny(f01, instrument, config): + manager = f01.manager() + instrument(manager, config=config) + FakeChat().invoke([human({"type": "image", "base64": b64(IMAGE), "mime_type": "image/png"})]) + flush(manager) + assert f01.after_bootstrap() == [] + + +def test_pii_redaction_beats_server_allowance(f01, instrument): + f01.state["policy"]["allow_unredacted_media_with_text_pii"] = True + manager = f01.manager() + instrument(manager, config=TraceConfig(pii_redaction=True)) + FakeChat().invoke([human({"type": "image", "base64": b64(IMAGE), "mime_type": "image/png"})]) + flush(manager) + assert f01.after_bootstrap() == [] + + +def test_hide_outputs_denies_output_only(f01, instrument): + manager = f01.manager() + instrument(manager, config=TraceConfig(hide_outputs=True)) + FakeChat(reply=[{"type": "image", "base64": b64(OUTPUT), "mime_type": "image/png"}]).invoke( + [human({"type": "image", "base64": b64(IMAGE), "mime_type": "image/png"})]) + flush(manager) + assert f01.payloads == [IMAGE] + + +def test_server_policy_audio_only_denies_images(f01, instrument): + f01.state["policy"]["allowed_modalities"] = ["audio"] + manager = f01.manager() + instrument(manager) + FakeChat().invoke([human({"type": "image", "base64": b64(IMAGE), "mime_type": "image/png"}, + {"type": "input_audio", "input_audio": {"data": b64(wav()), "format": "wav"}})]) + flush(manager) + assert f01.payloads == [wav()] + + +def test_disabled_missing_and_unsampled_capture_nothing(f01, instrument, monkeypatch): + block = {"type": "image", "base64": b64(IMAGE), "mime_type": "image/png"} + instrument(create_attachment_manager(config=AttachmentConfig(enabled=False))) + FakeChat().invoke([human(block)]) + LangChainInstrumentor().uninstrument() + instrument(None) + FakeChat().invoke([human(block)]) + LangChainInstrumentor().uninstrument() + manager = f01.manager() + exporter = InMemorySpanExporter() + unsampled = TracerProvider(sampler=ALWAYS_OFF) + unsampled.add_span_processor(SimpleSpanProcessor(exporter)) + instrument(manager, provider=unsampled) + FakeChat().invoke([human(block)]) + flush(manager) + assert f01.after_bootstrap() == [] + + +def test_env_disable_wins(f01, instrument, monkeypatch): + manager = f01.manager() + instrument(manager) + monkeypatch.setenv("FI_ATTACHMENTS_ENABLED", "false") + FakeChat().invoke([human({"type": "image", "base64": b64(IMAGE), "mime_type": "image/png"})]) + flush(manager) + assert f01.after_bootstrap() == [] + + +def test_unsupported_sources_are_ignored(f01, instrument): + manager = f01.manager() + instrument(manager) + FakeChat().invoke([human( + {"type": "image_url", "image_url": {"url": "https://example.invalid/remote.png"}}, + {"type": "image", "source_type": "id", "id": "file-abc"}, + {"type": "file", "file": {"file_id": "file-abc"}}, + {"type": "image", "base64": b64(IMAGE), "mime_type": "image/gif"}, + {"type": "image", "base64": "@@@@", "mime_type": "image/png"}, + {"type": "image", "base64": b64(b"GIF89a-not-png"), "mime_type": "image/png"}, + )]) + flush(manager) + assert f01.payloads == [] + + +def test_same_image_twice_in_one_trace_uploads_once(f01, instrument): + from langchain_core.runnables import RunnableLambda + + manager = f01.manager() + exporter = instrument(manager) + block = {"type": "image", "base64": b64(IMAGE), "mime_type": "image/png"} + chain = RunnableLambda(lambda _: FakeChat().invoke([human(block)])) | \ + RunnableLambda(lambda _: FakeChat().invoke([human(block)])) + chain.invoke("x") + flush(manager) + assert f01.payloads == [IMAGE] + llm_refs = [refs(s) for s in exporter.get_finished_spans() if s.name == "FakeChat"] + assert sorted(len(r) for r in llm_refs) == [0, 1] diff --git a/python/tests/attachments_frameworks/test_llamaindex_attachments.py b/python/tests/attachments_frameworks/test_llamaindex_attachments.py new file mode 100644 index 000000000..a6c43076b --- /dev/null +++ b/python/tests/attachments_frameworks/test_llamaindex_attachments.py @@ -0,0 +1,146 @@ +"""Real llama-index-core dispatcher events -> traceai_llamaindex -> loopback F01 (ChatMessage blocks).""" + +import io + +import pytest + +pytest.importorskip("llama_index.core") +pytest.importorskip("traceai_llamaindex") + +from conftest import PDF, b64, flush, png, refs, wav # noqa: E402 +from fi_instrumentation import TraceConfig # noqa: E402 +from fi_instrumentation.attachments import AttachmentConfig, create_attachment_manager # noqa: E402 +from traceai_llamaindex import LlamaIndexInstrumentor # noqa: E402 + +types = pytest.importorskip("llama_index.core.base.llms.types") +if not hasattr(types, "ImageBlock"): + pytest.skip("ChatMessage blocks need llama-index-core 0.12.3+", allow_module_level=True) + +from llama_index.core.base.llms.types import ( # noqa: E402 + AudioBlock, ChatMessage, ChatResponse, CompletionResponse, ImageBlock, LLMMetadata, TextBlock, +) +from llama_index.core.llms.callbacks import llm_chat_callback, llm_completion_callback # noqa: E402 +from llama_index.core.llms.custom import CustomLLM # noqa: E402 + +DocumentBlock = getattr(types, "DocumentBlock", None) +IMAGE = png(seed=41) +OUTPUT = png(seed=42) + + +class MediaLLM(CustomLLM): + """A real CustomLLM whose chat() is wrapped by LlamaIndex's own dispatcher callbacks.""" + + reply_blocks: list = [] + + @property + def metadata(self) -> LLMMetadata: + return LLMMetadata(model_name="media-llm", is_chat_model=True) + + @llm_chat_callback() + def chat(self, messages, **kwargs): + return ChatResponse(message=ChatMessage(role="assistant", blocks=list(self.reply_blocks))) + + @llm_completion_callback() + def complete(self, prompt, formatted=False, **kwargs): + return CompletionResponse(text="ok") + + @llm_completion_callback() + def stream_complete(self, prompt, formatted=False, **kwargs): + yield CompletionResponse(text="ok", delta="ok") + + +@pytest.fixture +def instrument(spans): + tracer_provider, exporter = spans + state = {} + + def install(manager, config=None): + LlamaIndexInstrumentor().instrument(tracer_provider=tracer_provider, attachments=manager, + config=config or TraceConfig()) + state["on"] = True + return exporter + + yield install + if state.get("on"): + LlamaIndexInstrumentor().uninstrument() + + +def chat_refs(exporter): + return [ref for span in exporter.get_finished_spans() for ref in refs(span)] + + +def test_input_and_output_blocks(f01, instrument): + manager = f01.manager() + exporter = instrument(manager) + llm = MediaLLM(reply_blocks=[TextBlock(text="here"), ImageBlock(image=OUTPUT, image_mimetype="image/png")]) + blocks = [TextBlock(text="what"), ImageBlock(image=IMAGE, image_mimetype="image/png"), + AudioBlock(audio=wav(), format="wav")] + if DocumentBlock is not None: + blocks.append(DocumentBlock(data=PDF, document_mimetype="application/pdf")) + llm.chat([ChatMessage(role="user", blocks=blocks)]) + flush(manager) + expected = [IMAGE, wav(), OUTPUT] + ([PDF] if DocumentBlock is not None else []) + assert sorted(f01.payloads) == sorted(expected) + found = {(r["direction"], r.get("message_index"), r.get("part_index"), r["mime_type"]) + for r in chat_refs(exporter)} + assert ("input", 0, 1, "image/png") in found and ("input", 0, 2, "audio/wav") in found + assert ("output", 0, 1, "image/png") in found + purposes = {body["mime_type"]: body["purpose"] for body in f01.uploads if body["direction"] == "input"} + assert purposes["image/png"] == "model_input" + if DocumentBlock is not None: + assert purposes["application/pdf"] == "document" + + +def test_url_path_and_stream_sources_are_never_read(f01, instrument, tmp_path): + manager = f01.manager() + instrument(manager) + path = tmp_path / "secret.png" + path.write_bytes(IMAGE) + stream = io.BytesIO(IMAGE) + blocks = [ImageBlock(url="http://127.0.0.1:9/never.png", image_mimetype="image/png"), + ImageBlock(path=path, image_mimetype="image/png")] + try: + blocks.append(ImageBlock(image=stream, image_mimetype="image/png")) + except Exception: + pass # older cores reject IOBase sources at construction + MediaLLM().chat([ChatMessage(role="user", blocks=blocks)]) + flush(manager) + assert f01.payloads == [] + assert stream.tell() in (0, len(IMAGE)) # never read by the instrumentation beyond what LlamaIndex did + + +def test_mismatched_magic_is_rejected(f01, instrument): + manager = f01.manager() + instrument(manager) + MediaLLM().chat([ChatMessage(role="user", blocks=[ImageBlock(image=PDF, image_mimetype="image/png")])]) + flush(manager) + assert f01.payloads == [] + + +@pytest.mark.parametrize("config", [TraceConfig(hide_inputs=True), TraceConfig(hide_input_images=True), + TraceConfig(pii_redaction=True), TraceConfig(hide_input_messages=True)]) +def test_local_restrictions_deny_input(f01, instrument, config): + manager = f01.manager() + instrument(manager, config=config) + MediaLLM().chat([ChatMessage(role="user", blocks=[ImageBlock(image=IMAGE, image_mimetype="image/png")])]) + flush(manager) + assert f01.after_bootstrap() == [] + + +def test_hide_outputs_denies_output(f01, instrument): + manager = f01.manager() + instrument(manager, config=TraceConfig(hide_outputs=True)) + MediaLLM(reply_blocks=[ImageBlock(image=OUTPUT, image_mimetype="image/png")]).chat( + [ChatMessage(role="user", content="hi")]) + flush(manager) + assert f01.after_bootstrap() == [] + + +def test_disabled_and_missing_manager_capture_nothing(f01, instrument): + instrument(create_attachment_manager(config=AttachmentConfig(enabled=False))) + MediaLLM().chat([ChatMessage(role="user", blocks=[ImageBlock(image=IMAGE, image_mimetype="image/png")])]) + LlamaIndexInstrumentor().uninstrument() + instrument(None) + MediaLLM().chat([ChatMessage(role="user", blocks=[ImageBlock(image=IMAGE, image_mimetype="image/png")])]) + f01.manager() + assert f01.after_bootstrap() == [] diff --git a/python/tests/attachments_frameworks/test_provider_dedup.py b/python/tests/attachments_frameworks/test_provider_dedup.py new file mode 100644 index 000000000..73f7c97e9 --- /dev/null +++ b/python/tests/attachments_frameworks/test_provider_dedup.py @@ -0,0 +1,155 @@ +"""Framework + OpenAI instrumentor on ONE manager: the same bytes upload exactly once. + +Each test drives the real framework against the real OpenAI SDK (loopback provider), +with both the framework instrumentor and ``traceai_openai`` given the same manager. +""" + +import json + +import pytest + +pytest.importorskip("openai") +pytest.importorskip("traceai_openai") + +from conftest import b64, flush, png, refs # noqa: E402 +from fi_instrumentation import TraceConfig # noqa: E402 +from traceai_openai import OpenAIInstrumentor # noqa: E402 + +IMAGE = png(seed=31) +URI = "data:image/png;base64," + b64(IMAGE) + + +@pytest.fixture +def openai_on(spans): + tracer_provider, exporter = spans + state = {} + + def install(manager, config=None): + OpenAIInstrumentor().instrument(tracer_provider=tracer_provider, attachments=manager, + config=config or TraceConfig()) + state["on"] = True + + yield install + if state.get("on"): + OpenAIInstrumentor().uninstrument() + + +def sent_image(provider): + """The image the provider actually received, so the test proves the bytes are the same ones.""" + for path, body, raw in provider.requests: + if path.endswith("/chat/completions"): + text = json.dumps(body) + if b64(IMAGE) in text: + return True + return False + + +def assert_once(f01, manager, provider): + flush(manager) + assert sent_image(provider), "the framework did not send the image to the provider" + assert f01.payloads == [IMAGE], len(f01.payloads) + assert len(f01.uploads) == 1 + + +# ------------------------------------------------------------------ LangChain + langchain-openai +def test_langchain_openai(f01, provider, spans, openai_on): + pytest.importorskip("langchain_openai") + from langchain_core.messages import HumanMessage + from langchain_openai import ChatOpenAI + from traceai_langchain import LangChainInstrumentor + + tracer_provider, exporter = spans + manager = f01.manager() + LangChainInstrumentor().instrument(tracer_provider=tracer_provider, attachments=manager) + openai_on(manager) + try: + model = ChatOpenAI(model="gpt-4o-mini", api_key="sk-synthetic", base_url=provider.url + "/v1", + max_retries=0) + model.invoke([HumanMessage(content=[{"type": "text", "text": "what"}, + {"type": "image_url", "image_url": {"url": URI}}])]) + finally: + LangChainInstrumentor().uninstrument() + assert_once(f01, manager, provider) + owners = [s.name for s in exporter.get_finished_spans() if refs(s)] + assert owners == ["ChatOpenAI"], owners # the framework span (captured first) owns the occurrence + + +# ------------------------------------------------------------------ LlamaIndex + llama-index-llms-openai +def test_llamaindex_openai(f01, provider, spans, openai_on): + pytest.importorskip("llama_index.llms.openai") + from llama_index.core.base.llms.types import ChatMessage, ImageBlock, TextBlock + from llama_index.llms.openai import OpenAI + from traceai_llamaindex import LlamaIndexInstrumentor + + tracer_provider, exporter = spans + manager = f01.manager() + LlamaIndexInstrumentor().instrument(tracer_provider=tracer_provider, attachments=manager) + openai_on(manager) + try: + llm = OpenAI(model="gpt-4o-mini", api_key="sk-synthetic", api_base=provider.url + "/v1", max_retries=0) + llm.chat([ChatMessage(role="user", blocks=[TextBlock(text="what"), ImageBlock(image=IMAGE, + image_mimetype="image/png")])]) + finally: + LlamaIndexInstrumentor().uninstrument() + assert_once(f01, manager, provider) + + +# ------------------------------------------------------------------ Agno + OpenAIChat +def test_agno_openai(f01, provider, spans, openai_on, monkeypatch): + pytest.importorskip("agno") + pytest.importorskip("openinference.instrumentation.agno") + from agno.agent import Agent + from agno.media import Image + from agno.models.openai import OpenAIChat + from traceai_agno import AgnoInstrumentorWrapper, configure_agno_tracing + + monkeypatch.setenv("AGNO_TELEMETRY", "false") + tracer_provider, exporter = spans + manager = f01.manager() + configure_agno_tracing(tracer_provider=tracer_provider, attachments=manager) + openai_on(manager) + try: + agent = Agent(model=OpenAIChat(id="gpt-4o-mini", api_key="sk-synthetic", base_url=provider.url + "/v1", + max_retries=0), telemetry=False) + agent.run("what is this", images=[Image(content=IMAGE, format="png")]) + finally: + AgnoInstrumentorWrapper().uninstrument() + assert_once(f01, manager, provider) + + +# ------------------------------------------------------------------ Pydantic AI + OpenAI model +def test_pydantic_ai_openai(f01, provider, spans, openai_on): + pytest.importorskip("pydantic_ai") + from pydantic_ai import Agent, BinaryContent + from traceai_pydantic_ai import PydanticAIInstrumentor + + try: + from pydantic_ai.models.openai import OpenAIChatModel as Model + except ImportError: + from pydantic_ai.models.openai import OpenAIModel as Model + from pydantic_ai.providers.openai import OpenAIProvider + + tracer_provider, exporter = spans + manager = f01.manager() + PydanticAIInstrumentor().instrument(tracer_provider=tracer_provider, attachments=manager) + openai_on(manager) + try: + model = Model("gpt-4o-mini", provider=OpenAIProvider(base_url=provider.url + "/v1", api_key="sk-synthetic")) + Agent(model).run_sync(["what is this", BinaryContent(data=IMAGE, media_type="image/png")]) + finally: + PydanticAIInstrumentor().uninstrument() + assert_once(f01, manager, provider) + + +def test_without_the_claim_both_spans_would_upload(f01, provider, spans, openai_on, monkeypatch): + """Control for the dedup tests: the provider alone uploads the image it sees (claims are per trace).""" + import openai + + manager = f01.manager() + openai_on(manager) + client = openai.OpenAI(api_key="sk-synthetic", base_url=provider.url + "/v1", max_retries=0) + for _ in range(2): # two separate traces + client.chat.completions.create(model="m", messages=[{"role": "user", "content": [ + {"type": "image_url", "image_url": {"url": URI}}]}]) + flush(manager) + assert f01.payloads == [IMAGE, IMAGE] diff --git a/python/tests/attachments_frameworks/test_pydantic_ai_attachments.py b/python/tests/attachments_frameworks/test_pydantic_ai_attachments.py new file mode 100644 index 000000000..54ffaa80e --- /dev/null +++ b/python/tests/attachments_frameworks/test_pydantic_ai_attachments.py @@ -0,0 +1,160 @@ +"""Real Pydantic AI agent runs (FunctionModel) -> traceai_pydantic_ai -> loopback F01.""" + +import http.server +import threading + +import pytest + +pytest.importorskip("pydantic_ai") +pytest.importorskip("traceai_pydantic_ai") + +from conftest import PDF, b64, flush, png, refs, wav # noqa: E402 +from fi_instrumentation import TraceConfig # noqa: E402 +from fi_instrumentation.attachments import AttachmentConfig, create_attachment_manager # noqa: E402 +from pydantic_ai import Agent, BinaryContent, ImageUrl # noqa: E402 +from pydantic_ai.messages import ModelResponse, TextPart, ToolCallPart # noqa: E402 +from pydantic_ai.models.function import AgentInfo, FunctionModel # noqa: E402 +from traceai_pydantic_ai import PydanticAIInstrumentor # noqa: E402 + +try: + from pydantic_ai.messages import FilePart +except ImportError: # older SDKs have no generated-file parts + FilePart = None + +IMAGE = png(seed=61) +GENERATED = png(seed=62) +SCREEN = png(seed=63) + + +def echo(messages, info: AgentInfo): + return ModelResponse(parts=[TextPart("ok")]) + + +@pytest.fixture +def instrument(spans): + tracer_provider, exporter = spans + state = {} + + def install(manager, config=None): + PydanticAIInstrumentor().instrument(tracer_provider=tracer_provider, attachments=manager, + config=config or TraceConfig()) + state["on"] = True + return exporter + + yield install + if state.get("on"): + PydanticAIInstrumentor().uninstrument() + + +def request_spans(exporter): + return [s for s in exporter.get_finished_spans() if "model" in s.name.lower() and "request" in s.name.lower()] + + +def test_binary_content_prompt(f01, instrument): + manager = f01.manager() + exporter = instrument(manager) + Agent(FunctionModel(echo)).run_sync(["what is this", BinaryContent(data=IMAGE, media_type="image/png"), + BinaryContent(data=PDF, media_type="application/pdf"), + BinaryContent(data=wav(), media_type="audio/wav")]) + flush(manager) + assert sorted(f01.payloads) == sorted([IMAGE, PDF, wav()]) + purposes = {body["mime_type"]: body["purpose"] for body in f01.uploads} + assert purposes == {"image/png": "model_input", "application/pdf": "document", "audio/wav": "model_input"} + owners = [s for s in request_spans(exporter) if refs(s)] + assert len(owners) == 1 and len(refs(owners[0])) == 3 + + +def test_tool_result_binary_is_captured_once_across_tool_and_model_spans(f01, instrument): + manager = f01.manager() + exporter = instrument(manager) + calls = {"n": 0} + + def model(messages, info: AgentInfo): + calls["n"] += 1 + if calls["n"] == 1: + return ModelResponse(parts=[ToolCallPart(tool_name="screenshot", args={}, tool_call_id="call_1")]) + return ModelResponse(parts=[TextPart("done")]) + + agent = Agent(FunctionModel(model)) + + @agent.tool_plain + def screenshot() -> BinaryContent: + return BinaryContent(data=SCREEN, media_type="image/png") + + agent.run_sync("take a screenshot") + flush(manager) + assert f01.payloads == [SCREEN] + (body,) = f01.uploads + assert body["purpose"] == "tool_result" and body.get("tool_call_id") == "call_1" + tool_spans = [s for s in exporter.get_finished_spans() if "tool" in s.name.lower() and refs(s)] + assert len(tool_spans) == 1 + + +@pytest.mark.skipif(FilePart is None, reason="FilePart needs a newer pydantic-ai") +def test_generated_file_output_and_error_path(f01, instrument): + manager = f01.manager() + instrument(manager) + + def generate(messages, info): + return ModelResponse(parts=[TextPart("here"), FilePart(content=BinaryContent(data=GENERATED, + media_type="image/png"))]) + + Agent(FunctionModel(generate)).run_sync("draw") + + def failing(messages, info): + raise RuntimeError("provider error body: secret") + + with pytest.raises(Exception): + Agent(FunctionModel(failing)).run_sync(["fail", BinaryContent(data=IMAGE, media_type="image/png")]) + flush(manager) + outputs = [body for body in f01.uploads if body["direction"] == "output"] + assert len(outputs) == 1 and outputs[0]["purpose"] == "model_output" + assert GENERATED in f01.payloads + + +def test_urls_are_never_fetched(f01, instrument): + hits = [] + + class Handler(http.server.BaseHTTPRequestHandler): + def do_GET(self): # noqa: N802 + hits.append(self.path) + self.send_response(200) + self.end_headers() + + def log_message(self, *args): + pass + + server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), Handler) + threading.Thread(target=server.serve_forever, daemon=True).start() + try: + manager = f01.manager() + instrument(manager) + Agent(FunctionModel(echo)).run_sync(["look", ImageUrl(url="http://127.0.0.1:%d/private.png" + % server.server_port), + ImageUrl(url="http://169.254.169.254/latest/meta-data")]) + flush(manager) + assert hits == [] and f01.after_bootstrap() == [] + finally: + server.shutdown() + server.server_close() + + +@pytest.mark.parametrize("config", [TraceConfig(hide_inputs=True), TraceConfig(hide_input_images=True), + TraceConfig(pii_redaction=True), TraceConfig(hide_input_text=True)]) +def test_local_restrictions_deny(f01, instrument, config): + manager = f01.manager() + instrument(manager, config=config) + Agent(FunctionModel(echo)).run_sync(["x", BinaryContent(data=IMAGE, media_type="image/png")]) + flush(manager) + assert f01.after_bootstrap() == [] + + +def test_mismatched_declared_type_and_disabled_manager(f01, instrument): + instrument(create_attachment_manager(config=AttachmentConfig(enabled=False))) + Agent(FunctionModel(echo)).run_sync(["x", BinaryContent(data=IMAGE, media_type="image/png")]) + PydanticAIInstrumentor().uninstrument() + manager = f01.manager() + instrument(manager) + Agent(FunctionModel(echo)).run_sync(["x", BinaryContent(data=PDF, media_type="image/png")]) + flush(manager) + assert f01.payloads == [] From 528ec088494d41c96972e6e024f008c462cea56f Mon Sep 17 00:00:00 2001 From: Nikhil Pareek <8347993+nik13@users.noreply.github.com> Date: Wed, 11 Nov 2026 09:38:50 -0800 Subject: [PATCH 02/17] ci(python): framework attachments floor uses the openai floor pydantic-ai 0.1.0 needs Co-Authored-By: Claude Opus 5.5 --- .github/workflows/attachments-python-frameworks-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/attachments-python-frameworks-tests.yml b/.github/workflows/attachments-python-frameworks-tests.yml index 9225990f7..f32d50ba7 100644 --- a/.github/workflows/attachments-python-frameworks-tests.yml +++ b/.github/workflows/attachments-python-frameworks-tests.yml @@ -48,7 +48,7 @@ jobs: python: '3.10' sdks: >- langchain-core==0.2.43 llama-index-core==0.12.41 agno==2.7.2 - openinference-instrumentation-agno==1.1.0 pydantic-ai-slim[openai]==0.1.0 openai==1.69.0 + openinference-instrumentation-agno==1.1.0 pydantic-ai-slim[openai]==0.1.0 openai==1.74.0 'httpx<0.28' # Latest releases, including the framework OpenAI integrations for the dedup tests. - name: latest From cd4e60b17c112a736875a4cf685803b36dd7696d Mon Sep 17 00:00:00 2001 From: Nikhil Pareek <8347993+nik13@users.noreply.github.com> Date: Wed, 11 Nov 2026 09:44:24 -0800 Subject: [PATCH 03/17] fix(python): scope attachment claims to the manager with a short TTL LangChain and LlamaIndex do not make their spans current, so the provider span beneath them starts a separate trace. Claims are now keyed by manager, MIME and digest with a lazy 300 s expiry, so the framework and provider observations of one call still upload once. Tests cover separate traces, another manager and claim expiry; pydantic-ai tests skip the dependency check because CI installs pydantic-ai-slim. Co-Authored-By: Claude Opus 5.5 --- .../attachments/media_capture.py | 69 ++++++++++--------- .../test_agno_attachments.py | 7 +- .../attachments_frameworks/test_claims.py | 25 +++++-- .../attachments_frameworks/test_exit_probe.py | 10 ++- .../test_llamaindex_attachments.py | 15 ++-- .../test_provider_dedup.py | 34 +++++---- .../test_pydantic_ai_attachments.py | 3 +- 7 files changed, 104 insertions(+), 59 deletions(-) diff --git a/python/fi_instrumentation/attachments/media_capture.py b/python/fi_instrumentation/attachments/media_capture.py index 8f70627eb..aa8581ff1 100644 --- a/python/fi_instrumentation/attachments/media_capture.py +++ b/python/fi_instrumentation/attachments/media_capture.py @@ -13,11 +13,12 @@ - One registry per manager remembers admitted (span, position) pairs, so a second observer of the same span and position gets the FIRST occurrence back and uploads nothing. -- One claim table per manager remembers the bytes each trace already uploaded, - so a framework span and the provider span beneath it (or any later observer in - the same trace) upload identical bytes once. A later observer must be admitted +- One claim table per manager remembers the bytes it uploaded in the last + ``CLAIM_TTL`` seconds, so a framework span and the provider span beneath it + upload identical bytes once, even when the framework does not make its span + the current context (separate traces). A later observer must be admitted under its OWN policy before it may reuse a claim, and only admitted bytes are - hashed. + hashed. One manager serves one project, so a reuse never points outside it. Every public function is non-throwing and returns ``None``/empty on denial. """ @@ -28,6 +29,7 @@ import hashlib import re import threading +import time import weakref from collections import OrderedDict from types import MappingProxyType @@ -53,6 +55,10 @@ # Framework boundaries hand over complete inline parts (mirrors @traceai/framework-attachments). MAX_FRAMEWORK_BYTES = 1 * MiB MAX_FRAMEWORK_PARTS = 64 +# Claims cover "the same call observed twice" (milliseconds apart), not long-term +# content dedup: a short window keeps the referenced occurrence live. +CLAIM_TTL = 300.0 +MAX_CLAIMS = 4096 TOOL_ID = re.compile(r"[a-zA-Z0-9_-]{1,128}") _DATA_URI = re.compile(r"data:([a-z]+/[a-z0-9.+-]+);base64") _BASE64 = re.compile(r"[A-Za-z0-9+/]*={0,2}") @@ -172,53 +178,53 @@ def put(self, manager, key: str, position: str, occurrence: Occurrence) -> None: class _Claims: - """Uploaded bytes per manager and trace: (mime, sha256) -> occurrence. Digests and IDs only. + """Uploaded bytes per manager: (mime, sha256) -> (occurrence, expiry). Digests and IDs only. - The lock is held across the lookup and ``ticket.attach`` (which only queues), - so two concurrent observers of identical bytes cannot both upload. + Expiry is lazy (checked on access; no timer). The lock is held across the + lookup and ``ticket.attach`` (which only queues), so two concurrent observers + of identical bytes cannot both upload. """ def __init__(self) -> None: self.lock = threading.Lock() self._managers: "weakref.WeakKeyDictionary" = weakref.WeakKeyDictionary() - def get(self, manager, trace: str, key: Tuple[str, str]) -> Optional[Occurrence]: - traces = self._managers.get(manager) - claims = traces.get(trace) if traces is not None else None - return claims.get(key) if claims is not None else None - - def put(self, manager, trace: str, key: Tuple[str, str], occurrence: Occurrence) -> None: - traces = self._managers.get(manager) - if traces is None: - traces = OrderedDict() - self._managers[manager] = traces - claims = traces.get(trace) + def get(self, manager, key: Tuple[str, str]) -> Optional[Occurrence]: + claims = self._managers.get(manager) + entry = claims.get(key) if claims is not None else None + if entry is None: + return None + if entry[1] <= time.monotonic(): + del claims[key] + return None + return entry[0] + + def put(self, manager, key: Tuple[str, str], occurrence: Occurrence) -> None: + claims = self._managers.get(manager) if claims is None: - if len(traces) >= MAX_SPANS: - traces.popitem(last=False) - claims = traces[trace] = {} - else: - traces.move_to_end(trace) - if len(claims) < MAX_PARTS: - claims[key] = occurrence + claims = self._managers[manager] = OrderedDict() + claims.pop(key, None) + while len(claims) >= MAX_CLAIMS: + claims.popitem(last=False) + claims[key] = (occurrence, time.monotonic() + CLAIM_TTL) CLAIMS = _Claims() def attach_claimed(manager, ticket, data: bytes, mime: str, fields: dict) -> Optional[Occurrence]: - """Attach ADMITTED bytes at most once per trace and manager. Never raises. + """Attach ADMITTED bytes at most once per manager within ``CLAIM_TTL``. Never raises. The caller's ``ticket`` was already admitted under its own policy, so reusing an earlier occurrence never widens what this observer may capture. A reuse releases the ticket and uploads nothing. """ try: - correlation = ticket.correlation - trace = correlation.trace_id if correlation is not None else None - key = (mime, hashlib.sha256(data).hexdigest()) if trace is not None else None + if not ticket.admitted: + return None + key = (mime, hashlib.sha256(data).hexdigest()) with CLAIMS.lock: - previous = CLAIMS.get(manager, trace, key) if key is not None else None + previous = CLAIMS.get(manager, key) if previous is not None: ticket.release() return previous @@ -226,8 +232,7 @@ def attach_claimed(manager, ticket, data: bytes, mime: str, fields: dict) -> Opt if handle.admission != "queued": return None occurrence = Occurrence(handle.occurrence_id, mime) - if key is not None: - CLAIMS.put(manager, trace, key, occurrence) + CLAIMS.put(manager, key, occurrence) return occurrence except Exception: try: diff --git a/python/tests/attachments_frameworks/test_agno_attachments.py b/python/tests/attachments_frameworks/test_agno_attachments.py index 8de479a14..b09ac0ed7 100644 --- a/python/tests/attachments_frameworks/test_agno_attachments.py +++ b/python/tests/attachments_frameworks/test_agno_attachments.py @@ -40,8 +40,11 @@ def install(manager, config=None): def agent(provider): - return Agent(model=OpenAIChat(id="gpt-4o-mini", api_key="sk-synthetic", base_url=provider.url + "/v1", - max_retries=0), telemetry=False) + model = OpenAIChat(id="gpt-4o-mini", api_key="sk-synthetic", base_url=provider.url + "/v1", max_retries=0) + try: + return Agent(model=model, telemetry=False) + except TypeError: # older Agno has no telemetry switch; AGNO_TELEMETRY=false covers it + return Agent(model=model) def model_spans(exporter): diff --git a/python/tests/attachments_frameworks/test_claims.py b/python/tests/attachments_frameworks/test_claims.py index 12fa347c9..02ca14843 100644 --- a/python/tests/attachments_frameworks/test_claims.py +++ b/python/tests/attachments_frameworks/test_claims.py @@ -49,16 +49,33 @@ def test_raw_bytes_and_inline_base64_of_the_same_image_share_one_claim(f01, span assert a == b and f01.payloads == [IMAGE] -def test_another_trace_uploads_again(f01, spans): +def test_separate_traces_on_one_manager_share_the_claim(f01, spans): + # LangChain and LlamaIndex do not make their span current, so the provider span is a separate root. tracer_provider, _ = spans manager = f01.manager() tracer = tracer_provider.get_tracer("t") - with tracer.start_as_current_span("one") as one: + with tracer.start_as_current_span("framework") as one: a = _inline(manager, one) - with tracer.start_as_current_span("two") as two: + with tracer.start_as_current_span("provider") as two: b = _inline(manager, two) flush(manager) - assert a != b and f01.payloads == [IMAGE, IMAGE] + assert a == b and f01.payloads == [IMAGE] + + +def test_another_manager_or_an_expired_claim_uploads_again(f01, spans, monkeypatch): + tracer_provider, _ = spans + first, second = f01.manager(), f01.manager() + tracer = tracer_provider.get_tracer("t") + with tracer.start_as_current_span("s") as span: + a = _inline(first, span) + b = _inline(second, span) + now = mc.time.monotonic() + monkeypatch.setattr(mc.time, "monotonic", lambda: now + mc.CLAIM_TTL + 1) + with tracer.start_as_current_span("later") as span: + c = _inline(first, span) + flush(first) + flush(second) + assert len({a, b, c}) == 3 and f01.payloads == [IMAGE, IMAGE, IMAGE] def test_a_later_observer_denied_by_its_own_policy_gets_nothing(f01, spans): diff --git a/python/tests/attachments_frameworks/test_exit_probe.py b/python/tests/attachments_frameworks/test_exit_probe.py index a16407fe4..0bbab7cc7 100644 --- a/python/tests/attachments_frameworks/test_exit_probe.py +++ b/python/tests/attachments_frameworks/test_exit_probe.py @@ -18,7 +18,9 @@ enabled = os.environ["PROBE_ENABLED"] == "1" framework = os.environ["PROBE_FRAMEWORK"] - before = threading.active_count() + def attachment_threads(): + return {t.ident for t in threading.enumerate() if t.name.startswith("fi-attachment")} + before = attachment_threads() if enabled: transport = HTTPTransport(os.environ["PROBE_F01"], {"X-API-Key": "SYNTHETIC_KEY"}, ssl_context=ssl.create_default_context(cafile=os.environ["PROBE_CERT"])) @@ -44,14 +46,16 @@ def call(): from pydantic_ai.messages import ModelResponse, TextPart from pydantic_ai.models.function import FunctionModel from traceai_pydantic_ai import PydanticAIInstrumentor - PydanticAIInstrumentor().instrument(tracer_provider=TracerProvider(), attachments=manager) + PydanticAIInstrumentor().instrument(tracer_provider=TracerProvider(), attachments=manager, + skip_dep_check=True) agent = Agent(FunctionModel(lambda m, i: ModelResponse(parts=[TextPart("ok")]))) def call(): agent.run_sync(["x", BinaryContent(data=image, media_type="image/png")]) call() if not enabled: - assert threading.active_count() == before, "disabled attachments started a thread" + # Frameworks may start their own threads (event loops, executors); attachments must not. + assert attachment_threads() == before, "disabled attachments started a thread" if os.environ.get("PROBE_FORK") == "1" and hasattr(os, "fork"): pid = os.fork() if pid == 0: diff --git a/python/tests/attachments_frameworks/test_llamaindex_attachments.py b/python/tests/attachments_frameworks/test_llamaindex_attachments.py index a6c43076b..a90af2b0f 100644 --- a/python/tests/attachments_frameworks/test_llamaindex_attachments.py +++ b/python/tests/attachments_frameworks/test_llamaindex_attachments.py @@ -17,12 +17,13 @@ pytest.skip("ChatMessage blocks need llama-index-core 0.12.3+", allow_module_level=True) from llama_index.core.base.llms.types import ( # noqa: E402 - AudioBlock, ChatMessage, ChatResponse, CompletionResponse, ImageBlock, LLMMetadata, TextBlock, + ChatMessage, ChatResponse, CompletionResponse, ImageBlock, LLMMetadata, TextBlock, ) from llama_index.core.llms.callbacks import llm_chat_callback, llm_completion_callback # noqa: E402 from llama_index.core.llms.custom import CustomLLM # noqa: E402 DocumentBlock = getattr(types, "DocumentBlock", None) +AudioBlock = getattr(types, "AudioBlock", None) IMAGE = png(seed=41) OUTPUT = png(seed=42) @@ -73,17 +74,21 @@ def test_input_and_output_blocks(f01, instrument): manager = f01.manager() exporter = instrument(manager) llm = MediaLLM(reply_blocks=[TextBlock(text="here"), ImageBlock(image=OUTPUT, image_mimetype="image/png")]) - blocks = [TextBlock(text="what"), ImageBlock(image=IMAGE, image_mimetype="image/png"), - AudioBlock(audio=wav(), format="wav")] + blocks = [TextBlock(text="what"), ImageBlock(image=IMAGE, image_mimetype="image/png")] + if AudioBlock is not None: + blocks.append(AudioBlock(audio=wav(), format="wav")) if DocumentBlock is not None: blocks.append(DocumentBlock(data=PDF, document_mimetype="application/pdf")) llm.chat([ChatMessage(role="user", blocks=blocks)]) flush(manager) - expected = [IMAGE, wav(), OUTPUT] + ([PDF] if DocumentBlock is not None else []) + expected = [IMAGE, OUTPUT] + ([wav()] if AudioBlock is not None else []) + \ + ([PDF] if DocumentBlock is not None else []) assert sorted(f01.payloads) == sorted(expected) found = {(r["direction"], r.get("message_index"), r.get("part_index"), r["mime_type"]) for r in chat_refs(exporter)} - assert ("input", 0, 1, "image/png") in found and ("input", 0, 2, "audio/wav") in found + assert ("input", 0, 1, "image/png") in found + if AudioBlock is not None: + assert ("input", 0, 2, "audio/wav") in found assert ("output", 0, 1, "image/png") in found purposes = {body["mime_type"]: body["purpose"] for body in f01.uploads if body["direction"] == "input"} assert purposes["image/png"] == "model_input" diff --git a/python/tests/attachments_frameworks/test_provider_dedup.py b/python/tests/attachments_frameworks/test_provider_dedup.py index 73f7c97e9..15113fb74 100644 --- a/python/tests/attachments_frameworks/test_provider_dedup.py +++ b/python/tests/attachments_frameworks/test_provider_dedup.py @@ -44,8 +44,12 @@ def sent_image(provider): return False -def assert_once(f01, manager, provider): +def assert_once(f01, manager, provider, exporter=None): flush(manager) + if exporter is not None: + # Diagnostic only: LangChain/LlamaIndex spans are not made current, so the provider span may + # be a separate trace. The claim is per manager, so the bytes still upload once. + print({s.name: format(s.context.trace_id, "032x") for s in exporter.get_finished_spans()}) assert sent_image(provider), "the framework did not send the image to the provider" assert f01.payloads == [IMAGE], len(f01.payloads) assert len(f01.uploads) == 1 @@ -69,7 +73,7 @@ def test_langchain_openai(f01, provider, spans, openai_on): {"type": "image_url", "image_url": {"url": URI}}])]) finally: LangChainInstrumentor().uninstrument() - assert_once(f01, manager, provider) + assert_once(f01, manager, provider, exporter) owners = [s.name for s in exporter.get_finished_spans() if refs(s)] assert owners == ["ChatOpenAI"], owners # the framework span (captured first) owns the occurrence @@ -91,7 +95,7 @@ def test_llamaindex_openai(f01, provider, spans, openai_on): image_mimetype="image/png")])]) finally: LlamaIndexInstrumentor().uninstrument() - assert_once(f01, manager, provider) + assert_once(f01, manager, provider, exporter) # ------------------------------------------------------------------ Agno + OpenAIChat @@ -109,8 +113,12 @@ def test_agno_openai(f01, provider, spans, openai_on, monkeypatch): configure_agno_tracing(tracer_provider=tracer_provider, attachments=manager) openai_on(manager) try: - agent = Agent(model=OpenAIChat(id="gpt-4o-mini", api_key="sk-synthetic", base_url=provider.url + "/v1", - max_retries=0), telemetry=False) + model = OpenAIChat(id="gpt-4o-mini", api_key="sk-synthetic", base_url=provider.url + "/v1", + max_retries=0) + try: + agent = Agent(model=model, telemetry=False) + except TypeError: + agent = Agent(model=model) agent.run("what is this", images=[Image(content=IMAGE, format="png")]) finally: AgnoInstrumentorWrapper().uninstrument() @@ -131,7 +139,8 @@ def test_pydantic_ai_openai(f01, provider, spans, openai_on): tracer_provider, exporter = spans manager = f01.manager() - PydanticAIInstrumentor().instrument(tracer_provider=tracer_provider, attachments=manager) + # skip_dep_check: CI installs pydantic-ai-slim, which the instrumentor's "pydantic-ai" requirement does not name. + PydanticAIInstrumentor().instrument(tracer_provider=tracer_provider, attachments=manager, skip_dep_check=True) openai_on(manager) try: model = Model("gpt-4o-mini", provider=OpenAIProvider(base_url=provider.url + "/v1", api_key="sk-synthetic")) @@ -141,15 +150,16 @@ def test_pydantic_ai_openai(f01, provider, spans, openai_on): assert_once(f01, manager, provider) -def test_without_the_claim_both_spans_would_upload(f01, provider, spans, openai_on, monkeypatch): - """Control for the dedup tests: the provider alone uploads the image it sees (claims are per trace).""" +def test_provider_alone_uploads_the_image(f01, provider, spans, openai_on): + """Control: with only traceai_openai on the manager, the provider span uploads the image itself.""" import openai + tracer_provider, exporter = spans manager = f01.manager() openai_on(manager) client = openai.OpenAI(api_key="sk-synthetic", base_url=provider.url + "/v1", max_retries=0) - for _ in range(2): # two separate traces - client.chat.completions.create(model="m", messages=[{"role": "user", "content": [ - {"type": "image_url", "image_url": {"url": URI}}]}]) + client.chat.completions.create(model="m", messages=[{"role": "user", "content": [ + {"type": "image_url", "image_url": {"url": URI}}]}]) flush(manager) - assert f01.payloads == [IMAGE, IMAGE] + assert f01.payloads == [IMAGE] + assert [s.name for s in exporter.get_finished_spans() if refs(s)] diff --git a/python/tests/attachments_frameworks/test_pydantic_ai_attachments.py b/python/tests/attachments_frameworks/test_pydantic_ai_attachments.py index 54ffaa80e..fc1256665 100644 --- a/python/tests/attachments_frameworks/test_pydantic_ai_attachments.py +++ b/python/tests/attachments_frameworks/test_pydantic_ai_attachments.py @@ -36,8 +36,9 @@ def instrument(spans): state = {} def install(manager, config=None): + # skip_dep_check: pydantic-ai-slim does not satisfy the instrumentor's "pydantic-ai" requirement. PydanticAIInstrumentor().instrument(tracer_provider=tracer_provider, attachments=manager, - config=config or TraceConfig()) + config=config or TraceConfig(), skip_dep_check=True) state["on"] = True return exporter From a8e9e0799e9945c1092514d86638c04ba98a8b6e Mon Sep 17 00:00:00 2001 From: Nikhil Pareek <8347993+nik13@users.noreply.github.com> Date: Wed, 11 Nov 2026 09:53:32 -0800 Subject: [PATCH 04/17] test(python): claim expiry test uses the claim table's own clock Patching time.monotonic globally also moved the manager's own deadlines. The floor tier pins an OpenTelemetry SDK that pydantic-ai 0.1.0 can import. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/attachments-python-frameworks-tests.yml | 2 +- python/fi_instrumentation/attachments/media_capture.py | 5 +++-- python/tests/attachments_frameworks/test_claims.py | 4 ++-- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/attachments-python-frameworks-tests.yml b/.github/workflows/attachments-python-frameworks-tests.yml index f32d50ba7..215cec348 100644 --- a/.github/workflows/attachments-python-frameworks-tests.yml +++ b/.github/workflows/attachments-python-frameworks-tests.yml @@ -49,7 +49,7 @@ jobs: sdks: >- langchain-core==0.2.43 llama-index-core==0.12.41 agno==2.7.2 openinference-instrumentation-agno==1.1.0 pydantic-ai-slim[openai]==0.1.0 openai==1.74.0 - 'httpx<0.28' + 'httpx<0.28' 'opentelemetry-sdk>=1.28' # Latest releases, including the framework OpenAI integrations for the dedup tests. - name: latest python: '3.12' diff --git a/python/fi_instrumentation/attachments/media_capture.py b/python/fi_instrumentation/attachments/media_capture.py index aa8581ff1..1a4170a62 100644 --- a/python/fi_instrumentation/attachments/media_capture.py +++ b/python/fi_instrumentation/attachments/media_capture.py @@ -59,6 +59,7 @@ # content dedup: a short window keeps the referenced occurrence live. CLAIM_TTL = 300.0 MAX_CLAIMS = 4096 +_clock = time.monotonic # the claim table's own clock (tests replace it, never the global one) TOOL_ID = re.compile(r"[a-zA-Z0-9_-]{1,128}") _DATA_URI = re.compile(r"data:([a-z]+/[a-z0-9.+-]+);base64") _BASE64 = re.compile(r"[A-Za-z0-9+/]*={0,2}") @@ -194,7 +195,7 @@ def get(self, manager, key: Tuple[str, str]) -> Optional[Occurrence]: entry = claims.get(key) if claims is not None else None if entry is None: return None - if entry[1] <= time.monotonic(): + if entry[1] <= _clock(): del claims[key] return None return entry[0] @@ -206,7 +207,7 @@ def put(self, manager, key: Tuple[str, str], occurrence: Occurrence) -> None: claims.pop(key, None) while len(claims) >= MAX_CLAIMS: claims.popitem(last=False) - claims[key] = (occurrence, time.monotonic() + CLAIM_TTL) + claims[key] = (occurrence, _clock() + CLAIM_TTL) CLAIMS = _Claims() diff --git a/python/tests/attachments_frameworks/test_claims.py b/python/tests/attachments_frameworks/test_claims.py index 02ca14843..f17ac250d 100644 --- a/python/tests/attachments_frameworks/test_claims.py +++ b/python/tests/attachments_frameworks/test_claims.py @@ -69,8 +69,8 @@ def test_another_manager_or_an_expired_claim_uploads_again(f01, spans, monkeypat with tracer.start_as_current_span("s") as span: a = _inline(first, span) b = _inline(second, span) - now = mc.time.monotonic() - monkeypatch.setattr(mc.time, "monotonic", lambda: now + mc.CLAIM_TTL + 1) + now = mc._clock() + monkeypatch.setattr(mc, "_clock", lambda: now + mc.CLAIM_TTL + 1) with tracer.start_as_current_span("later") as span: c = _inline(first, span) flush(first) From 47ba30d6195766834b50e77631ff767556ec406d Mon Sep 17 00:00:00 2001 From: Nikhil Pareek <8347993+nik13@users.noreply.github.com> Date: Wed, 11 Nov 2026 10:06:39 -0800 Subject: [PATCH 05/17] ci(python): framework attachments floor pins OpenTelemetry below 1.39 and installs fastapi pydantic-ai 0.1.0 imports opentelemetry._events, which API 1.39+ no longer ships, and Agno imports fastapi from its workflow module. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/attachments-python-frameworks-tests.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/attachments-python-frameworks-tests.yml b/.github/workflows/attachments-python-frameworks-tests.yml index 215cec348..7fc3f8936 100644 --- a/.github/workflows/attachments-python-frameworks-tests.yml +++ b/.github/workflows/attachments-python-frameworks-tests.yml @@ -43,20 +43,21 @@ jobs: fail-fast: false matrix: include: - # Declared floors (ChatMessage blocks need llama-index-core 0.12.3+). + # Declared floors (ChatMessage blocks need llama-index-core 0.12.3+). pydantic-ai 0.1.0 imports + # opentelemetry._events, which OpenTelemetry API 1.39+ no longer ships; Agno imports fastapi. - name: floor python: '3.10' sdks: >- langchain-core==0.2.43 llama-index-core==0.12.41 agno==2.7.2 openinference-instrumentation-agno==1.1.0 pydantic-ai-slim[openai]==0.1.0 openai==1.74.0 - 'httpx<0.28' 'opentelemetry-sdk>=1.28' + fastapi 'httpx<0.28' 'opentelemetry-api>=1.28,<1.39' 'opentelemetry-sdk>=1.28,<1.39' # Latest releases, including the framework OpenAI integrations for the dedup tests. - name: latest python: '3.12' sdks: >- langchain-core==1.8.0 langchain-openai==1.7.4 llama-index-core==0.14.25 llama-index-llms-openai==0.8.4 agno==3.1.12 openinference-instrumentation-agno==1.1.0 - pydantic-ai-slim[openai]==2.67.0 + pydantic-ai-slim[openai]==2.67.0 fastapi steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: From 2ea677329ea37f40a0784eb666b56c00feab20e6 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek <8347993+nik13@users.noreply.github.com> Date: Wed, 11 Nov 2026 10:21:24 -0800 Subject: [PATCH 06/17] test(python): keep floor-only SDK limits out of the framework attachment results pydantic-ai 0.1.0 hangs in a forked child with no instrumentation at all, so the fork exit probe runs on LangChain only. Its FunctionModel also cannot serialize a binary tool return on the second request; the tool span has captured the image by then, which the test still asserts. Co-Authored-By: Claude Opus 5.5 --- python/tests/attachments_frameworks/test_exit_probe.py | 4 ++++ .../test_pydantic_ai_attachments.py | 8 +++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/python/tests/attachments_frameworks/test_exit_probe.py b/python/tests/attachments_frameworks/test_exit_probe.py index 0bbab7cc7..748eae8e2 100644 --- a/python/tests/attachments_frameworks/test_exit_probe.py +++ b/python/tests/attachments_frameworks/test_exit_probe.py @@ -82,6 +82,10 @@ def _available(framework): def test_process_exits_promptly(f01, tmp_path, framework, enabled, fork): if not _available(framework): pytest.skip(framework + " not installed") + if fork and framework == "pydantic_ai": + # run_sync in a forked child hangs with NO instrumentation at all (pydantic-ai 0.1.0 on a GCP VM); + # the fork row is covered by LangChain. + pytest.skip("pydantic-ai run_sync is not fork-safe on its own") script = tmp_path / "probe.py" script.write_text(PROBE) env = dict(os.environ, PROBE_DIR=os.path.dirname(__file__), PROBE_ENABLED="1" if enabled else "0", diff --git a/python/tests/attachments_frameworks/test_pydantic_ai_attachments.py b/python/tests/attachments_frameworks/test_pydantic_ai_attachments.py index fc1256665..417cb0b75 100644 --- a/python/tests/attachments_frameworks/test_pydantic_ai_attachments.py +++ b/python/tests/attachments_frameworks/test_pydantic_ai_attachments.py @@ -82,7 +82,13 @@ def model(messages, info: AgentInfo): def screenshot() -> BinaryContent: return BinaryContent(data=SCREEN, media_type="image/png") - agent.run_sync("take a screenshot") + try: + agent.run_sync("take a screenshot") + except Exception as exc: # noqa: BLE001 + # pydantic-ai 0.1.0's FunctionModel cannot JSON-serialize a binary tool return when it estimates + # usage for the second request; the tool span has already captured it by then. + if "serializ" not in str(exc).lower(): + raise flush(manager) assert f01.payloads == [SCREEN] (body,) = f01.uploads From 3ad81031dd1644a1f7d77f9561a92c5f0cb840bb Mon Sep 17 00:00:00 2001 From: Nikhil Pareek <8347993+nik13@users.noreply.github.com> Date: Wed, 11 Nov 2026 10:40:10 -0800 Subject: [PATCH 07/17] test(python): skip the pydantic-ai fork exit probe only on pydantic-ai 0.x Co-Authored-By: Claude Opus 5.5 --- .../attachments_frameworks/test_exit_probe.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/python/tests/attachments_frameworks/test_exit_probe.py b/python/tests/attachments_frameworks/test_exit_probe.py index 748eae8e2..58f004389 100644 --- a/python/tests/attachments_frameworks/test_exit_probe.py +++ b/python/tests/attachments_frameworks/test_exit_probe.py @@ -67,6 +67,17 @@ def call(): """) +def _pydantic_ai_major(): + from importlib.metadata import PackageNotFoundError, version + + for name in ("pydantic-ai-slim", "pydantic-ai"): + try: + return int(version(name).split(".")[0]) + except (PackageNotFoundError, ValueError): + continue + return 0 + + def _available(framework): module = {"langchain": "traceai_langchain", "pydantic_ai": "traceai_pydantic_ai"}[framework] try: @@ -82,10 +93,10 @@ def _available(framework): def test_process_exits_promptly(f01, tmp_path, framework, enabled, fork): if not _available(framework): pytest.skip(framework + " not installed") - if fork and framework == "pydantic_ai": - # run_sync in a forked child hangs with NO instrumentation at all (pydantic-ai 0.1.0 on a GCP VM); - # the fork row is covered by LangChain. - pytest.skip("pydantic-ai run_sync is not fork-safe on its own") + if fork and framework == "pydantic_ai" and _pydantic_ai_major() < 1: + # pydantic-ai 0.x run_sync hangs in a forked child with NO instrumentation at all (seen with 0.1.0); + # the fork row is still covered there by LangChain. + pytest.skip("pydantic-ai 0.x run_sync is not fork-safe on its own") script = tmp_path / "probe.py" script.write_text(PROBE) env = dict(os.environ, PROBE_DIR=os.path.dirname(__file__), PROBE_ENABLED="1" if enabled else "0", From 9a3c7ca1b76afda48d9dd71d3fd15f02926a61f5 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek <8347993+nik13@users.noreply.github.com> Date: Wed, 11 Nov 2026 12:29:27 -0800 Subject: [PATCH 08/17] fix(python): one occurrence per observing span; tool results fold both hide directions Remove the per-manager claim table from media_capture. It reused one occurrence across spans and traces, so a later span carried a fi-attachment URI that its own fi.attachment.refs did not list and that F01 resolve (which checks the expected parent) could never resolve. Every observing span now uploads and references its own occurrence; byte deduplication is left to the server. This also removes the dropped-upload reuse and the claim lock held across fork. capture_parts gains policy_directions. The Pydantic AI tool span folds both the input and the output hide flags (deny-wins) for the media a tool returns, because that span redacts its result under the output flags and the result is also a later model input (as in the TS MCP binding). Tests: claim tests inverted (two spans, two occurrences, each referenced on its own span); provider-only two-call regression; a fi-attachment URI resolution invariant on every real-provider test; hide_outputs / hide_output_text / hide_output_messages negative controls on the tool span. Co-Authored-By: Claude Opus 5.5 --- .../attachments/media_capture.py | 101 +++-------- .../agno/traceai_agno/_attachments.py | 5 +- .../traceai_agno/_wrappers/_model_wrapper.py | 2 +- .../traceai_langchain/_attachments.py | 6 +- .../traceai_llamaindex/_attachments.py | 5 +- .../traceai_llamaindex/_handler.py | 2 +- .../traceai_pydantic_ai/_attachments.py | 20 ++- .../_execution_wrappers.py | 2 +- .../tests/attachments_frameworks/conftest.py | 19 ++ .../test_langchain_attachments.py | 13 +- .../{test_claims.py => test_occurrences.py} | 163 ++++++++++++------ ...edup.py => test_provider_and_framework.py} | 56 +++--- .../test_pydantic_ai_attachments.py | 60 +++++-- 13 files changed, 278 insertions(+), 176 deletions(-) rename python/tests/attachments_frameworks/{test_claims.py => test_occurrences.py} (58%) rename python/tests/attachments_frameworks/{test_provider_dedup.py => test_provider_and_framework.py} (69%) diff --git a/python/fi_instrumentation/attachments/media_capture.py b/python/fi_instrumentation/attachments/media_capture.py index 1a4170a62..7853b08dc 100644 --- a/python/fi_instrumentation/attachments/media_capture.py +++ b/python/fi_instrumentation/attachments/media_capture.py @@ -13,12 +13,11 @@ - One registry per manager remembers admitted (span, position) pairs, so a second observer of the same span and position gets the FIRST occurrence back and uploads nothing. -- One claim table per manager remembers the bytes it uploaded in the last - ``CLAIM_TTL`` seconds, so a framework span and the provider span beneath it - upload identical bytes once, even when the framework does not make its span - the current context (separate traces). A later observer must be admitted - under its OWN policy before it may reuse a claim, and only admitted bytes are - hashed. One manager serves one project, so a reuse never points outside it. +- Every observing span gets its OWN occurrence. A framework span and the + provider span beneath it each upload what they observed, and identical bytes + at two positions are two occurrences; byte deduplication is server-side. A + ``fi-attachment:`` reference is only ever projected onto the span whose + ``fi.attachment.refs`` carries that occurrence. Every public function is non-throwing and returns ``None``/empty on denial. """ @@ -26,10 +25,8 @@ import base64 import binascii import dataclasses -import hashlib import re import threading -import time import weakref from collections import OrderedDict from types import MappingProxyType @@ -55,11 +52,6 @@ # Framework boundaries hand over complete inline parts (mirrors @traceai/framework-attachments). MAX_FRAMEWORK_BYTES = 1 * MiB MAX_FRAMEWORK_PARTS = 64 -# Claims cover "the same call observed twice" (milliseconds apart), not long-term -# content dedup: a short window keeps the referenced occurrence live. -CLAIM_TTL = 300.0 -MAX_CLAIMS = 4096 -_clock = time.monotonic # the claim table's own clock (tests replace it, never the global one) TOOL_ID = re.compile(r"[a-zA-Z0-9_-]{1,128}") _DATA_URI = re.compile(r"data:([a-z]+/[a-z0-9.+-]+);base64") _BASE64 = re.compile(r"[A-Za-z0-9+/]*={0,2}") @@ -178,63 +170,17 @@ def put(self, manager, key: str, position: str, occurrence: Occurrence) -> None: REGISTRY = _Registry() -class _Claims: - """Uploaded bytes per manager: (mime, sha256) -> (occurrence, expiry). Digests and IDs only. +def _attach_admitted(ticket, data: bytes, mime: str, fields: dict) -> Optional[Occurrence]: + """Attach ADMITTED bytes as a new occurrence of the ticket's span. Never raises. - Expiry is lazy (checked on access; no timer). The lock is held across the - lookup and ``ticket.attach`` (which only queues), so two concurrent observers - of identical bytes cannot both upload. - """ - - def __init__(self) -> None: - self.lock = threading.Lock() - self._managers: "weakref.WeakKeyDictionary" = weakref.WeakKeyDictionary() - - def get(self, manager, key: Tuple[str, str]) -> Optional[Occurrence]: - claims = self._managers.get(manager) - entry = claims.get(key) if claims is not None else None - if entry is None: - return None - if entry[1] <= _clock(): - del claims[key] - return None - return entry[0] - - def put(self, manager, key: Tuple[str, str], occurrence: Occurrence) -> None: - claims = self._managers.get(manager) - if claims is None: - claims = self._managers[manager] = OrderedDict() - claims.pop(key, None) - while len(claims) >= MAX_CLAIMS: - claims.popitem(last=False) - claims[key] = (occurrence, _clock() + CLAIM_TTL) - - -CLAIMS = _Claims() - - -def attach_claimed(manager, ticket, data: bytes, mime: str, fields: dict) -> Optional[Occurrence]: - """Attach ADMITTED bytes at most once per manager within ``CLAIM_TTL``. Never raises. - - The caller's ``ticket`` was already admitted under its own policy, so reusing an - earlier occurrence never widens what this observer may capture. A reuse - releases the ticket and uploads nothing. + No cross-span reuse: an occurrence belongs to the span that observed it + (F01 resolve checks the parent), so every observer uploads its own. """ try: if not ticket.admitted: return None - key = (mime, hashlib.sha256(data).hexdigest()) - with CLAIMS.lock: - previous = CLAIMS.get(manager, key) - if previous is not None: - ticket.release() - return previous - handle = ticket.attach(Bytes(data), **fields) - if handle.admission != "queued": - return None - occurrence = Occurrence(handle.occurrence_id, mime) - CLAIMS.put(manager, key, occurrence) - return occurrence + handle = ticket.attach(Bytes(data), **fields) + return Occurrence(handle.occurrence_id, mime) if handle.admission == "queued" else None except Exception: try: ticket.release() @@ -307,8 +253,7 @@ def attach_inline(manager, span, read: Callable[[], Any], mime_types: Tuple[str, """Preflight, then read once, decode strictly and attach. Never raises. ``verify`` also requires the decoded container (magic bytes) to match the - MIME it was admitted under. Identical bytes already uploaded in this trace - through the same manager are reused (``attach_claimed``). + MIME it was admitted under. """ try: state = {} @@ -347,7 +292,7 @@ def preflight(mime): if data is None or (verify and sniff_media(data[:16]) != mime): ticket.release() return None - return attach_claimed(manager, ticket, data, mime, _fields(message_index, part_index, role, tool_call_id)) + return _attach_admitted(ticket, data, mime, _fields(message_index, part_index, role, tool_call_id)) except Exception: return None @@ -397,7 +342,7 @@ def attach_bytes(manager, span, data: Any, mime_types: Tuple[str, ...], *, direc if not ticket.admitted: return None payload = data if isinstance(data, bytes) else bytes(data) - return attach_claimed(manager, ticket, payload, mime, _fields(message_index, part_index, role, tool_call_id)) + return _attach_admitted(ticket, payload, mime, _fields(message_index, part_index, role, tool_call_id)) except Exception: return None @@ -488,7 +433,7 @@ def attach_audio(manager, span, data: Any, *, direction: str, purpose: Optional[ ticket = manager.preflight(span, mime_type=mime, **options) if not ticket.admitted: return None - return attach_claimed(manager, ticket, bytes(data) if not isinstance(data, bytes) else data, mime, + return _attach_admitted(ticket, bytes(data) if not isinstance(data, bytes) else data, mime, _fields(message_index, part_index, role)) except Exception: return None @@ -635,17 +580,23 @@ def intersect(*policies: CapturePolicy) -> CapturePolicy: def capture_parts(manager, span, namespace: str, parts: Iterable[MediaPart], *, direction: str, trace_configs: Sequence[Any] = (), capture: Optional[CapturePolicy] = None, - max_bytes: int = MAX_FRAMEWORK_BYTES) -> Mapping[str, Occurrence]: + max_bytes: int = MAX_FRAMEWORK_BYTES, + policy_directions: Sequence[str] = ()) -> Mapping[str, Occurrence]: """Capture typed framework parts on ``span``; returns admitted ``"{dir}/{msg}/{part}"`` positions. Every local ``trace_configs`` entry (FI TraceConfig, OpenInference TraceConfig, ...) - folds in deny-only; the strictest wins. At most ``MAX_FRAMEWORK_PARTS`` positions per - span. Never raises, never walks arbitrary objects, never fetches or opens anything. + folds in deny-only for every direction in ``policy_directions`` (default: the upload + ``direction``); the strictest wins. A tool span's result is that span's OUTPUT and a + later model INPUT, so it passes ``("input", "output")``: any input or output hide flag + denies it. At most ``MAX_FRAMEWORK_PARTS`` positions per span. Never raises, never + walks arbitrary objects, never fetches or opens anything. """ observation = Observation(manager, span, namespace) if not observation.usable: return EMPTY try: + directions = tuple(policy_directions) or (direction,) + configs = tuple(trace_configs) or (None,) visited = 0 for part in parts: visited += 1 @@ -654,8 +605,8 @@ def capture_parts(manager, span, namespace: str, parts: Iterable[MediaPart], *, if not isinstance(part, MediaPart) or not part.mime_types: continue image = part.mime_types[0].startswith("image/") - policy = intersect(*(local_capture_policy(config, direction=direction, base=capture, image=image) - for config in (tuple(trace_configs) or (None,)))) + policy = intersect(*(local_capture_policy(config, direction=fold, base=capture, image=image) + for config in configs for fold in directions)) fields = dict(direction=direction, purpose=part.purpose, capture=policy, message_index=part.message_index, part_index=part.part_index, role=part.role, tool_call_id=part.tool_call_id) diff --git a/python/frameworks/agno/traceai_agno/_attachments.py b/python/frameworks/agno/traceai_agno/_attachments.py index c4766a3d4..c9872d312 100644 --- a/python/frameworks/agno/traceai_agno/_attachments.py +++ b/python/frameworks/agno/traceai_agno/_attachments.py @@ -14,8 +14,9 @@ Privacy is the INTERSECTION of the FI TraceConfig and the OpenInference TraceConfig the instrumentor runs with (either one hiding wins), the manager's -own policy, and the per-call capture policy. Bytes identical to ones a provider -instrumentor already uploaded in the same trace are not uploaded again. +own policy, and the per-call capture policy. The Agno span and a provider +instrumentor's span each get their own occurrence (byte deduplication is +server-side). """ from types import MappingProxyType diff --git a/python/frameworks/agno/traceai_agno/_wrappers/_model_wrapper.py b/python/frameworks/agno/traceai_agno/_wrappers/_model_wrapper.py index 422260d03..16e3ed46b 100644 --- a/python/frameworks/agno/traceai_agno/_wrappers/_model_wrapper.py +++ b/python/frameworks/agno/traceai_agno/_wrappers/_model_wrapper.py @@ -633,7 +633,7 @@ def __init__( self._trace_configs = trace_configs def _capture_input(self, span: Any, arguments: Mapping[str, Any]) -> None: - # Before the provider call, so a provider instrumentor sharing the manager reuses it. + # Input media is captured before the provider call; this span owns its own occurrences. if self._attachments is None: return try: diff --git a/python/frameworks/langchain/traceai_langchain/_attachments.py b/python/frameworks/langchain/traceai_langchain/_attachments.py index 87ac18004..58ce71201 100644 --- a/python/frameworks/langchain/traceai_langchain/_attachments.py +++ b/python/frameworks/langchain/traceai_langchain/_attachments.py @@ -10,9 +10,9 @@ - Anthropic-style ``source: {"type": "base64"}`` blocks. Remote URLs go to the core's ``RemoteUrl`` policy and are never fetched; provider -file IDs and anything else are ignored. Bytes identical to ones a provider -instrumentor (or another observer) already uploaded in the same trace are not -uploaded again (``media_capture`` claim table). +file IDs and anything else are ignored. The LangChain span and a provider +instrumentor's span each get their own occurrence of what they observed (byte +deduplication is server-side). This module imports ``fi_instrumentation.attachments``; the tracer imports it only when an enabled attachment manager is passed. diff --git a/python/frameworks/llama_index/traceai_llamaindex/_attachments.py b/python/frameworks/llama_index/traceai_llamaindex/_attachments.py index ae1a9054b..e0ddf638f 100644 --- a/python/frameworks/llama_index/traceai_llamaindex/_attachments.py +++ b/python/frameworks/llama_index/traceai_llamaindex/_attachments.py @@ -9,8 +9,9 @@ - the magic bytes must match the admitted type. An ``IOBase`` source is never read, a ``path`` is never opened, and a ``url`` goes -to the core's ``RemoteUrl`` policy without being fetched. Bytes identical to ones -a provider instrumentor already uploaded in the same trace are not uploaded again. +to the core's ``RemoteUrl`` policy without being fetched. The LlamaIndex span and a +provider instrumentor's span each get their own occurrence (byte deduplication is +server-side). This module imports ``fi_instrumentation.attachments``; the handler imports it only when an enabled attachment manager is passed. diff --git a/python/frameworks/llama_index/traceai_llamaindex/_handler.py b/python/frameworks/llama_index/traceai_llamaindex/_handler.py index b9b2a92e3..2748145f2 100644 --- a/python/frameworks/llama_index/traceai_llamaindex/_handler.py +++ b/python/frameworks/llama_index/traceai_llamaindex/_handler.py @@ -654,7 +654,7 @@ def _(self, event: LLMChatStartEvent) -> None: GEN_AI_INPUT_MESSAGES, *event.messages, ) - # Before the provider call, so a provider instrumentor sharing the manager reuses it. + # Input media is captured before the provider call; this span owns its own occurrences. self._capture_media(event.messages, "input") self[INPUT_VALUE] = safe_json_dumps(event.messages) self[INPUT_VALUE] = safe_json_dumps(_to_dict(event)) diff --git a/python/frameworks/pydantic-ai/traceai_pydantic_ai/_attachments.py b/python/frameworks/pydantic-ai/traceai_pydantic_ai/_attachments.py index a6a3813dd..6e386e766 100644 --- a/python/frameworks/pydantic-ai/traceai_pydantic_ai/_attachments.py +++ b/python/frameworks/pydantic-ai/traceai_pydantic_ai/_attachments.py @@ -8,13 +8,16 @@ core's ``RemoteUrl`` policy and are never fetched or resolved here; - model request span, output: ``FilePart`` content of a successful, non-streaming response (never an error body); -- tool call span: ``BinaryContent`` a tool returned. +- tool call span: ``BinaryContent`` a tool returned. It is the tool span's + OUTPUT and the next request's model INPUT, so the local input AND output + hide flags both apply (deny-wins, as in the TypeScript MCP binding); it is + uploaded as model input (``direction="input"``, ``purpose="tool_result"``). ``BinaryContent.data`` must be complete bytes whose magic bytes match its declared ``media_type``. Videos, uploaded provider files and unknown content -objects are not captured, and nothing is serialized to look for media. Bytes -identical to ones a provider instrumentor (or the tool span) already uploaded in -the same trace are not uploaded again. +objects are not captured, and nothing is serialized to look for media. Every +observing span (tool span, model request span, a provider instrumentor's span) +gets its own occurrence; byte deduplication is server-side. """ import dataclasses @@ -134,12 +137,12 @@ class PydanticAIMedia: def __init__(self, manager: Any, trace_config: Any) -> None: self.manager, self.trace_config = manager, trace_config - def _capture(self, span, parts, direction: str) -> Mapping[str, Occurrence]: + def _capture(self, span, parts, direction: str, policy_directions=()) -> Mapping[str, Occurrence]: if span is None: return EMPTY try: return capture_parts(self.manager, span, "pydantic_ai", parts, direction=direction, - trace_configs=(self.trace_config,)) + trace_configs=(self.trace_config,), policy_directions=policy_directions) except Exception: return EMPTY @@ -155,4 +158,7 @@ def response(self, span, response: Any) -> Mapping[str, Occurrence]: def tool_result(self, span, result: Any, tool_call_id: Any) -> Mapping[str, Occurrence]: tool_call_id = tool_call_id if isinstance(tool_call_id, str) else None - return self._capture(span, _content_parts(result, 0, 0, "tool", "tool_result", tool_call_id), "input") + # The tool span redacts this value under the OUTPUT hide flags (TOOL_RESULT -> OUTPUT_VALUE), + # and it is a later model input: fold both directions so either side's hide flag denies. + return self._capture(span, _content_parts(result, 0, 0, "tool", "tool_result", tool_call_id), "input", + policy_directions=("input", "output")) diff --git a/python/frameworks/pydantic-ai/traceai_pydantic_ai/_execution_wrappers.py b/python/frameworks/pydantic-ai/traceai_pydantic_ai/_execution_wrappers.py index bde7a0bf6..a59041b55 100644 --- a/python/frameworks/pydantic-ai/traceai_pydantic_ai/_execution_wrappers.py +++ b/python/frameworks/pydantic-ai/traceai_pydantic_ai/_execution_wrappers.py @@ -158,7 +158,7 @@ async def stream_wrapper(self, *args, **kwargs): @functools.wraps(original) async def wrapper(self, *args, **kwargs): with _model_request_span(self, tracer) as span: - # Before the provider call, so a provider instrumentor sharing the manager reuses it. + # Input media is captured before the provider call; this span owns its own occurrences. _capture(media, "request", span, _request_messages(args, kwargs)) response = await original(self, *args, **kwargs) _record_model_usage(span, response) diff --git a/python/tests/attachments_frameworks/conftest.py b/python/tests/attachments_frameworks/conftest.py index 2a340382a..f9e7d293b 100644 --- a/python/tests/attachments_frameworks/conftest.py +++ b/python/tests/attachments_frameworks/conftest.py @@ -6,6 +6,7 @@ """ import importlib.util +import re import sys from pathlib import Path @@ -54,3 +55,21 @@ def all_refs(exporter): for span in exporter.get_finished_spans(): found.extend((span.name, ref) for ref in refs(span)) return found + + +_URI = re.compile(r"fi-attachment:([A-Za-z0-9_-]+)") + + +def assert_uris_resolve(exporter): + """F01 resolve checks the expected parent: every ``fi-attachment:`` a span carries + must be an occurrence in THAT span's own ``fi.attachment.refs``. Returns the URI count.""" + seen = 0 + for span in exporter.get_finished_spans(): + own = {ref["occurrence_id"] for ref in refs(span)} + for name, value in (span.attributes or {}).items(): + values = value if isinstance(value, (list, tuple)) else (value,) + for item in values: + for occurrence in _URI.findall(item) if isinstance(item, str) else (): + seen += 1 + assert occurrence in own, (span.name, name, occurrence, sorted(own)) + return seen diff --git a/python/tests/attachments_frameworks/test_langchain_attachments.py b/python/tests/attachments_frameworks/test_langchain_attachments.py index 32efdeae3..3f7267c91 100644 --- a/python/tests/attachments_frameworks/test_langchain_attachments.py +++ b/python/tests/attachments_frameworks/test_langchain_attachments.py @@ -7,7 +7,7 @@ pytest.importorskip("langchain_core") pytest.importorskip("traceai_langchain") -from conftest import PDF, b64, flush, png, refs, wav # noqa: E402 +from conftest import PDF, assert_uris_resolve, b64, flush, png, refs, wav # noqa: E402 from fi_instrumentation import TraceConfig # noqa: E402 from fi_instrumentation.attachments import AttachmentConfig, create_attachment_manager # noqa: E402 from langchain_core.language_models.chat_models import BaseChatModel # noqa: E402 @@ -209,7 +209,7 @@ def test_unsupported_sources_are_ignored(f01, instrument): assert f01.payloads == [] -def test_same_image_twice_in_one_trace_uploads_once(f01, instrument): +def test_same_image_on_two_chat_spans_gives_each_span_its_own_occurrence(f01, instrument): from langchain_core.runnables import RunnableLambda manager = f01.manager() @@ -219,6 +219,9 @@ def test_same_image_twice_in_one_trace_uploads_once(f01, instrument): RunnableLambda(lambda _: FakeChat().invoke([human(block)])) chain.invoke("x") flush(manager) - assert f01.payloads == [IMAGE] - llm_refs = [refs(s) for s in exporter.get_finished_spans() if s.name == "FakeChat"] - assert sorted(len(r) for r in llm_refs) == [0, 1] + assert f01.payloads == [IMAGE, IMAGE] # byte deduplication is server-side + chats = [s for s in exporter.get_finished_spans() if s.name == "FakeChat"] + assert [len(refs(s)) for s in chats] == [1, 1] + owned = {r["occurrence_id"]: format(s.context.span_id, "016x") for s in chats for r in refs(s)} + assert owned == {b["occurrence_id"]: b["association"]["parent"]["span_id"] for b in f01.uploads} + assert_uris_resolve(exporter) diff --git a/python/tests/attachments_frameworks/test_claims.py b/python/tests/attachments_frameworks/test_occurrences.py similarity index 58% rename from python/tests/attachments_frameworks/test_claims.py rename to python/tests/attachments_frameworks/test_occurrences.py index f17ac250d..54f2c568e 100644 --- a/python/tests/attachments_frameworks/test_claims.py +++ b/python/tests/attachments_frameworks/test_occurrences.py @@ -1,6 +1,9 @@ -"""Shared helper: cross-span claims, raw bytes, URL policy and part limits (no framework installed).""" +"""Shared helper: per-span occurrences, raw bytes, URL policy and part limits (no framework installed). + +F01 resolves an occurrence only under its own parent span, so every observing span owns +its own occurrence: nothing here reuses an occurrence across spans or traces. +""" -import hashlib import http.server import threading @@ -9,20 +12,25 @@ from fi_instrumentation import TraceConfig from fi_instrumentation.attachments import CapturePolicy from fi_instrumentation.attachments import media_capture as mc -from opentelemetry.sdk.trace.sampling import ALWAYS_OFF +from opentelemetry import context as otel_context from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.sampling import ALWAYS_OFF IMAGE = png(seed=11) URI = "data:image/png;base64," + b64(IMAGE) -def _inline(manager, span, value=URI, policy=None, raw=False): +def _inline(manager, span, value=URI, policy=None, raw=False, part_index=0): return mc.attach_inline(manager, span, lambda: value, mc.IMAGE_MIMES, raw=raw, direction="input", purpose="model_input", capture=policy or mc.local_capture_policy(None, direction="input"), - message_index=0, part_index=0, role="user") + message_index=0, part_index=part_index, role="user") -def test_framework_then_provider_span_uploads_identical_bytes_once(f01, spans): +def _parents(f01): + return [body["association"]["parent"]["span_id"] for body in f01.uploads] + + +def test_framework_and_provider_spans_each_own_their_occurrence(f01, spans): tracer_provider, exporter = spans manager = f01.manager() tracer = tracer_provider.get_tracer("t") @@ -31,27 +39,32 @@ def test_framework_then_provider_span_uploads_identical_bytes_once(f01, spans): with tracer.start_as_current_span("provider") as child: second = _inline(manager, child) flush(manager) - assert first is not None and second == first - assert f01.payloads == [IMAGE] + assert first is not None and second is not None and first != second + assert f01.payloads == [IMAGE, IMAGE] # byte deduplication is the server's job names = {span.name: refs(span) for span in exporter.get_finished_spans()} assert [r["occurrence_id"] for r in names["framework"]] == [first.occurrence_id] - assert names["provider"] == [] # the occurrence stays bound to the first span + assert [r["occurrence_id"] for r in names["provider"]] == [second.occurrence_id] + assert _parents(f01) == ["%016x" % parent.get_span_context().span_id, + "%016x" % child.get_span_context().span_id] -def test_raw_bytes_and_inline_base64_of_the_same_image_share_one_claim(f01, spans): - tracer_provider, _ = spans +def test_raw_bytes_and_inline_base64_at_two_positions_are_two_occurrences(f01, spans): + tracer_provider, exporter = spans manager = f01.manager() with tracer_provider.get_tracer("t").start_as_current_span("s") as span: a = mc.attach_bytes(manager, span, IMAGE, mc.IMAGE_MIMES, direction="input", purpose="model_input", capture=CapturePolicy(model_input=True), message_index=0, part_index=0) - b = _inline(manager, span) + b = _inline(manager, span, part_index=1) flush(manager) - assert a == b and f01.payloads == [IMAGE] + assert a is not None and b is not None and a != b + assert f01.payloads == [IMAGE, IMAGE] + (span,) = exporter.get_finished_spans() + assert sorted(r["occurrence_id"] for r in refs(span)) == sorted([a.occurrence_id, b.occurrence_id]) -def test_separate_traces_on_one_manager_share_the_claim(f01, spans): +def test_separate_traces_on_one_manager_never_share_an_occurrence(f01, spans): # LangChain and LlamaIndex do not make their span current, so the provider span is a separate root. - tracer_provider, _ = spans + tracer_provider, exporter = spans manager = f01.manager() tracer = tracer_provider.get_tracer("t") with tracer.start_as_current_span("framework") as one: @@ -59,69 +72,123 @@ def test_separate_traces_on_one_manager_share_the_claim(f01, spans): with tracer.start_as_current_span("provider") as two: b = _inline(manager, two) flush(manager) - assert a == b and f01.payloads == [IMAGE] + assert one.get_span_context().trace_id != two.get_span_context().trace_id + assert a is not None and b is not None and a != b and f01.payloads == [IMAGE, IMAGE] + for span, occurrence in zip(exporter.get_finished_spans(), (a, b)): + assert [r["occurrence_id"] for r in refs(span)] == [occurrence.occurrence_id] -def test_another_manager_or_an_expired_claim_uploads_again(f01, spans, monkeypatch): - tracer_provider, _ = spans - first, second = f01.manager(), f01.manager() +def test_same_span_and_position_observed_twice_is_one_occurrence(f01, spans): + # The (span, position) registry still dedups ONE span observing ONE position twice. + tracer_provider, exporter = spans + manager = f01.manager() + part = mc.MediaPart("inline", lambda: URI, mc.IMAGE_MIMES, "model_input", 0, 0, raw=False) + with tracer_provider.get_tracer("t").start_as_current_span("s") as span: + first = mc.capture_parts(manager, span, "test", [part], direction="input") + again = mc.capture_parts(manager, span, "test", [part], direction="input") + flush(manager) + assert dict(first) == dict(again) and f01.payloads == [IMAGE] + (span,) = exporter.get_finished_spans() + assert len(refs(span)) == 1 + + +class _FailingStore(list): + """The loopback server fails every content PUT while this is its payload store.""" + + def append(self, item): + raise RuntimeError("synthetic content PUT failure") + + +def test_a_failed_upload_is_not_reused_by_a_later_span(f01, spans): + # The first content PUT fails; the same bytes on a later span upload again under that span. + tracer_provider, exporter = spans + manager = f01.manager() tracer = tracer_provider.get_tracer("t") - with tracer.start_as_current_span("s") as span: - a = _inline(first, span) - b = _inline(second, span) - now = mc._clock() - monkeypatch.setattr(mc, "_clock", lambda: now + mc.CLAIM_TTL + 1) - with tracer.start_as_current_span("later") as span: - c = _inline(first, span) - flush(first) - flush(second) - assert len({a, b, c}) == 3 and f01.payloads == [IMAGE, IMAGE, IMAGE] + good = f01.state["payloads"] + f01.state["payloads"] = _FailingStore() + with tracer.start_as_current_span("first") as one: + _inline(manager, one) + manager.flush(15) + f01.state["payloads"] = good + with tracer.start_as_current_span("second") as two: + b = _inline(manager, two) + flush(manager) + assert b is not None and f01.payloads == [IMAGE] + second = [s for s in exporter.get_finished_spans() if s.name == "second"][0] + assert [r["occurrence_id"] for r in refs(second)] == [b.occurrence_id] + assert _parents(f01)[-1] == "%016x" % two.get_span_context().span_id def test_a_later_observer_denied_by_its_own_policy_gets_nothing(f01, spans): - tracer_provider, _ = spans + tracer_provider, exporter = spans manager = f01.manager() tracer = tracer_provider.get_tracer("t") hidden = mc.local_capture_policy(TraceConfig(hide_input_images=True), direction="input", image=True) with tracer.start_as_current_span("framework") as parent: assert _inline(manager, parent) is not None with tracer.start_as_current_span("provider") as child: - assert _inline(manager, child, policy=hidden) is None # reuse never widens permission + assert _inline(manager, child, policy=hidden) is None flush(manager) assert f01.payloads == [IMAGE] + assert refs([s for s in exporter.get_finished_spans() if s.name == "provider"][0]) == [] -def test_denied_bytes_are_never_hashed(f01, spans, monkeypatch): +def test_denied_sources_are_never_read(f01, spans): tracer_provider, _ = spans manager = f01.manager() - calls = [] - real = hashlib.sha256 - monkeypatch.setattr(mc.hashlib, "sha256", lambda data=b"": calls.append(1) or real(data)) + reads = [] hidden = mc.local_capture_policy(TraceConfig(hide_inputs=True), direction="input") with tracer_provider.get_tracer("t").start_as_current_span("s") as span: - assert _inline(manager, span, policy=hidden) is None - assert mc.attach_bytes(manager, span, IMAGE, mc.IMAGE_MIMES, direction="input", purpose="model_input", - capture=hidden) is None - assert calls == [] and f01.after_bootstrap() == [] + assert mc.attach_inline(manager, span, lambda: reads.append(1) or URI, mc.IMAGE_MIMES, raw=False, + direction="input", purpose="model_input", capture=hidden) is None + part = mc.MediaPart("bytes", lambda: reads.append(1) or IMAGE, mc.IMAGE_MIMES, "model_input", 0, 1) + assert mc.capture_parts(manager, span, "test", [part], direction="input", + trace_configs=(TraceConfig(hide_inputs=True),)) == {} + assert reads == [] and f01.after_bootstrap() == [] -def test_concurrent_observers_of_identical_bytes_upload_once(f01, spans): - tracer_provider, _ = spans +def test_concurrent_observers_of_identical_bytes_each_own_one(f01, spans): + tracer_provider, exporter = spans manager = f01.manager() tracer = tracer_provider.get_tracer("t") results = [] - with tracer.start_as_current_span("root") as root: + with tracer.start_as_current_span("root"): + ctx = otel_context.get_current() + def observe(): - with tracer.start_as_current_span("child", context=None) as child: + otel_context.attach(ctx) + with tracer.start_as_current_span("child") as child: results.append(_inline(manager, child)) - from opentelemetry import context as otel_context - ctx = otel_context.get_current() - threads = [threading.Thread(target=lambda: (otel_context.attach(ctx), observe())) for _ in range(8)] + + threads = [threading.Thread(target=observe) for _ in range(8)] [t.start() for t in threads] [t.join() for t in threads] flush(manager) - assert len(set(results)) == 1 and None not in results - assert f01.payloads == [IMAGE] + assert None not in results and len(set(results)) == 8 + assert f01.payloads == [IMAGE] * 8 + children = [s for s in exporter.get_finished_spans() if s.name == "child"] + assert sorted(r["occurrence_id"] for s in children for r in refs(s)) == \ + sorted(o.occurrence_id for o in results) + + +def test_tool_policy_folds_input_and_output_hide_flags(f01, spans): + tracer_provider, _ = spans + manager = f01.manager() + part = mc.MediaPart("bytes", lambda: IMAGE, mc.IMAGE_MIMES, "tool_result", 0, 0, role="tool") + tracer = tracer_provider.get_tracer("t") + for config in (TraceConfig(hide_outputs=True), TraceConfig(hide_output_text=True), + TraceConfig(hide_output_messages=True), TraceConfig(hide_inputs=True), + TraceConfig(hide_input_images=True), TraceConfig(pii_redaction=True)): + with tracer.start_as_current_span("tool") as span: + assert mc.capture_parts(manager, span, "test", [part], direction="input", trace_configs=(config,), + policy_directions=("input", "output")) == {}, config + with tracer.start_as_current_span("tool") as span: + captured = mc.capture_parts(manager, span, "test", [part], direction="input", + trace_configs=(TraceConfig(),), policy_directions=("input", "output")) + flush(manager) + assert len(captured) == 1 and f01.payloads == [IMAGE] + (body,) = f01.uploads + assert body["direction"] == "input" and body["purpose"] == "tool_result" def test_magic_bytes_must_match_and_select_the_exact_type(f01, spans): diff --git a/python/tests/attachments_frameworks/test_provider_dedup.py b/python/tests/attachments_frameworks/test_provider_and_framework.py similarity index 69% rename from python/tests/attachments_frameworks/test_provider_dedup.py rename to python/tests/attachments_frameworks/test_provider_and_framework.py index 15113fb74..a80d5b193 100644 --- a/python/tests/attachments_frameworks/test_provider_dedup.py +++ b/python/tests/attachments_frameworks/test_provider_and_framework.py @@ -1,7 +1,11 @@ -"""Framework + OpenAI instrumentor on ONE manager: the same bytes upload exactly once. +"""Framework + OpenAI instrumentor on ONE manager: each observing span owns its occurrence. Each test drives the real framework against the real OpenAI SDK (loopback provider), with both the framework instrumentor and ``traceai_openai`` given the same manager. +LangChain and LlamaIndex do not make their span current, so the provider span may be +a separate trace: an occurrence shared across spans could not resolve there (F01 +resolve checks the parent). So both spans upload, each references only its own +occurrence, and byte deduplication is left to the server. """ import json @@ -11,7 +15,7 @@ pytest.importorskip("openai") pytest.importorskip("traceai_openai") -from conftest import b64, flush, png, refs # noqa: E402 +from conftest import assert_uris_resolve, b64, flush, png, refs # noqa: E402 from fi_instrumentation import TraceConfig # noqa: E402 from traceai_openai import OpenAIInstrumentor # noqa: E402 @@ -44,15 +48,18 @@ def sent_image(provider): return False -def assert_once(f01, manager, provider, exporter=None): +def assert_each_span_owns_its_occurrence(f01, manager, provider, exporter): + """Framework span + provider span: two uploads, each parented by and referenced on its own span.""" flush(manager) - if exporter is not None: - # Diagnostic only: LangChain/LlamaIndex spans are not made current, so the provider span may - # be a separate trace. The claim is per manager, so the bytes still upload once. - print({s.name: format(s.context.trace_id, "032x") for s in exporter.get_finished_spans()}) assert sent_image(provider), "the framework did not send the image to the provider" - assert f01.payloads == [IMAGE], len(f01.payloads) - assert len(f01.uploads) == 1 + assert f01.payloads == [IMAGE, IMAGE], len(f01.payloads) + owners = [s for s in exporter.get_finished_spans() if refs(s)] + assert len(owners) == 2, [s.name for s in owners] + owned = {r["occurrence_id"]: format(s.context.span_id, "016x") for s in owners for r in refs(s)} + parents = {b["occurrence_id"]: b["association"]["parent"]["span_id"] for b in f01.uploads} + assert owned == parents # every occurrence is referenced on exactly the span it was uploaded under + assert assert_uris_resolve(exporter) >= 1 # the provider span's fi-attachment URI is its own + return owners # ------------------------------------------------------------------ LangChain + langchain-openai @@ -73,9 +80,8 @@ def test_langchain_openai(f01, provider, spans, openai_on): {"type": "image_url", "image_url": {"url": URI}}])]) finally: LangChainInstrumentor().uninstrument() - assert_once(f01, manager, provider, exporter) - owners = [s.name for s in exporter.get_finished_spans() if refs(s)] - assert owners == ["ChatOpenAI"], owners # the framework span (captured first) owns the occurrence + owners = assert_each_span_owns_its_occurrence(f01, manager, provider, exporter) + assert "ChatOpenAI" in [s.name for s in owners] # ------------------------------------------------------------------ LlamaIndex + llama-index-llms-openai @@ -95,7 +101,7 @@ def test_llamaindex_openai(f01, provider, spans, openai_on): image_mimetype="image/png")])]) finally: LlamaIndexInstrumentor().uninstrument() - assert_once(f01, manager, provider, exporter) + assert_each_span_owns_its_occurrence(f01, manager, provider, exporter) # ------------------------------------------------------------------ Agno + OpenAIChat @@ -122,7 +128,7 @@ def test_agno_openai(f01, provider, spans, openai_on, monkeypatch): agent.run("what is this", images=[Image(content=IMAGE, format="png")]) finally: AgnoInstrumentorWrapper().uninstrument() - assert_once(f01, manager, provider) + assert_each_span_owns_its_occurrence(f01, manager, provider, exporter) # ------------------------------------------------------------------ Pydantic AI + OpenAI model @@ -147,19 +153,27 @@ def test_pydantic_ai_openai(f01, provider, spans, openai_on): Agent(model).run_sync(["what is this", BinaryContent(data=IMAGE, media_type="image/png")]) finally: PydanticAIInstrumentor().uninstrument() - assert_once(f01, manager, provider) + assert_each_span_owns_its_occurrence(f01, manager, provider, exporter) -def test_provider_alone_uploads_the_image(f01, provider, spans, openai_on): - """Control: with only traceai_openai on the manager, the provider span uploads the image itself.""" +def test_provider_alone_two_calls_same_image_each_own_their_occurrence(f01, provider, spans, openai_on): + """Provider only (no framework): two independent calls with the same image on one manager. + + Each call is its own trace. Each span uploads, references its own occurrence, and its + telemetry URI points at that occurrence (never at the other trace's). + """ import openai tracer_provider, exporter = spans manager = f01.manager() openai_on(manager) client = openai.OpenAI(api_key="sk-synthetic", base_url=provider.url + "/v1", max_retries=0) - client.chat.completions.create(model="m", messages=[{"role": "user", "content": [ - {"type": "image_url", "image_url": {"url": URI}}]}]) + for _ in range(2): + client.chat.completions.create(model="m", messages=[{"role": "user", "content": [ + {"type": "image_url", "image_url": {"url": URI}}]}]) flush(manager) - assert f01.payloads == [IMAGE] - assert [s.name for s in exporter.get_finished_spans() if refs(s)] + assert f01.payloads == [IMAGE, IMAGE] + owners = [s for s in exporter.get_finished_spans() if refs(s)] + assert len(owners) == 2 and len({s.context.trace_id for s in owners}) == 2 + assert len({r["occurrence_id"] for s in owners for r in refs(s)}) == 2 + assert assert_uris_resolve(exporter) == 2 diff --git a/python/tests/attachments_frameworks/test_pydantic_ai_attachments.py b/python/tests/attachments_frameworks/test_pydantic_ai_attachments.py index 417cb0b75..a9ac6effe 100644 --- a/python/tests/attachments_frameworks/test_pydantic_ai_attachments.py +++ b/python/tests/attachments_frameworks/test_pydantic_ai_attachments.py @@ -8,7 +8,7 @@ pytest.importorskip("pydantic_ai") pytest.importorskip("traceai_pydantic_ai") -from conftest import PDF, b64, flush, png, refs, wav # noqa: E402 +from conftest import PDF, assert_uris_resolve, b64, flush, png, refs, wav # noqa: E402 from fi_instrumentation import TraceConfig # noqa: E402 from fi_instrumentation.attachments import AttachmentConfig, create_attachment_manager # noqa: E402 from pydantic_ai import Agent, BinaryContent, ImageUrl # noqa: E402 @@ -65,9 +65,7 @@ def test_binary_content_prompt(f01, instrument): assert len(owners) == 1 and len(refs(owners[0])) == 3 -def test_tool_result_binary_is_captured_once_across_tool_and_model_spans(f01, instrument): - manager = f01.manager() - exporter = instrument(manager) +def _screenshot_agent(): calls = {"n": 0} def model(messages, info: AgentInfo): @@ -86,15 +84,57 @@ def screenshot() -> BinaryContent: agent.run_sync("take a screenshot") except Exception as exc: # noqa: BLE001 # pydantic-ai 0.1.0's FunctionModel cannot JSON-serialize a binary tool return when it estimates - # usage for the second request; the tool span has already captured it by then. + # usage for the second request; the tool span has already finished by then. if "serializ" not in str(exc).lower(): raise + + +def tool_spans(exporter): + return [s for s in exporter.get_finished_spans() if s.name.startswith("pydantic_ai.tool.call")] + + +def test_tool_result_binary_is_owned_by_each_observing_span(f01, instrument): + manager = f01.manager() + exporter = instrument(manager) + _screenshot_agent() + flush(manager) + (tool,) = [s for s in tool_spans(exporter) if refs(s)] + (ref,) = refs(tool) + assert ref["purpose"] == "tool_result" and ref.get("tool_call_id") == "call_1" + by_occurrence = {b["occurrence_id"]: b for b in f01.uploads} + body = by_occurrence[ref["occurrence_id"]] + assert body["association"]["parent"]["span_id"] == format(tool.context.span_id, "016x") + assert body["purpose"] == "tool_result" and body["direction"] == "input" and body.get("tool_call_id") == "call_1" + # Newer SDKs reach the second model request, which re-sends the tool return as model input: that span + # owns ANOTHER occurrence of the same bytes (never the tool span's). + owners = [s for s in exporter.get_finished_spans() if refs(s)] + assert len(f01.payloads) == len(owners) and set(f01.payloads) == {SCREEN} + for span in owners: + for r in refs(span): + assert by_occurrence[r["occurrence_id"]]["association"]["parent"]["span_id"] == \ + format(span.context.span_id, "016x") + assert_uris_resolve(exporter) + + +@pytest.mark.parametrize("flag", ["hide_outputs", "hide_output_text", "hide_output_messages"]) +def test_tool_span_output_hide_flags_deny_its_tool_result_media(f01, instrument, flag): + """The tool span redacts its result under the OUTPUT hide flags; the same media must not be captured there.""" + manager = f01.manager() + exporter = instrument(manager, config=TraceConfig(**{flag: True})) + _screenshot_agent() flush(manager) - assert f01.payloads == [SCREEN] - (body,) = f01.uploads - assert body["purpose"] == "tool_result" and body.get("tool_call_id") == "call_1" - tool_spans = [s for s in exporter.get_finished_spans() if "tool" in s.name.lower() and refs(s)] - assert len(tool_spans) == 1 + tools = tool_spans(exporter) + assert tools, [s.name for s in exporter.get_finished_spans()] + tool_ids = {format(s.context.span_id, "016x") for s in tools} + for span in tools: + assert refs(span) == [], (flag, span.name) + result = (span.attributes or {}).get("pydantic_ai.tool.result") + assert result is None or result == "__REDACTED__", (flag, result) + assert not [b for b in f01.uploads if b["association"]["parent"]["span_id"] in tool_ids], flag + # Only a later model request (input-governed, so not hidden by an output flag) may own a copy. + owners = {s.name for s in exporter.get_finished_spans() if refs(s)} + assert all(span in {s.name for s in request_spans(exporter)} for span in owners), owners + assert_uris_resolve(exporter) @pytest.mark.skipif(FilePart is None, reason="FilePart needs a newer pydantic-ai") From 2eae0678d5aad2e9c28c4f7af7eb3bd9f3d79747 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek <8347993+nik13@users.noreply.github.com> Date: Wed, 11 Nov 2026 15:37:05 -0800 Subject: [PATCH 09/17] ci(python): name the provider-and-framework tests in the frameworks workflow comment Co-Authored-By: Claude Opus 5.5 --- .github/workflows/attachments-python-frameworks-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/attachments-python-frameworks-tests.yml b/.github/workflows/attachments-python-frameworks-tests.yml index 7fc3f8936..4c6152ea0 100644 --- a/.github/workflows/attachments-python-frameworks-tests.yml +++ b/.github/workflows/attachments-python-frameworks-tests.yml @@ -51,7 +51,7 @@ jobs: langchain-core==0.2.43 llama-index-core==0.12.41 agno==2.7.2 openinference-instrumentation-agno==1.1.0 pydantic-ai-slim[openai]==0.1.0 openai==1.74.0 fastapi 'httpx<0.28' 'opentelemetry-api>=1.28,<1.39' 'opentelemetry-sdk>=1.28,<1.39' - # Latest releases, including the framework OpenAI integrations for the dedup tests. + # Latest releases, including the framework OpenAI integrations for the provider-and-framework tests. - name: latest python: '3.12' sdks: >- From d1d18014c7a1e6e95df1fa47ded0424252fa4737 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek <8347993+nik13@users.noreply.github.com> Date: Wed, 11 Nov 2026 15:43:44 -0800 Subject: [PATCH 10/17] fix(python): never read a denied framework bytes part; correct two occurrence tests capture_parts handed attach_bytes the bytes of a "bytes" part before any preflight, so a part denied by a local hide flag was still read. attach_bytes now also takes a zero-argument callable and calls it only after a preflight at the part cap passed every privacy, correlation and sampling step (too_large / queue_full still size exactly, as in attach_inline). Tests: span refs carry purpose and indices, not tool_call_id (that is in the upload request), and a provider span projects its fi-attachment URI into several attributes, so the provider-only test now checks that every URI resolves to its own span and each span carries its own. Co-Authored-By: Claude Opus 5.5 --- .../attachments/media_capture.py | 23 +++++++++++++++---- .../test_provider_and_framework.py | 7 +++++- .../test_pydantic_ai_attachments.py | 3 ++- 3 files changed, 27 insertions(+), 6 deletions(-) diff --git a/python/fi_instrumentation/attachments/media_capture.py b/python/fi_instrumentation/attachments/media_capture.py index 7853b08dc..509d8a9c8 100644 --- a/python/fi_instrumentation/attachments/media_capture.py +++ b/python/fi_instrumentation/attachments/media_capture.py @@ -319,11 +319,26 @@ def attach_bytes(manager, span, data: Any, mime_types: Tuple[str, ...], *, direc tool_call_id: Optional[str] = None, max_bytes: int = MAX_FRAMEWORK_BYTES) -> Optional[Occurrence]: """Complete in-memory bytes a framework already holds (never a stream, path or URL). Never raises. - Admitted at the exact length under the declared type first; the magic bytes - must then name one of ``mime_types``, re-admitted under that exact type. + ``data`` may be a zero-argument callable returning the bytes: it is called only + after a preflight at ``max_bytes`` passed every privacy, correlation and sampling + step, so a denied part is never read. The bytes are then admitted at their exact + length under the declared type; the magic bytes must name one of ``mime_types``, + re-admitted under that exact type. """ try: - if not isinstance(data, (bytes, bytearray, memoryview)) or not mime_types: + if not mime_types: + return None + if callable(data): + probe = manager.preflight(span, mime_type=mime_types[0], direction=direction, purpose=purpose, + capture_policy=capture, max_bytes=max_bytes) + # too_large and queue_full come after every privacy, correlation and sampling step, + # so reading and sizing exactly is still allowed for them. + if not probe.admitted and probe.reason not in ("too_large", "queue_full"): + return None + if probe.admitted: + probe.release() + data = data() + if not isinstance(data, (bytes, bytearray, memoryview)): return None length = data.nbytes if isinstance(data, memoryview) else len(data) if not 0 < length <= max_bytes: @@ -616,7 +631,7 @@ def capture_parts(manager, span, namespace: str, parts: Iterable[MediaPart], *, manager, span, p.read, p.mime_types, raw=p.raw, max_bytes=max_bytes, verify=True, **f)) elif part.kind == "bytes": observation.visit(position, lambda p=part, f=fields: attach_bytes( - manager, span, p.read(), p.mime_types, max_bytes=max_bytes, **f)) + manager, span, p.read, p.mime_types, max_bytes=max_bytes, **f)) elif part.kind == "url": observation.visit(position, lambda p=part, f=fields: _remote(manager, span, p, f)) except Exception: diff --git a/python/tests/attachments_frameworks/test_provider_and_framework.py b/python/tests/attachments_frameworks/test_provider_and_framework.py index a80d5b193..fcdff241b 100644 --- a/python/tests/attachments_frameworks/test_provider_and_framework.py +++ b/python/tests/attachments_frameworks/test_provider_and_framework.py @@ -176,4 +176,9 @@ def test_provider_alone_two_calls_same_image_each_own_their_occurrence(f01, prov owners = [s for s in exporter.get_finished_spans() if refs(s)] assert len(owners) == 2 and len({s.context.trace_id for s in owners}) == 2 assert len({r["occurrence_id"] for s in owners for r in refs(s)}) == 2 - assert assert_uris_resolve(exporter) == 2 + # A span may project its URI into several attributes; each must resolve to its OWN occurrence. + assert assert_uris_resolve(exporter) >= 2 + for span in owners: + (own,) = [r["occurrence_id"] for r in refs(span)] + assert any("fi-attachment:" + own in value for value in (span.attributes or {}).values() + if isinstance(value, str)), span.name diff --git a/python/tests/attachments_frameworks/test_pydantic_ai_attachments.py b/python/tests/attachments_frameworks/test_pydantic_ai_attachments.py index a9ac6effe..d8927f608 100644 --- a/python/tests/attachments_frameworks/test_pydantic_ai_attachments.py +++ b/python/tests/attachments_frameworks/test_pydantic_ai_attachments.py @@ -100,7 +100,8 @@ def test_tool_result_binary_is_owned_by_each_observing_span(f01, instrument): flush(manager) (tool,) = [s for s in tool_spans(exporter) if refs(s)] (ref,) = refs(tool) - assert ref["purpose"] == "tool_result" and ref.get("tool_call_id") == "call_1" + # Span refs carry purpose and indices only; tool_call_id travels in the upload request. + assert ref["purpose"] == "tool_result" by_occurrence = {b["occurrence_id"]: b for b in f01.uploads} body = by_occurrence[ref["occurrence_id"]] assert body["association"]["parent"]["span_id"] == format(tool.context.span_id, "016x") From 806938904296fd14d65c40a0a80958a2b85750c1 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek <8347993+nik13@users.noreply.github.com> Date: Wed, 11 Nov 2026 16:12:46 -0800 Subject: [PATCH 11/17] test(python): do not assume upload order across two spans Upload workers run concurrently, so the two parents can reach the loopback F01 server in either order (seen once in 15 runs on a VM). Co-Authored-By: Claude Opus 5.5 --- python/tests/attachments_frameworks/test_occurrences.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/python/tests/attachments_frameworks/test_occurrences.py b/python/tests/attachments_frameworks/test_occurrences.py index 54f2c568e..1548c2174 100644 --- a/python/tests/attachments_frameworks/test_occurrences.py +++ b/python/tests/attachments_frameworks/test_occurrences.py @@ -44,8 +44,9 @@ def test_framework_and_provider_spans_each_own_their_occurrence(f01, spans): names = {span.name: refs(span) for span in exporter.get_finished_spans()} assert [r["occurrence_id"] for r in names["framework"]] == [first.occurrence_id] assert [r["occurrence_id"] for r in names["provider"]] == [second.occurrence_id] - assert _parents(f01) == ["%016x" % parent.get_span_context().span_id, - "%016x" % child.get_span_context().span_id] + # Upload workers run concurrently, so the order the server sees them in is not fixed. + assert sorted(_parents(f01)) == sorted(["%016x" % parent.get_span_context().span_id, + "%016x" % child.get_span_context().span_id]) def test_raw_bytes_and_inline_base64_at_two_positions_are_two_occurrences(f01, spans): From 3fe575c40fdf4dc1b44a735e916e8babda37a143 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek <8347993+nik13@users.noreply.github.com> Date: Thu, 12 Nov 2026 07:07:43 -0800 Subject: [PATCH 12/17] feat(python): share the per-call capture-policy and owner context keys Cherry-picked from e1586827 (feat/attachments-python-providers, #697). Every non-docstring hunk of media_capture.py is byte-identical to that commit (and to feat/attachments-python-auto), so merging those branches later is a no-op here. The docstring keeps both bullets; the next commit rewrites the per-span one. (cherry picked from commit e1586827) Co-Authored-By: Claude Opus 5.5 --- .../attachments/media_capture.py | 53 ++++++++++++++++++- 1 file changed, 52 insertions(+), 1 deletion(-) diff --git a/python/fi_instrumentation/attachments/media_capture.py b/python/fi_instrumentation/attachments/media_capture.py index 509d8a9c8..1105371fe 100644 --- a/python/fi_instrumentation/attachments/media_capture.py +++ b/python/fi_instrumentation/attachments/media_capture.py @@ -13,6 +13,11 @@ - One registry per manager remembers admitted (span, position) pairs, so a second observer of the same span and position gets the FIRST occurrence back and uploads nothing. +- A per-call ``CapturePolicy`` set in the OpenTelemetry context under + ``CAPTURE_POLICY_KEY`` (``traceai.with_attachment_capture_policy``) can only + add restrictions. A context marked under ``OWNER_KEY`` + (``traceai.with_attachment_ownership``) belongs to an adapter that captures + the media itself, so nothing is captured here for that call. - Every observing span gets its OWN occurrence. A framework span and the provider span beneath it each upload what they observed, and identical bytes at two positions are two occurrences; byte deduplication is server-side. A @@ -32,6 +37,8 @@ from types import MappingProxyType from typing import Any, Callable, Iterable, Mapping, NamedTuple, Optional, Sequence, Tuple +from opentelemetry import context as otel_context + from .policy import CapturePolicy from .stream import StreamOptions from .types import Bytes, CorrelationHandle, RemoteUrl @@ -61,6 +68,13 @@ # mulaw have no F01 MIME, so they are never captured. SPEECH_FORMATS = {"mp3": "audio/mpeg", "opus": "audio/ogg", "ogg": "audio/ogg", "flac": "audio/flac", "wav": "audio/wav"} +# Context keys shared BY NAME with the auto module (``traceai``) and any adapter, like the +# TypeScript ``Symbol.for`` keys: plain strings, so no package has to import another. +CAPTURE_POLICY_KEY = "traceai.auto.attachment_policy.v1" +OWNER_KEY = "traceai.auto.attachment_owner.v1" +# CapturePolicy facts that grant capture; every other fact only restricts. +_PERMISSIONS = frozenset(("capture_content", "record_inputs", "record_outputs", "requested")) +_DENY_ALL = CapturePolicy(capture_content=False, requested=False) class Occurrence(NamedTuple): @@ -104,6 +118,40 @@ def _restricted(config, name: str) -> bool: return True +def merge_capture_policies(*policies: Any) -> CapturePolicy: + """Deny-wins intersection: a grant needs every policy, a restriction needs any one. + + ``None`` adds nothing. Any other value that is not a ``CapturePolicy`` denies. + """ + present = [policy for policy in policies if policy is not None] + if any(not isinstance(policy, CapturePolicy) for policy in present): + return _DENY_ALL + if not present: + return CapturePolicy() + values = {} + for item in dataclasses.fields(CapturePolicy): + facts = [getattr(policy, item.name) for policy in present] + values[item.name] = all(facts) if item.name in _PERMISSIONS else any(facts) + return CapturePolicy(**values) + + +def context_capture_policy(context=None) -> Any: + """The per-call policy in the (current) OpenTelemetry context; None when unset.""" + try: + return otel_context.get_value(CAPTURE_POLICY_KEY, context) + except Exception: + # An unreadable restriction is a restriction, never permission. + return _DENY_ALL + + +def owned_elsewhere(context=None) -> bool: + """True when another adapter marked this call's media as its own.""" + try: + return otel_context.get_value(OWNER_KEY, context) is not None + except Exception: + return True + + def local_capture_policy(trace_config=None, *, direction: str, image: bool = False, error_detail: bool = False, base: Optional[CapturePolicy] = None) -> CapturePolicy: """Fold the instrumentor's local TraceConfig into a deny-only CapturePolicy. @@ -114,6 +162,7 @@ def local_capture_policy(trace_config=None, *, direction: str, image: bool = Fal ancestry, so a tool screenshot is still treated as model input. """ base = base if isinstance(base, CapturePolicy) else CapturePolicy() + base = merge_capture_policies(base, context_capture_policy()) names = _INPUT_HIDE if direction == "input" else _OUTPUT_HIDE hidden = any(_restricted(trace_config, name) for name in names) or _restricted(trace_config, "pii_redaction") if image and direction == "input" and _restricted(trace_config, "hide_input_images"): @@ -127,6 +176,8 @@ def local_capture_policy(trace_config=None, *, direction: str, image: bool = Fal changes["model_input"] = True if error_detail: changes["error_detail"] = True + if owned_elsewhere(): + changes["requested"] = False return dataclasses.replace(base, **changes) @@ -367,7 +418,7 @@ class Observation: def __init__(self, manager, span, namespace: str): self.manager, self.span, self.namespace = manager, span, namespace - self.key = span_key(span) if active(manager) else None + self.key = span_key(span) if active(manager) and not owned_elsewhere() else None self.captured: "OrderedDict[str, Occurrence]" = OrderedDict() self._visited = 0 From 16311a37bd27d969f9b243d75315c0fb7807a957 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek <8347993+nik13@users.noreply.github.com> Date: Thu, 12 Nov 2026 07:22:51 -0800 Subject: [PATCH 13/17] feat(python): one model call, one owner for the framework adapters When the LangChain, LlamaIndex, Agno or Pydantic AI adapter captures media for a model call (attachments enabled), the downstream provider SDK call now runs under the owner mark (media_capture.OWNER_KEY), so traceai_openai / traceai_anthropic beneath it upload nothing for that call. Framework + OpenAI on one manager goes from 2 uploads per part to 1, held by the framework span. - core: CallOwner, call_owner (gated on an active manager and an unowned call), owner_scope / owned_call, and owned_stream, which marks each advance of a returned (async) generator and never the caller's code between chunks. capture_parts treats a CallOwner of its own namespace as its own, so the LangChain and LlamaIndex callbacks still capture inside the mark. - LangChain: BaseChatModel generate/agenerate/stream/astream marked (patched only while attachments are enabled; restored on uninstrument). - LlamaIndex: chat/achat/stream_chat/astream_chat of every LLM class (patched like StreamInstrumentation, including later-imported providers). - Agno: the owner mark joins span_scope's values (call and each stream advance). - Pydantic AI: request marked; request_stream only while entered and exited, never the caller's async-with body. - DIFFERENT calls still each own their occurrence (same bytes twice = 2). - Attachments off: nothing marked or patched; tracing is the base path. Co-Authored-By: Claude Opus 5.5 --- .../attachments/media_capture.py | 166 +++++- .../agno/traceai_agno/_attachments.py | 9 +- .../traceai_agno/_wrappers/_model_wrapper.py | 25 +- .../langchain/traceai_langchain/__init__.py | 14 +- .../traceai_langchain/_attachments.py | 74 ++- .../traceai_llamaindex/__init__.py | 13 +- .../traceai_llamaindex/_attachments.py | 121 ++++- .../traceai_pydantic_ai/_attachments.py | 12 +- .../_execution_wrappers.py | 57 +- .../test_occurrences.py | 6 +- .../attachments_frameworks/test_owner.py | 124 +++++ .../test_provider_and_framework.py | 485 ++++++++++++++---- 12 files changed, 954 insertions(+), 152 deletions(-) create mode 100644 python/tests/attachments_frameworks/test_owner.py diff --git a/python/fi_instrumentation/attachments/media_capture.py b/python/fi_instrumentation/attachments/media_capture.py index 1105371fe..58364feb3 100644 --- a/python/fi_instrumentation/attachments/media_capture.py +++ b/python/fi_instrumentation/attachments/media_capture.py @@ -18,17 +18,25 @@ add restrictions. A context marked under ``OWNER_KEY`` (``traceai.with_attachment_ownership``) belongs to an adapter that captures the media itself, so nothing is captured here for that call. -- Every observing span gets its OWN occurrence. A framework span and the - provider span beneath it each upload what they observed, and identical bytes - at two positions are two occurrences; byte deduplication is server-side. A - ``fi-attachment:`` reference is only ever projected onto the span whose - ``fi.attachment.refs`` carries that occurrence. +- One model call, one owner. A framework adapter (LangChain, LlamaIndex, Agno, + Pydantic AI) that captures a model call's media runs the downstream provider + SDK call under ``owned_call`` (a ``CallOwner`` under ``OWNER_KEY``), so a + provider instrumentor beneath it (``traceai_openai``, ``traceai_anthropic``, + ...) uploads nothing for that call. Inside its own mark the adapter's + ``capture_parts`` still captures. With attachments off nothing is marked. +- Across DIFFERENT calls every observing span gets its OWN occurrence: + identical bytes in two calls, or at two positions, are two occurrences; + byte deduplication is server-side. A ``fi-attachment:`` reference is only + ever projected onto the span whose ``fi.attachment.refs`` carries that + occurrence. Every public function is non-throwing and returns ``None``/empty on denial. """ import base64 import binascii +import collections.abc +import contextlib import dataclasses import re import threading @@ -644,6 +652,145 @@ def intersect(*policies: CapturePolicy) -> CapturePolicy: return CapturePolicy(**values) +class CallOwner: + """The ``OWNER_KEY`` value a framework adapter sets around ONE model call whose media it captures. + + Any value other than None marks the call as owned (``owned_elsewhere``), so provider + instrumentors beneath the adapter skip it. ``namespace`` is set only by an adapter that + captures INSIDE its own mark (LangChain, LlamaIndex callbacks): ``capture_parts`` with the + same namespace treats the mark as its own and still captures. An adapter that captures + before marking (Agno, Pydantic AI) passes None, so a nested request of the same framework + (a wrapper or fallback model) is owned too. + """ + + __slots__ = ("namespace",) + + def __init__(self, namespace: Optional[str] = None) -> None: + self.namespace = namespace + + +def call_owner(manager, namespace: Optional[str] = None) -> Optional[CallOwner]: + """The owner mark for one model call this adapter captures, or None to leave the context as is. + + None when ``manager`` cannot admit (attachments off: the base code path, nothing marked) + or when the call is already owned (an outer adapter, ``traceai.with_attachment_ownership``). + """ + try: + if not active(manager) or owned_elsewhere(): + return None + return CallOwner(namespace) + except Exception: + return None + + +@contextlib.contextmanager +def owner_scope(owner: Optional[CallOwner]): + """Attach ``owner`` for the enclosed synchronous step or awaited operation, never across a yield.""" + if owner is None: + yield + return + token = otel_context.attach(otel_context.set_value(OWNER_KEY, owner)) + try: + yield + finally: + otel_context.detach(token) + + +def owned_call(manager, namespace: Optional[str] = None): + """Run the enclosed downstream provider SDK call as the owner of its media (see ``call_owner``).""" + return owner_scope(call_owner(manager, namespace)) + + +class _OwnedStream: + """A returned generator advanced (and closed) under the owner mark, restored before every yield. + + Implements the full generator protocol, so ``collections.abc.Generator`` checks still pass; + any other attribute is the wrapped generator's. + """ + + __slots__ = ("_stream", "_owner") + + def __init__(self, stream: Any, owner: CallOwner) -> None: + self._stream, self._owner = stream, owner + + def __getattr__(self, name: str) -> Any: + return getattr(self._stream, name) + + def __iter__(self) -> "_OwnedStream": + return self + + def __next__(self) -> Any: + with owner_scope(self._owner): + return next(self._stream) + + def send(self, value: Any) -> Any: + with owner_scope(self._owner): + return self._stream.send(value) + + def throw(self, *args: Any) -> Any: + with owner_scope(self._owner): + return self._stream.throw(*args) + + def close(self) -> Any: + with owner_scope(self._owner): + return self._stream.close() + + +class _OwnedAsyncStream: + """The async-generator form of ``_OwnedStream``: each awaited step runs under the mark.""" + + __slots__ = ("_stream", "_owner") + + def __init__(self, stream: Any, owner: CallOwner) -> None: + self._stream, self._owner = stream, owner + + def __getattr__(self, name: str) -> Any: + return getattr(self._stream, name) + + def __aiter__(self) -> "_OwnedAsyncStream": + return self + + async def __anext__(self) -> Any: + with owner_scope(self._owner): + return await self._stream.__anext__() + + async def asend(self, value: Any) -> Any: + with owner_scope(self._owner): + return await self._stream.asend(value) + + async def athrow(self, *args: Any) -> Any: + with owner_scope(self._owner): + return await self._stream.athrow(*args) + + async def aclose(self) -> Any: + with owner_scope(self._owner): + return await self._stream.aclose() + + +def owned_stream(result: Any, owner: Optional[CallOwner]) -> Any: + """``result`` itself, or a generator / async generator proxy advanced under ``owner``. + + A lazy SDK stream makes its provider request on the first advance, after the call that + returned it; the mark covers every advance but is never left attached across a yield, so + the caller's code between chunks (and an abandoned stream) is never marked. + """ + if owner is None: + return result + if isinstance(result, collections.abc.Generator): + return _OwnedStream(result, owner) + if isinstance(result, collections.abc.AsyncGenerator): + return _OwnedAsyncStream(result, owner) + return result + + +def _own_mark(namespace: str) -> bool: + try: + owner = otel_context.get_value(OWNER_KEY) + except Exception: + return False + return isinstance(owner, CallOwner) and owner.namespace is not None and owner.namespace == namespace + + def capture_parts(manager, span, namespace: str, parts: Iterable[MediaPart], *, direction: str, trace_configs: Sequence[Any] = (), capture: Optional[CapturePolicy] = None, max_bytes: int = MAX_FRAMEWORK_BYTES, @@ -657,6 +804,15 @@ def capture_parts(manager, span, namespace: str, parts: Iterable[MediaPart], *, denies it. At most ``MAX_FRAMEWORK_PARTS`` positions per span. Never raises, never walks arbitrary objects, never fetches or opens anything. """ + if _own_mark(namespace): + # This adapter owns the call: the mark is for the provider beneath it, not for itself. + token = otel_context.attach(otel_context.set_value(OWNER_KEY, None)) + try: + return capture_parts(manager, span, namespace, parts, direction=direction, + trace_configs=trace_configs, capture=capture, max_bytes=max_bytes, + policy_directions=policy_directions) + finally: + otel_context.detach(token) observation = Observation(manager, span, namespace) if not observation.usable: return EMPTY diff --git a/python/frameworks/agno/traceai_agno/_attachments.py b/python/frameworks/agno/traceai_agno/_attachments.py index c9872d312..47544245c 100644 --- a/python/frameworks/agno/traceai_agno/_attachments.py +++ b/python/frameworks/agno/traceai_agno/_attachments.py @@ -14,8 +14,13 @@ Privacy is the INTERSECTION of the FI TraceConfig and the OpenInference TraceConfig the instrumentor runs with (either one hiding wins), the manager's -own policy, and the per-call capture policy. The Agno span and a provider -instrumentor's span each get their own occurrence (byte deduplication is +own policy, and the per-call capture policy. + +One model call, one owner: while attachments are enabled, the wrapped model call +runs under the owner mark (``media_capture.OWNER_KEY``, a stream only while it +advances), so a provider instrumentor beneath Agno (``traceai_openai``, ...) +uploads nothing for that call; the Agno span holds the occurrence. A DIFFERENT +call with the same bytes gets its own occurrence (byte deduplication is server-side). """ diff --git a/python/frameworks/agno/traceai_agno/_wrappers/_model_wrapper.py b/python/frameworks/agno/traceai_agno/_wrappers/_model_wrapper.py index 16e3ed46b..9289f40b3 100644 --- a/python/frameworks/agno/traceai_agno/_wrappers/_model_wrapper.py +++ b/python/frameworks/agno/traceai_agno/_wrappers/_model_wrapper.py @@ -646,6 +646,23 @@ def _capture_input(self, span: Any, arguments: Mapping[str, Any]) -> None: except Exception: pass + def _owned(self, values: Dict[str, Any]) -> Dict[str, Any]: + """``values`` plus the owner mark while attachments are enabled (one call, one owner). + + span_scope attaches it only around the downstream call and each stream advance, + so the provider instrumentor beneath Agno skips this call's media. Attachments + off, or a call already owned further out: ``values`` unchanged. + """ + if self._attachments is None: + return values + try: + from fi_instrumentation.attachments.media_capture import OWNER_KEY, call_owner + + owner = call_owner(self._attachments) + except Exception: + return values + return values if owner is None else {**values, OWNER_KEY: owner} + def _capture_output(self, span: Any, response: Any) -> None: if self._attachments is None: return @@ -696,7 +713,7 @@ def run( span.set_attribute(LLM_SYSTEM, llm_system) self._capture_input(span, arguments) - with span_scope(span, _invocation_context(instance, "invoke", wrapped)): + with span_scope(span, self._owned(_invocation_context(instance, "invoke", wrapped))): response = wrapped(*args, **kwargs) output_message = _parse_model_output(response) self._capture_output(span, response) @@ -751,7 +768,7 @@ def run_stream( # Token usage will be set after streaming completes based on final response responses = [] - values = _invocation_context(instance, "invoke_stream", wrapped) + values = self._owned(_invocation_context(instance, "invoke_stream", wrapped)) with span_scope(span, values): source = iter(wrapped(*args, **kwargs)) with closing(iterate(source, span, values)) as iterator: @@ -811,7 +828,7 @@ async def arun( span.set_attribute(LLM_SYSTEM, llm_system) self._capture_input(span, arguments) - with span_scope(span, _invocation_context(instance, "ainvoke", wrapped)): + with span_scope(span, self._owned(_invocation_context(instance, "ainvoke", wrapped))): response = await wrapped(*args, **kwargs) output_message = _parse_model_output(response) self._capture_output(span, response) @@ -868,7 +885,7 @@ async def arun_stream( # Token usage will be set after streaming completes based on final response responses = [] - values = _invocation_context(instance, "ainvoke_stream", wrapped) + values = self._owned(_invocation_context(instance, "ainvoke_stream", wrapped)) with span_scope(span, values): source = wrapped(*args, **kwargs) async with aclosing(aiterate(source, span, values)) as iterator: diff --git a/python/frameworks/langchain/traceai_langchain/__init__.py b/python/frameworks/langchain/traceai_langchain/__init__.py index 61e8be978..8d79b6df9 100644 --- a/python/frameworks/langchain/traceai_langchain/__init__.py +++ b/python/frameworks/langchain/traceai_langchain/__init__.py @@ -80,11 +80,18 @@ def _instrument(self, **kwargs: Any) -> None: trace_api.get_tracer(__name__, __version__, tracer_provider), config=config, ) + attachments = _attachment_manager(kwargs) self._tracer: Optional[LangChainFiTracer] = LangChainFiTracer( self.fi_tracer, - attachments=_attachment_manager(kwargs), + attachments=attachments, trace_config=config, ) + # One model call, one owner: only while attachments are enabled (else nothing is patched). + self._owner_patches: List[Any] = [] + if attachments is not None: + from traceai_langchain._attachments import own_chat_model_calls + + self._owner_patches = own_chat_model_calls(attachments) self._original_callback_manager_init = BaseCallbackManager.__init__ wrap_function_wrapper( module="langchain_core.callbacks", @@ -103,6 +110,11 @@ def _instrument(self, **kwargs: Any) -> None: self._original_protect = None def _uninstrument(self, **kwargs: Any) -> None: + if getattr(self, "_owner_patches", None): + from traceai_langchain._attachments import release_chat_model_calls + + release_chat_model_calls(self._owner_patches) + self._owner_patches = [] langchain_core.callbacks.BaseCallbackManager.__init__ = self._original_callback_manager_init # type: ignore self._original_callback_manager_init = None # type: ignore if self._protect_class is not None: diff --git a/python/frameworks/langchain/traceai_langchain/_attachments.py b/python/frameworks/langchain/traceai_langchain/_attachments.py index 58ce71201..556826040 100644 --- a/python/frameworks/langchain/traceai_langchain/_attachments.py +++ b/python/frameworks/langchain/traceai_langchain/_attachments.py @@ -10,16 +10,22 @@ - Anthropic-style ``source: {"type": "base64"}`` blocks. Remote URLs go to the core's ``RemoteUrl`` policy and are never fetched; provider -file IDs and anything else are ignored. The LangChain span and a provider -instrumentor's span each get their own occurrence of what they observed (byte -deduplication is server-side). +file IDs and anything else are ignored. + +One model call, one owner: while attachments are enabled, ``BaseChatModel`` +``generate`` / ``agenerate`` / ``stream`` / ``astream`` run under the owner mark +(``media_capture.owned_call``), so a provider instrumentor beneath LangChain +(``traceai_openai``, ...) uploads nothing for that call; the LangChain span holds +the occurrence. A DIFFERENT call with the same bytes gets its own occurrence (byte +deduplication is server-side). The mark also carries LangChain's hide flags down: +what the LangChain span may not capture, the provider span does not capture either. This module imports ``fi_instrumentation.attachments``; the tracer imports it only when an enabled attachment manager is passed. """ from types import MappingProxyType -from typing import Any, Iterator, List, Mapping, Optional +from typing import Any, Iterator, List, Mapping, Optional, Tuple from fi_instrumentation.attachments.media_capture import ( AUDIO_MIMES, @@ -30,12 +36,17 @@ MAX_FRAMEWORK_PARTS, MediaPart, Occurrence, + call_owner, capture_parts, get, items, + owned_stream, + owner_scope, ) +from wrapt import FunctionWrapper -__all__ = ("capture_langchain_messages", "langchain_attachment_capabilities") +__all__ = ("capture_langchain_messages", "langchain_attachment_capabilities", "own_chat_model_calls", + "release_chat_model_calls") langchain_attachment_capabilities = MappingProxyType({ "schema_version": 1, @@ -180,3 +191,56 @@ def run_messages(run: Any, direction: str) -> Optional[list]: if message is not None] except Exception: return None + + +# ------------------------------------------------------------------ one call, one owner +# BaseChatModel entry points that run one or more chat-model calls (invoke/batch go through +# generate/agenerate). The tracer captures inside them, so the mark carries this namespace. +_OWNED_METHODS = ("generate", "agenerate", "stream", "astream") +_NAMESPACE = "langchain" + + +def _owned(manager): + def wrapper(wrapped, instance, args, kwargs): + owner = call_owner(manager, _NAMESPACE) + if owner is None: + return wrapped(*args, **kwargs) + with owner_scope(owner): + result = wrapped(*args, **kwargs) + # stream/astream return a lazy generator: mark each advance, never across a yield. + return owned_stream(result, owner) + return wrapper + + +def _owned_async(manager): + async def wrapper(wrapped, instance, args, kwargs): + owner = call_owner(manager, _NAMESPACE) + if owner is None: + return await wrapped(*args, **kwargs) + with owner_scope(owner): + return await wrapped(*args, **kwargs) + return wrapper + + +def own_chat_model_calls(manager) -> List[Tuple[type, str, Any]]: + """Run every chat-model call under the owner mark while ``manager`` can admit. + + A provider instrumentor beneath LangChain then uploads nothing for that call; the + LangChain span owns its media. Returns the patches for ``release_chat_model_calls``. + """ + from langchain_core.language_models.chat_models import BaseChatModel + + patches = [] + for name in _OWNED_METHODS: + original = BaseChatModel.__dict__.get(name) + if original is None: + continue + wrapper = _owned_async(manager) if name == "agenerate" else _owned(manager) + setattr(BaseChatModel, name, FunctionWrapper(original, wrapper)) + patches.append((BaseChatModel, name, original)) + return patches + + +def release_chat_model_calls(patches: List[Tuple[type, str, Any]]) -> None: + for cls, name, original in reversed(patches): + setattr(cls, name, original) diff --git a/python/frameworks/llama_index/traceai_llamaindex/__init__.py b/python/frameworks/llama_index/traceai_llamaindex/__init__.py index ddd59556f..b5af16e25 100644 --- a/python/frameworks/llama_index/traceai_llamaindex/__init__.py +++ b/python/frameworks/llama_index/traceai_llamaindex/__init__.py @@ -87,6 +87,7 @@ def _instrument(self, **kwargs: Any) -> None: self._event_handler = None self._workflow_run_patches = [] self._stream_instrumentation = None + self._chat_ownership = None if self._use_legacy_callback_handler: import llama_index.core @@ -100,8 +101,9 @@ def _instrument(self, **kwargs: Any) -> None: from ._handler import EventHandler, instrument_workflow_runs from ._stream import StreamInstrumentation + attachments = _attachment_manager(kwargs) self._event_handler = EventHandler( - tracer=tracer, config=config, attachments=_attachment_manager(kwargs) + tracer=tracer, config=config, attachments=attachments ) self._span_handler = self._event_handler._span_handler dispatcher = get_dispatcher() @@ -118,6 +120,12 @@ def _instrument(self, **kwargs: Any) -> None: self._workflow_run_patches = instrument_workflow_runs() self._stream_instrumentation = StreamInstrumentation() self._stream_instrumentation.install() + if attachments is not None: + # One model call, one owner: only while attachments are enabled. + from ._attachments import ChatOwnership + + self._chat_ownership = ChatOwnership(attachments) + self._chat_ownership.install() if Protect is not None: self._original_protect = Protect.protect @@ -142,6 +150,9 @@ def _uninstrument(self, **kwargs: Any) -> None: from ._handler import uninstrument_workflow_runs + if getattr(self, "_chat_ownership", None) is not None: + self._chat_ownership.uninstall() + self._chat_ownership = None if self._stream_instrumentation is not None: self._stream_instrumentation.uninstall() self._stream_instrumentation = None diff --git a/python/frameworks/llama_index/traceai_llamaindex/_attachments.py b/python/frameworks/llama_index/traceai_llamaindex/_attachments.py index e0ddf638f..a0958373a 100644 --- a/python/frameworks/llama_index/traceai_llamaindex/_attachments.py +++ b/python/frameworks/llama_index/traceai_llamaindex/_attachments.py @@ -9,16 +9,23 @@ - the magic bytes must match the admitted type. An ``IOBase`` source is never read, a ``path`` is never opened, and a ``url`` goes -to the core's ``RemoteUrl`` policy without being fetched. The LlamaIndex span and a -provider instrumentor's span each get their own occurrence (byte deduplication is -server-side). +to the core's ``RemoteUrl`` policy without being fetched. + +One model call, one owner: while attachments are enabled, every LLM's ``chat`` / +``achat`` / ``stream_chat`` / ``astream_chat`` (the calls whose messages are observed +here) runs under the owner mark (``media_capture.owned_call``; a stream only while it +advances), so a provider instrumentor beneath LlamaIndex (``traceai_openai``, ...) +uploads nothing for that call; the LlamaIndex span holds the occurrence. Completion +calls are not observed here and are not marked. A DIFFERENT call with the same bytes +gets its own occurrence (byte deduplication is server-side). This module imports ``fi_instrumentation.attachments``; the handler imports it only when an enabled attachment manager is passed. """ +from threading import RLock from types import MappingProxyType -from typing import Any, Iterator, Mapping, Optional, Sequence +from typing import Any, Dict, Iterator, Mapping, Optional, Sequence, Tuple from fi_instrumentation.attachments.media_capture import ( AUDIO_MIMES, @@ -29,11 +36,15 @@ MAX_FRAMEWORK_PARTS, MediaPart, Occurrence, + call_owner, capture_parts, looks_base64, + owned_stream, + owner_scope, ) +from wrapt import FunctionWrapper -__all__ = ("capture_llamaindex_messages", "llamaindex_attachment_capabilities") +__all__ = ("ChatOwnership", "capture_llamaindex_messages", "llamaindex_attachment_capabilities") llamaindex_attachment_capabilities = MappingProxyType({ "schema_version": 1, @@ -122,3 +133,103 @@ def capture_llamaindex_messages(manager, span, messages: Any, *, direction: str, trace_configs=(trace_config,), capture=capture) except Exception: return EMPTY + + +# ------------------------------------------------------------------ one call, one owner +# The LLM methods whose messages this module observes (LLMChatStart/End events). Completion +# methods are not observed, so they are never marked. +_SYNC_METHODS = ("chat", "stream_chat") +_ASYNC_METHODS = ("achat", "astream_chat") +_NAMESPACE = "llamaindex" +_MISSING = object() + + +class ChatOwnership: + """Run every LLM chat call under the owner mark while ``manager`` can admit. + + Patched like ``StreamInstrumentation``: every loaded ``BaseLLM`` / ``MultiModalLLM`` + subclass that defines a chat method, plus an ``__init__`` hook for providers imported + later. A returned stream is marked only while it advances. Install after and uninstall + before ``StreamInstrumentation``. + """ + + def __init__(self, manager: Any) -> None: + self._manager = manager + self._patches: Dict[Tuple[type, str], Any] = {} + self._lock = RLock() + self._enabled = True + + def install(self) -> None: + from llama_index.core.base.llms.base import BaseLLM + from llama_index.core.multi_modal_llms import MultiModalLLM + + seen = set() + + def visit(cls: type) -> None: + if cls in seen: + return + seen.add(cls) + self._patch_class(cls) + for child in cls.__subclasses__(): + visit(child) + + for base in (BaseLLM, MultiModalLLM): + visit(base) + self._patch(base, "__init__", self._initialize) + + def _patch_class(self, cls: type) -> None: + for base in reversed(cls.__mro__): + for name in _SYNC_METHODS + _ASYNC_METHODS: + attr = base.__dict__.get(name) + if attr is not None and not getattr(attr, "__isabstractmethod__", False): + self._patch(base, name, self._call_async if name in _ASYNC_METHODS else self._call) + + def _patch(self, cls: type, name: str, wrapper: Any) -> None: + with self._lock: + if not self._enabled or (cls, name) in self._patches: + return + attr = getattr(cls, name) + self._patches[cls, name] = cls.__dict__.get(name, _MISSING) + setattr(cls, name, FunctionWrapper(attr, wrapper)) + + def _initialize(self, wrapped: Any, instance: Any, args: Any, kwargs: Any) -> Any: + # After the inner hooks, so a later provider class is patched in install order too. + result = wrapped(*args, **kwargs) + if self._enabled: + self._patch_class(type(instance)) + return result + + def _owner(self): + return call_owner(self._manager, _NAMESPACE) if self._enabled else None + + def _call(self, wrapped: Any, instance: Any, args: Any, kwargs: Any) -> Any: + owner = self._owner() + if owner is None: + return wrapped(*args, **kwargs) + with owner_scope(owner): + result = wrapped(*args, **kwargs) + return owned_stream(result, owner) + + async def _call_async(self, wrapped: Any, instance: Any, args: Any, kwargs: Any) -> Any: + owner = self._owner() + if owner is None: + return await wrapped(*args, **kwargs) + with owner_scope(owner): + result = await wrapped(*args, **kwargs) + return owned_stream(result, owner) + + def uninstall(self) -> None: + self._enabled = False + with self._lock: + for (cls, name), original in reversed(list(self._patches.items())): + attr = cls.__dict__.get(name) + if ( + not isinstance(attr, FunctionWrapper) + or getattr(attr._self_wrapper, "__self__", None) is not self + ): + continue # Do not overwrite another instrumentor's patch. + if original is _MISSING: + delattr(cls, name) + else: + setattr(cls, name, original) + self._patches.clear() diff --git a/python/frameworks/pydantic-ai/traceai_pydantic_ai/_attachments.py b/python/frameworks/pydantic-ai/traceai_pydantic_ai/_attachments.py index 6e386e766..fcb44e88c 100644 --- a/python/frameworks/pydantic-ai/traceai_pydantic_ai/_attachments.py +++ b/python/frameworks/pydantic-ai/traceai_pydantic_ai/_attachments.py @@ -15,9 +15,15 @@ ``BinaryContent.data`` must be complete bytes whose magic bytes match its declared ``media_type``. Videos, uploaded provider files and unknown content -objects are not captured, and nothing is serialized to look for media. Every -observing span (tool span, model request span, a provider instrumentor's span) -gets its own occurrence; byte deduplication is server-side. +objects are not captured, and nothing is serialized to look for media. + +One model call, one owner: while attachments are enabled, the model request runs +under the owner mark (``media_capture.owned_call``; a streamed request only while +the stream is entered and exited, never the caller's ``async with`` body), so a +provider instrumentor beneath Pydantic AI (``traceai_openai``, ...) uploads nothing +for that call; the model request span holds the occurrence. DIFFERENT calls each +get their own occurrence (a tool span's result and the next request that re-sends +it are two observations; byte deduplication is server-side). """ import dataclasses diff --git a/python/frameworks/pydantic-ai/traceai_pydantic_ai/_execution_wrappers.py b/python/frameworks/pydantic-ai/traceai_pydantic_ai/_execution_wrappers.py index a59041b55..f368b4498 100644 --- a/python/frameworks/pydantic-ai/traceai_pydantic_ai/_execution_wrappers.py +++ b/python/frameworks/pydantic-ai/traceai_pydantic_ai/_execution_wrappers.py @@ -99,23 +99,26 @@ class _ModelStreamContext: The span may outlive either task, but a ContextVar token may not cross them. """ - def __init__(self, manager, model, span): + def __init__(self, manager, model, span, owner=None): self.manager = manager self.model = model self.span = span + # The attachment owner mark (one call, one owner): entered and exited, never the caller's body. + self.owner = owner @contextmanager def _current(self): - if self.span is None: - yield - return - token = _ACTIVE_MODEL_REQUEST.set((self.model, _ACTIVE_USAGE.get())) - try: - with use_span(self.span, end_on_exit=False, record_exception=False, - set_status_on_exception=False): + with _owner_scope(self.owner): + if self.span is None: yield - finally: - _ACTIVE_MODEL_REQUEST.reset(token) + return + token = _ACTIVE_MODEL_REQUEST.set((self.model, _ACTIVE_USAGE.get())) + try: + with use_span(self.span, end_on_exit=False, record_exception=False, + set_status_on_exception=False): + yield + finally: + _ACTIVE_MODEL_REQUEST.reset(token) async def __aenter__(self): with self._current(): @@ -126,6 +129,31 @@ async def __aexit__(self, exc_type, exc_value, traceback): return await self.manager.__aexit__(exc_type, exc_value, traceback) +def _call_owner(media): + """The owner mark for one model request while attachments are enabled, else None.""" + if media is None: + return None + try: + from fi_instrumentation.attachments.media_capture import call_owner + + return call_owner(media.manager) + except Exception: + return None + + +@contextmanager +def _owner_scope(owner): + """Run the enclosed provider SDK call as the owner of its media, so the provider + instrumentor beneath Pydantic AI skips it. No mark: the context is unchanged.""" + if owner is None: + yield + return + from fi_instrumentation.attachments.media_capture import owner_scope + + with owner_scope(owner): + yield + + def _request_messages(args, kwargs): return args[0] if args else kwargs.get("messages") @@ -149,8 +177,10 @@ async def stream_wrapper(self, *args, **kwargs): with _model_request_span(self, tracer, current=False) as span: # Input only: streamed output is not captured. _capture(media, "request", span, _request_messages(args, kwargs)) - manager = original(self, *args, **kwargs) - async with _ModelStreamContext(manager, self, span) as response: + owner = _call_owner(media) + with _owner_scope(owner): + manager = original(self, *args, **kwargs) + async with _ModelStreamContext(manager, self, span, owner) as response: yield response _record_model_usage(span, response) return stream_wrapper @@ -160,7 +190,8 @@ async def wrapper(self, *args, **kwargs): with _model_request_span(self, tracer) as span: # Input media is captured before the provider call; this span owns its own occurrences. _capture(media, "request", span, _request_messages(args, kwargs)) - response = await original(self, *args, **kwargs) + with _owner_scope(_call_owner(media)): + response = await original(self, *args, **kwargs) _record_model_usage(span, response) # Only a successful response: an error never reaches this line. _capture(media, "response", span, response) diff --git a/python/tests/attachments_frameworks/test_occurrences.py b/python/tests/attachments_frameworks/test_occurrences.py index 1548c2174..4650af2d3 100644 --- a/python/tests/attachments_frameworks/test_occurrences.py +++ b/python/tests/attachments_frameworks/test_occurrences.py @@ -1,7 +1,9 @@ """Shared helper: per-span occurrences, raw bytes, URL policy and part limits (no framework installed). F01 resolves an occurrence only under its own parent span, so every observing span owns -its own occurrence: nothing here reuses an occurrence across spans or traces. +its own occurrence: nothing here reuses an occurrence across spans or traces. Two layers of +ONE model call do not both observe it: the framework adapter's owner mark makes the provider +skip (``test_owner.py``, ``test_provider_and_framework.py``); here no mark is set. """ import http.server @@ -30,7 +32,7 @@ def _parents(f01): return [body["association"]["parent"]["span_id"] for body in f01.uploads] -def test_framework_and_provider_spans_each_own_their_occurrence(f01, spans): +def test_two_unmarked_observing_spans_each_own_their_occurrence(f01, spans): tracer_provider, exporter = spans manager = f01.manager() tracer = tracer_provider.get_tracer("t") diff --git a/python/tests/attachments_frameworks/test_owner.py b/python/tests/attachments_frameworks/test_owner.py new file mode 100644 index 000000000..044e31e9c --- /dev/null +++ b/python/tests/attachments_frameworks/test_owner.py @@ -0,0 +1,124 @@ +"""One model call, one owner: the core owner-mark helpers (no framework installed).""" + +import asyncio + +from conftest import b64, flush, png, refs +from fi_instrumentation.attachments import AttachmentConfig, create_attachment_manager +from fi_instrumentation.attachments import media_capture as mc +from opentelemetry import context as otel_context + +IMAGE = png(seed=21) +URI = "data:image/png;base64," + b64(IMAGE) + + +def _part(): + return mc.MediaPart("inline", lambda: URI, mc.IMAGE_MIMES, "model_input", 0, 0, role="user", raw=False) + + +def _capture(manager, span, namespace="fw"): + return mc.capture_parts(manager, span, namespace, [_part()], direction="input") + + +def _under(value): + return otel_context.attach(otel_context.set_value(mc.OWNER_KEY, value)) + + +def test_call_owner_is_gated_on_an_enabled_manager_and_an_unowned_call(f01): + assert mc.call_owner(None, "fw") is None + assert mc.call_owner(create_attachment_manager(config=AttachmentConfig(enabled=False)), "fw") is None + manager = f01.manager() + owner = mc.call_owner(manager, "fw") + assert isinstance(owner, mc.CallOwner) and owner.namespace == "fw" + token = _under(True) # an outer adapter or traceai.with_attachment_ownership + try: + assert mc.call_owner(manager, "fw") is None + finally: + otel_context.detach(token) + + +def test_owned_call_marks_only_the_enclosed_block(): + assert not mc.owned_elsewhere() + with mc.owner_scope(mc.CallOwner("fw")): + assert mc.owned_elsewhere() + assert not mc.owned_elsewhere() + try: + with mc.owner_scope(mc.CallOwner("fw")): + raise RuntimeError("provider failed") + except RuntimeError: + pass + assert not mc.owned_elsewhere() + with mc.owner_scope(None): + assert not mc.owned_elsewhere() + + +def test_capture_parts_captures_inside_its_own_mark_only(f01, spans): + tracer_provider, exporter = spans + manager = f01.manager() + tracer = tracer_provider.get_tracer("t") + results = {} + for label, mark, namespace in (("own", mc.CallOwner("fw"), "fw"), ("other", mc.CallOwner("other"), "fw"), + ("anonymous", mc.CallOwner(None), "fw"), ("user", True, "fw")): + with tracer.start_as_current_span(label) as span: + token = _under(mark) + try: + results[label] = _capture(manager, span, namespace) + # The adapter's capture restores the mark for the provider call that follows. + assert otel_context.get_value(mc.OWNER_KEY) is mark + finally: + otel_context.detach(token) + flush(manager) + assert len(results["own"]) == 1 + assert results["other"] == {} and results["anonymous"] == {} and results["user"] == {} + assert f01.payloads == [IMAGE] + assert [s.name for s in exporter.get_finished_spans() if refs(s)] == ["own"] + + +def test_owned_stream_marks_each_advance_never_the_callers_code(): + seen = [] + + def stream(): + seen.append(mc.owned_elsewhere()) + yield 1 + seen.append(mc.owned_elsewhere()) + yield 2 + + owner = mc.CallOwner("fw") + wrapped = mc.owned_stream(stream(), owner) + caller = [] + for _ in wrapped: + caller.append(mc.owned_elsewhere()) + assert seen == [True, True] and caller == [False, False] + + closed = [] + + def abandoned(): + try: + yield 1 + yield 2 + finally: + closed.append(mc.owned_elsewhere()) + + wrapped = mc.owned_stream(abandoned(), owner) + next(wrapped) + assert not mc.owned_elsewhere() # suspended mid-call: the caller's context is unmarked + wrapped.close() + assert closed == [True] and not mc.owned_elsewhere() + assert mc.owned_stream(stream(), None) is not None + assert mc.owned_stream([1], owner) == [1] # not a generator: returned as is + + +def test_owned_async_stream_marks_each_awaited_step_only(): + seen, caller = [], [] + + async def stream(): + seen.append(mc.owned_elsewhere()) + yield 1 + seen.append(mc.owned_elsewhere()) + yield 2 + + async def run(): + async for _ in mc.owned_stream(stream(), mc.CallOwner("fw")): + caller.append(mc.owned_elsewhere()) + + asyncio.run(run()) + assert seen == [True, True] and caller == [False, False] diff --git a/python/tests/attachments_frameworks/test_provider_and_framework.py b/python/tests/attachments_frameworks/test_provider_and_framework.py index fcdff241b..1ff9bf755 100644 --- a/python/tests/attachments_frameworks/test_provider_and_framework.py +++ b/python/tests/attachments_frameworks/test_provider_and_framework.py @@ -1,13 +1,14 @@ -"""Framework + OpenAI instrumentor on ONE manager: each observing span owns its occurrence. - -Each test drives the real framework against the real OpenAI SDK (loopback provider), -with both the framework instrumentor and ``traceai_openai`` given the same manager. -LangChain and LlamaIndex do not make their span current, so the provider span may be -a separate trace: an occurrence shared across spans could not resolve there (F01 -resolve checks the parent). So both spans upload, each references only its own -occurrence, and byte deduplication is left to the server. +"""Framework + OpenAI instrumentor on ONE manager: one model call, one owner. + +Each test drives the real framework against the real OpenAI SDK (loopback provider). When a +framework adapter captures a model call's media it runs the downstream SDK call under the owner +mark (``media_capture.OWNER_KEY``), so ``traceai_openai`` uploads nothing for that call: each +part is uploaded exactly once, under the framework span. DIFFERENT calls still each own their +occurrence, even with identical bytes (byte deduplication is server-side). With attachments off +for the framework nothing is marked and the provider captures as before. """ +import asyncio import json import pytest @@ -21,6 +22,7 @@ IMAGE = png(seed=31) URI = "data:image/png;base64," + b64(IMAGE) +OPENAI_SCOPE = "traceai_openai" @pytest.fixture @@ -38,122 +40,387 @@ def install(manager, config=None): OpenAIInstrumentor().uninstrument() -def sent_image(provider): - """The image the provider actually received, so the test proves the bytes are the same ones.""" - for path, body, raw in provider.requests: - if path.endswith("/chat/completions"): - text = json.dumps(body) - if b64(IMAGE) in text: - return True - return False +def sent_image(provider, count=1): + """The provider actually received the image ``count`` times (the same bytes, not a copy).""" + seen = sum(1 for path, body, raw in provider.requests + if path.endswith("/chat/completions") and b64(IMAGE) in json.dumps(body)) + return seen >= count + + +def owners(exporter): + return [s for s in exporter.get_finished_spans() if refs(s)] + +def is_provider_span(span): + return getattr(span.instrumentation_scope, "name", "") == OPENAI_SCOPE -def assert_each_span_owns_its_occurrence(f01, manager, provider, exporter): - """Framework span + provider span: two uploads, each parented by and referenced on its own span.""" + +def assert_one_owner(f01, manager, provider, exporter): + """Exactly one upload, parented by and referenced on the framework span; the provider span holds none.""" flush(manager) assert sent_image(provider), "the framework did not send the image to the provider" - assert f01.payloads == [IMAGE, IMAGE], len(f01.payloads) - owners = [s for s in exporter.get_finished_spans() if refs(s)] - assert len(owners) == 2, [s.name for s in owners] - owned = {r["occurrence_id"]: format(s.context.span_id, "016x") for s in owners for r in refs(s)} - parents = {b["occurrence_id"]: b["association"]["parent"]["span_id"] for b in f01.uploads} - assert owned == parents # every occurrence is referenced on exactly the span it was uploaded under - assert assert_uris_resolve(exporter) >= 1 # the provider span's fi-attachment URI is its own - return owners - - -# ------------------------------------------------------------------ LangChain + langchain-openai -def test_langchain_openai(f01, provider, spans, openai_on): - pytest.importorskip("langchain_openai") - from langchain_core.messages import HumanMessage - from langchain_openai import ChatOpenAI - from traceai_langchain import LangChainInstrumentor + assert f01.payloads == [IMAGE], len(f01.payloads) + (owner,) = owners(exporter) + assert not is_provider_span(owner), owner.name + assert any(is_provider_span(s) for s in exporter.get_finished_spans()), "no provider span: not a 2-layer call" + (ref,) = refs(owner) + (upload,) = f01.uploads + assert upload["occurrence_id"] == ref["occurrence_id"] + assert upload["association"]["parent"]["span_id"] == format(owner.context.span_id, "016x") + assert_uris_resolve(exporter) + return owner - tracer_provider, exporter = spans + +def direct_openai_call(provider): + import openai + + client = openai.OpenAI(api_key="sk-synthetic", base_url=provider.url + "/v1", max_retries=0) + client.chat.completions.create(model="m", messages=[{"role": "user", "content": [ + {"type": "image_url", "image_url": {"url": URI}}]}]) + + +async def _drain(stream): + async for _ in stream: + pass + + +# ------------------------------------------------------------------ framework drivers +class LangChain: + name = "langchain" + modes = ("sync", "stream", "async", "astream") + + def __init__(self, provider, tracer_provider): + pytest.importorskip("langchain_openai") + self.provider, self.tracer_provider = provider, tracer_provider + + def instrument(self, manager, config=None): + from traceai_langchain import LangChainInstrumentor + + kwargs = {"attachments": manager} if manager is not None else {} + LangChainInstrumentor().instrument(tracer_provider=self.tracer_provider, config=config or TraceConfig(), + **kwargs) + + def uninstrument(self): + from traceai_langchain import LangChainInstrumentor + + LangChainInstrumentor().uninstrument() + + def _model(self): + from langchain_openai import ChatOpenAI + + return ChatOpenAI(model="gpt-4o-mini", api_key="sk-synthetic", base_url=self.provider.url + "/v1", + max_retries=0) + + def _messages(self): + from langchain_core.messages import HumanMessage + + return [HumanMessage(content=[{"type": "text", "text": "what"}, + {"type": "image_url", "image_url": {"url": URI}}])] + + def call(self, mode="sync"): + model, messages = self._model(), self._messages() + if mode == "sync": + model.invoke(messages) + elif mode == "stream": + for _ in model.stream(messages): + pass + elif mode == "async": + asyncio.run(model.ainvoke(messages)) + else: + asyncio.run(_drain(model.astream(messages))) + + def open_stream(self): + stream = self._model().stream(self._messages()) + next(stream) # the provider request has been made; the stream is suspended mid-call + return stream + + +class LlamaIndex: + name = "llamaindex" + modes = ("sync", "stream", "async", "astream") + + def __init__(self, provider, tracer_provider): + pytest.importorskip("llama_index.llms.openai") + self.provider, self.tracer_provider = provider, tracer_provider + + def instrument(self, manager, config=None): + from traceai_llamaindex import LlamaIndexInstrumentor + + kwargs = {"attachments": manager} if manager is not None else {} + LlamaIndexInstrumentor().instrument(tracer_provider=self.tracer_provider, config=config or TraceConfig(), + **kwargs) + + def uninstrument(self): + from traceai_llamaindex import LlamaIndexInstrumentor + + LlamaIndexInstrumentor().uninstrument() + + def _llm(self): + from llama_index.llms.openai import OpenAI + + return OpenAI(model="gpt-4o-mini", api_key="sk-synthetic", api_base=self.provider.url + "/v1", + max_retries=0) + + def _messages(self): + from llama_index.core.base.llms.types import ChatMessage, ImageBlock, TextBlock + + return [ChatMessage(role="user", blocks=[TextBlock(text="what"), + ImageBlock(image=IMAGE, image_mimetype="image/png")])] + + def call(self, mode="sync"): + llm, messages = self._llm(), self._messages() + if mode == "sync": + llm.chat(messages) + elif mode == "stream": + for _ in llm.stream_chat(messages): + pass + elif mode == "async": + asyncio.run(llm.achat(messages)) + else: + async def run(): + await _drain(await llm.astream_chat(messages)) + asyncio.run(run()) + + def open_stream(self): + stream = self._llm().stream_chat(self._messages()) + next(stream) + return stream + + +class Agno: + name = "agno" + modes = ("sync", "stream") + + def __init__(self, provider, tracer_provider): + pytest.importorskip("agno") + pytest.importorskip("openinference.instrumentation.agno") + self.provider, self.tracer_provider = provider, tracer_provider + + def instrument(self, manager, config=None): + from traceai_agno import configure_agno_tracing + + assert config is None, "Agno takes an OpenInference TraceConfig; not exercised here" + kwargs = {"attachments": manager} if manager is not None else {} + configure_agno_tracing(tracer_provider=self.tracer_provider, **kwargs) + + def uninstrument(self): + from traceai_agno import AgnoInstrumentorWrapper + + AgnoInstrumentorWrapper().uninstrument() + + def _agent(self): + from agno.agent import Agent + from agno.models.openai import OpenAIChat + + model = OpenAIChat(id="gpt-4o-mini", api_key="sk-synthetic", base_url=self.provider.url + "/v1", + max_retries=0) + try: + return Agent(model=model, telemetry=False) + except TypeError: + return Agent(model=model) + + def call(self, mode="sync"): + from agno.media import Image + + images = [Image(content=IMAGE, format="png")] + if mode == "sync": + self._agent().run("what is this", images=images) + else: + for _ in self._agent().run("what is this", images=images, stream=True): + pass + + +class PydanticAI: + name = "pydantic_ai" + modes = ("sync", "stream") + + def __init__(self, provider, tracer_provider): + pytest.importorskip("pydantic_ai") + self.provider, self.tracer_provider = provider, tracer_provider + + def instrument(self, manager, config=None): + from traceai_pydantic_ai import PydanticAIInstrumentor + + kwargs = {"attachments": manager} if manager is not None else {} + # skip_dep_check: CI installs pydantic-ai-slim, which the instrumentor's "pydantic-ai" requirement + # does not name. + PydanticAIInstrumentor().instrument(tracer_provider=self.tracer_provider, config=config or TraceConfig(), + skip_dep_check=True, **kwargs) + + def uninstrument(self): + from traceai_pydantic_ai import PydanticAIInstrumentor + + PydanticAIInstrumentor().uninstrument() + + def _agent(self): + from pydantic_ai import Agent + + try: + from pydantic_ai.models.openai import OpenAIChatModel as Model + except ImportError: + from pydantic_ai.models.openai import OpenAIModel as Model + from pydantic_ai.providers.openai import OpenAIProvider + + return Agent(Model("gpt-4o-mini", provider=OpenAIProvider(base_url=self.provider.url + "/v1", + api_key="sk-synthetic"))) + + def _prompt(self): + from pydantic_ai import BinaryContent + + return ["what is this", BinaryContent(data=IMAGE, media_type="image/png")] + + def call(self, mode="sync", body=None): + if mode == "sync": + self._agent().run_sync(self._prompt()) + return + + async def run(): + async with self._agent().run_stream(self._prompt()): + if body is not None: + body() # the caller's own code inside the stream's ``async with`` + asyncio.run(run()) + + +FRAMEWORKS = {cls.name: cls for cls in (LangChain, LlamaIndex, Agno, PydanticAI)} +CALLS = [(name, mode) for name, cls in FRAMEWORKS.items() for mode in cls.modes] + + +@pytest.fixture +def framework(request, provider, spans): + tracer_provider, _ = spans + driver = FRAMEWORKS[request.param](provider, tracer_provider) + state = {} + original = driver.instrument + + def instrument(*args, **kwargs): + original(*args, **kwargs) + state["on"] = True + + driver.instrument = instrument + yield driver + if state.get("on"): + driver.uninstrument() + + +# ------------------------------------------------------------------ one call, one owner +@pytest.mark.parametrize("framework, mode", CALLS, indirect=["framework"]) +def test_framework_and_provider_one_call_uploads_each_part_once(f01, provider, spans, openai_on, framework, mode): + """Was 2 (framework span + provider span); the provider instrumentor now skips the owned call.""" + _, exporter = spans manager = f01.manager() - LangChainInstrumentor().instrument(tracer_provider=tracer_provider, attachments=manager) + framework.instrument(manager) openai_on(manager) - try: - model = ChatOpenAI(model="gpt-4o-mini", api_key="sk-synthetic", base_url=provider.url + "/v1", - max_retries=0) - model.invoke([HumanMessage(content=[{"type": "text", "text": "what"}, - {"type": "image_url", "image_url": {"url": URI}}])]) - finally: - LangChainInstrumentor().uninstrument() - owners = assert_each_span_owns_its_occurrence(f01, manager, provider, exporter) - assert "ChatOpenAI" in [s.name for s in owners] + framework.call(mode) + assert_one_owner(f01, manager, provider, exporter) -# ------------------------------------------------------------------ LlamaIndex + llama-index-llms-openai -def test_llamaindex_openai(f01, provider, spans, openai_on): - pytest.importorskip("llama_index.llms.openai") - from llama_index.core.base.llms.types import ChatMessage, ImageBlock, TextBlock - from llama_index.llms.openai import OpenAI - from traceai_llamaindex import LlamaIndexInstrumentor +@pytest.mark.parametrize("framework", list(FRAMEWORKS), indirect=True) +def test_framework_alone_still_uploads_inside_its_own_mark(f01, provider, spans, framework): + """Control: the adapter's own capture must not be suppressed by the mark it sets.""" + _, exporter = spans + manager = f01.manager() + framework.instrument(manager) + framework.call() + flush(manager) + assert sent_image(provider) + assert f01.payloads == [IMAGE] + (owner,) = owners(exporter) + assert not is_provider_span(owner) - tracer_provider, exporter = spans + +@pytest.mark.parametrize("framework", list(FRAMEWORKS), indirect=True) +def test_framework_attachments_off_marks_nothing(f01, provider, spans, openai_on, framework): + """Attachments off for the framework: no mark, so the provider captures exactly as before.""" + _, exporter = spans manager = f01.manager() - LlamaIndexInstrumentor().instrument(tracer_provider=tracer_provider, attachments=manager) + framework.instrument(None) openai_on(manager) - try: - llm = OpenAI(model="gpt-4o-mini", api_key="sk-synthetic", api_base=provider.url + "/v1", max_retries=0) - llm.chat([ChatMessage(role="user", blocks=[TextBlock(text="what"), ImageBlock(image=IMAGE, - image_mimetype="image/png")])]) - finally: - LlamaIndexInstrumentor().uninstrument() - assert_each_span_owns_its_occurrence(f01, manager, provider, exporter) + framework.call() + flush(manager) + assert sent_image(provider) + assert f01.payloads == [IMAGE] + (owner,) = owners(exporter) + assert is_provider_span(owner) + +# ------------------------------------------------------------------ different calls +@pytest.mark.parametrize("framework", list(FRAMEWORKS), indirect=True) +def test_same_bytes_in_two_framework_calls_are_two_occurrences(f01, provider, spans, openai_on, framework): + _, exporter = spans + manager = f01.manager() + framework.instrument(manager) + openai_on(manager) + framework.call() + framework.call() + flush(manager) + assert sent_image(provider, 2) + assert f01.payloads == [IMAGE, IMAGE] + found = owners(exporter) + assert len(found) == 2 and not any(is_provider_span(s) for s in found) + assert len({r["occurrence_id"] for s in found for r in refs(s)}) == 2 + assert_uris_resolve(exporter) -# ------------------------------------------------------------------ Agno + OpenAIChat -def test_agno_openai(f01, provider, spans, openai_on, monkeypatch): - pytest.importorskip("agno") - pytest.importorskip("openinference.instrumentation.agno") - from agno.agent import Agent - from agno.media import Image - from agno.models.openai import OpenAIChat - from traceai_agno import AgnoInstrumentorWrapper, configure_agno_tracing - monkeypatch.setenv("AGNO_TELEMETRY", "false") - tracer_provider, exporter = spans +@pytest.mark.parametrize("framework", list(FRAMEWORKS), indirect=True) +def test_framework_call_then_direct_provider_call_each_own_one(f01, provider, spans, openai_on, framework): + """The mark ends with the framework call: a later direct SDK call with the same bytes uploads its own.""" + _, exporter = spans manager = f01.manager() - configure_agno_tracing(tracer_provider=tracer_provider, attachments=manager) + framework.instrument(manager) openai_on(manager) - try: - model = OpenAIChat(id="gpt-4o-mini", api_key="sk-synthetic", base_url=provider.url + "/v1", - max_retries=0) - try: - agent = Agent(model=model, telemetry=False) - except TypeError: - agent = Agent(model=model) - agent.run("what is this", images=[Image(content=IMAGE, format="png")]) - finally: - AgnoInstrumentorWrapper().uninstrument() - assert_each_span_owns_its_occurrence(f01, manager, provider, exporter) + framework.call() + direct_openai_call(provider) + flush(manager) + assert f01.payloads == [IMAGE, IMAGE] + found = owners(exporter) + assert sorted(is_provider_span(s) for s in found) == [False, True] + assert_uris_resolve(exporter) -# ------------------------------------------------------------------ Pydantic AI + OpenAI model -def test_pydantic_ai_openai(f01, provider, spans, openai_on): - pytest.importorskip("pydantic_ai") - from pydantic_ai import Agent, BinaryContent - from traceai_pydantic_ai import PydanticAIInstrumentor +@pytest.mark.parametrize("framework", ["langchain", "llamaindex"], indirect=True) +def test_an_abandoned_framework_stream_leaves_no_mark(f01, provider, spans, openai_on, framework): + """A stream suspended mid-call (or dropped) never leaves the mark on the caller's context.""" + _, exporter = spans + manager = f01.manager() + framework.instrument(manager) + openai_on(manager) + stream = framework.open_stream() + direct_openai_call(provider) # while the framework stream is suspended + stream.close() + direct_openai_call(provider) # after it was abandoned + flush(manager) + assert f01.payloads == [IMAGE, IMAGE, IMAGE] + found = owners(exporter) + assert sorted(is_provider_span(s) for s in found) == [False, True, True] - try: - from pydantic_ai.models.openai import OpenAIChatModel as Model - except ImportError: - from pydantic_ai.models.openai import OpenAIModel as Model - from pydantic_ai.providers.openai import OpenAIProvider - tracer_provider, exporter = spans +@pytest.mark.parametrize("framework", ["pydantic_ai"], indirect=True) +def test_the_callers_stream_body_is_not_marked(f01, provider, spans, openai_on, framework): + """Pydantic AI marks entering and exiting the stream, never the caller's ``async with`` body.""" + _, exporter = spans manager = f01.manager() - # skip_dep_check: CI installs pydantic-ai-slim, which the instrumentor's "pydantic-ai" requirement does not name. - PydanticAIInstrumentor().instrument(tracer_provider=tracer_provider, attachments=manager, skip_dep_check=True) + framework.instrument(manager) openai_on(manager) - try: - model = Model("gpt-4o-mini", provider=OpenAIProvider(base_url=provider.url + "/v1", api_key="sk-synthetic")) - Agent(model).run_sync(["what is this", BinaryContent(data=IMAGE, media_type="image/png")]) - finally: - PydanticAIInstrumentor().uninstrument() - assert_each_span_owns_its_occurrence(f01, manager, provider, exporter) + framework.call("stream", body=lambda: direct_openai_call(provider)) + flush(manager) + assert f01.payloads == [IMAGE, IMAGE] + found = owners(exporter) + assert sorted(is_provider_span(s) for s in found) == [False, True] + + +# ------------------------------------------------------------------ the mark carries the framework's deny +@pytest.mark.parametrize("framework", ["langchain", "llamaindex", "pydantic_ai"], indirect=True) +def test_framework_hide_inputs_is_not_bypassed_through_the_provider(f01, provider, spans, openai_on, framework): + """The framework hides its input: it uploads nothing, and the provider beneath it does not either.""" + _, exporter = spans + manager = f01.manager() + framework.instrument(manager, config=TraceConfig(hide_inputs=True)) + openai_on(manager) + framework.call() + flush(manager) + assert sent_image(provider) + assert f01.payloads == [] + assert owners(exporter) == [] def test_provider_alone_two_calls_same_image_each_own_their_occurrence(f01, provider, spans, openai_on): @@ -162,23 +429,19 @@ def test_provider_alone_two_calls_same_image_each_own_their_occurrence(f01, prov Each call is its own trace. Each span uploads, references its own occurrence, and its telemetry URI points at that occurrence (never at the other trace's). """ - import openai - - tracer_provider, exporter = spans + _, exporter = spans manager = f01.manager() openai_on(manager) - client = openai.OpenAI(api_key="sk-synthetic", base_url=provider.url + "/v1", max_retries=0) for _ in range(2): - client.chat.completions.create(model="m", messages=[{"role": "user", "content": [ - {"type": "image_url", "image_url": {"url": URI}}]}]) + direct_openai_call(provider) flush(manager) assert f01.payloads == [IMAGE, IMAGE] - owners = [s for s in exporter.get_finished_spans() if refs(s)] - assert len(owners) == 2 and len({s.context.trace_id for s in owners}) == 2 - assert len({r["occurrence_id"] for s in owners for r in refs(s)}) == 2 + found = owners(exporter) + assert len(found) == 2 and len({s.context.trace_id for s in found}) == 2 + assert len({r["occurrence_id"] for s in found for r in refs(s)}) == 2 # A span may project its URI into several attributes; each must resolve to its OWN occurrence. assert assert_uris_resolve(exporter) >= 2 - for span in owners: + for span in found: (own,) = [r["occurrence_id"] for r in refs(span)] assert any("fi-attachment:" + own in value for value in (span.attributes or {}).values() if isinstance(value, str)), span.name From 1080f58d91e97d965abb2004ed33e4b89a192b9e Mon Sep 17 00:00:00 2001 From: Nikhil Pareek <8347993+nik13@users.noreply.github.com> Date: Thu, 12 Nov 2026 07:28:40 -0800 Subject: [PATCH 14/17] fix(python): treat a stream proxy with both protocols as async when marking it LlamaIndex's async stream proxy also defines send/throw/close, so owned_stream checked it as a sync generator first (CI: llamaindex-astream TypeError). Co-Authored-By: Claude Opus 5.5 --- .../attachments/media_capture.py | 5 +- .../attachments_frameworks/test_owner.py | 46 +++++++++++++++++++ 2 files changed, 49 insertions(+), 2 deletions(-) diff --git a/python/fi_instrumentation/attachments/media_capture.py b/python/fi_instrumentation/attachments/media_capture.py index 58364feb3..be6988837 100644 --- a/python/fi_instrumentation/attachments/media_capture.py +++ b/python/fi_instrumentation/attachments/media_capture.py @@ -776,10 +776,11 @@ def owned_stream(result: Any, owner: Optional[CallOwner]) -> Any: """ if owner is None: return result - if isinstance(result, collections.abc.Generator): - return _OwnedStream(result, owner) + # Async first: a stream proxy (LlamaIndex's) can implement both protocols. if isinstance(result, collections.abc.AsyncGenerator): return _OwnedAsyncStream(result, owner) + if isinstance(result, collections.abc.Generator): + return _OwnedStream(result, owner) return result diff --git a/python/tests/attachments_frameworks/test_owner.py b/python/tests/attachments_frameworks/test_owner.py index 044e31e9c..7821244c0 100644 --- a/python/tests/attachments_frameworks/test_owner.py +++ b/python/tests/attachments_frameworks/test_owner.py @@ -122,3 +122,49 @@ async def run(): asyncio.run(run()) assert seen == [True, True] and caller == [False, False] + + +def test_a_proxy_with_both_protocols_is_treated_as_async(): + """LlamaIndex's async stream proxy also defines send/throw/close: it must still be iterated async.""" + + async def stream(): + yield mc.owned_elsewhere() + + class BothProtocols: + def __init__(self, inner): + self.inner = inner + + def __iter__(self): + return self + + def __next__(self): + raise StopIteration + + def send(self, value): + raise StopIteration + + def throw(self, *args): + raise StopIteration + + def close(self): + pass + + def __aiter__(self): + return self + + def __anext__(self): + return self.inner.__anext__() + + def asend(self, value): + return self.inner.asend(value) + + def athrow(self, *args): + return self.inner.athrow(*args) + + def aclose(self): + return self.inner.aclose() + + async def run(): + return [item async for item in mc.owned_stream(BothProtocols(stream()), mc.CallOwner("fw"))] + + assert asyncio.run(run()) == [True] From 2296837ff9c935996c3e113e2fefc953a208c24b Mon Sep 17 00:00:00 2001 From: Nikhil Pareek <8347993+nik13@users.noreply.github.com> Date: Thu, 12 Nov 2026 08:09:18 -0800 Subject: [PATCH 15/17] fix(python): mark a framework model call only when the adapter observes its media 16311a37 set the owner mark on every framework model call while attachments were enabled. Media the adapter deliberately skips was then captured by nobody: an Agno Image/File(filepath=...) or a LlamaIndex ImageBlock/DocumentBlock(path=...) is inlined as a data URI by the SDK integration, but traceai_openai skipped the marked call. Framework + traceai_openai on one manager (the auto default) went from 1 upload (provider span) to 0. - Agno, LangChain, LlamaIndex and Pydantic AI now mark a call only when the adapter's own parts scan yields at least one input media part (observes_agno_input / observes_langchain_input / observes_llamaindex_messages / PydanticAIMedia.observes_request). Every gate fails open: an error means no mark. - The gate is on OBSERVED parts, not admitted ones: a part the framework hides (hide_inputs, pii_redaction) is still observed, so the mark keeps carrying the framework's deny to the provider (test_framework_hide_inputs_... stays 0). - Regression rows (framework + traceai_openai): Agno filepath image/file and LlamaIndex path image/document -> 1 upload, on the provider span; framework alone -> 0 (negative control); a call mixing a content part and a path part -> 1 upload on the framework span (disclosed residual: the path part is skipped). - Probe rows: the provider call beneath LangChain/LlamaIndex is unmarked for text, provider-file and path-only calls, and marked for an observed image. - media_capture.py: docstring only. Co-Authored-By: Claude Opus 5.5 --- .../attachments/media_capture.py | 3 + .../agno/traceai_agno/_attachments.py | 29 ++- .../traceai_agno/_wrappers/_model_wrapper.py | 19 +- .../traceai_langchain/_attachments.py | 63 ++++-- .../traceai_llamaindex/_attachments.py | 53 ++++- .../traceai_pydantic_ai/_attachments.py | 28 ++- .../_execution_wrappers.py | 14 +- .../attachments_frameworks/test_owner_gate.py | 181 ++++++++++++++++++ .../test_provider_and_framework.py | 101 +++++++++- 9 files changed, 446 insertions(+), 45 deletions(-) create mode 100644 python/tests/attachments_frameworks/test_owner_gate.py diff --git a/python/fi_instrumentation/attachments/media_capture.py b/python/fi_instrumentation/attachments/media_capture.py index be6988837..18db7b6d6 100644 --- a/python/fi_instrumentation/attachments/media_capture.py +++ b/python/fi_instrumentation/attachments/media_capture.py @@ -24,6 +24,9 @@ provider instrumentor beneath it (``traceai_openai``, ``traceai_anthropic``, ...) uploads nothing for that call. Inside its own mark the adapter's ``capture_parts`` still captures. With attachments off nothing is marked. + The adapter marks only a call in which it observes at least one input + media part: a source it never reads (a local path the SDK inlines) is left + to the provider instrumentor. - Across DIFFERENT calls every observing span gets its OWN occurrence: identical bytes in two calls, or at two positions, are two occurrences; byte deduplication is server-side. A ``fi-attachment:`` reference is only diff --git a/python/frameworks/agno/traceai_agno/_attachments.py b/python/frameworks/agno/traceai_agno/_attachments.py index 47544245c..27182b93e 100644 --- a/python/frameworks/agno/traceai_agno/_attachments.py +++ b/python/frameworks/agno/traceai_agno/_attachments.py @@ -16,10 +16,14 @@ TraceConfig the instrumentor runs with (either one hiding wins), the manager's own policy, and the per-call capture policy. -One model call, one owner: while attachments are enabled, the wrapped model call -runs under the owner mark (``media_capture.OWNER_KEY``, a stream only while it -advances), so a provider instrumentor beneath Agno (``traceai_openai``, ...) -uploads nothing for that call; the Agno span holds the occurrence. A DIFFERENT +One model call, one owner: while attachments are enabled, a model call in which +this adapter observes at least one input media part runs under the owner mark +(``media_capture.OWNER_KEY``, a stream only while it advances), so a provider +instrumentor beneath Agno (``traceai_openai``, ...) uploads nothing for that call; +the Agno span holds the occurrence. A call with no observed part is not marked: +Agno inlines a ``filepath`` image or file as a data URI before the SDK call, so the +provider span captures it. Residual: a call that mixes an observed part with a +``filepath`` one is marked, so the provider skips the ``filepath`` part too. A DIFFERENT call with the same bytes gets its own occurrence (byte deduplication is server-side). """ @@ -40,7 +44,7 @@ looks_base64, ) -__all__ = ("capture_agno_input", "capture_agno_output", "agno_attachment_capabilities") +__all__ = ("capture_agno_input", "capture_agno_output", "agno_attachment_capabilities", "observes_agno_input") agno_attachment_capabilities = MappingProxyType({ "schema_version": 1, @@ -144,6 +148,21 @@ def capture_agno_input(manager, span, messages: Any, *, trace_configs: Sequence[ return EMPTY +def observes_agno_input(messages: Any) -> bool: + """True when this adapter observes at least one input media part (owner-mark gate). Never raises. + + A ``filepath`` part is never observed, so a call holding only those is left to the + provider instrumentor. An error means False (no mark): the provider is never + suppressed by mistake. + """ + try: + if not isinstance(messages, (list, tuple)): + return False + return next(_input_parts(messages), None) is not None + except Exception: + return False + + def capture_agno_output(manager, span, response: Any, *, trace_configs: Sequence[Any] = (), capture=None) -> Mapping[str, Occurrence]: """Capture generated media of a non-streaming ``ModelResponse``. Never raises.""" diff --git a/python/frameworks/agno/traceai_agno/_wrappers/_model_wrapper.py b/python/frameworks/agno/traceai_agno/_wrappers/_model_wrapper.py index 9289f40b3..283f3fb7a 100644 --- a/python/frameworks/agno/traceai_agno/_wrappers/_model_wrapper.py +++ b/python/frameworks/agno/traceai_agno/_wrappers/_model_wrapper.py @@ -646,18 +646,23 @@ def _capture_input(self, span: Any, arguments: Mapping[str, Any]) -> None: except Exception: pass - def _owned(self, values: Dict[str, Any]) -> Dict[str, Any]: - """``values`` plus the owner mark while attachments are enabled (one call, one owner). + def _owned(self, values: Dict[str, Any], arguments: Mapping[str, Any]) -> Dict[str, Any]: + """``values`` plus the owner mark when this call's input media is observed here. span_scope attaches it only around the downstream call and each stream advance, so the provider instrumentor beneath Agno skips this call's media. Attachments - off, or a call already owned further out: ``values`` unchanged. + off, a call with no input part this adapter observes (only ``filepath`` media, + which the SDK inlines and the provider captures), or a call already owned + further out: ``values`` unchanged. """ if self._attachments is None: return values try: from fi_instrumentation.attachments.media_capture import OWNER_KEY, call_owner + from traceai_agno._attachments import observes_agno_input + if not observes_agno_input(arguments.get("messages")): + return values owner = call_owner(self._attachments) except Exception: return values @@ -713,7 +718,7 @@ def run( span.set_attribute(LLM_SYSTEM, llm_system) self._capture_input(span, arguments) - with span_scope(span, self._owned(_invocation_context(instance, "invoke", wrapped))): + with span_scope(span, self._owned(_invocation_context(instance, "invoke", wrapped), arguments)): response = wrapped(*args, **kwargs) output_message = _parse_model_output(response) self._capture_output(span, response) @@ -768,7 +773,7 @@ def run_stream( # Token usage will be set after streaming completes based on final response responses = [] - values = self._owned(_invocation_context(instance, "invoke_stream", wrapped)) + values = self._owned(_invocation_context(instance, "invoke_stream", wrapped), arguments) with span_scope(span, values): source = iter(wrapped(*args, **kwargs)) with closing(iterate(source, span, values)) as iterator: @@ -828,7 +833,7 @@ async def arun( span.set_attribute(LLM_SYSTEM, llm_system) self._capture_input(span, arguments) - with span_scope(span, self._owned(_invocation_context(instance, "ainvoke", wrapped))): + with span_scope(span, self._owned(_invocation_context(instance, "ainvoke", wrapped), arguments)): response = await wrapped(*args, **kwargs) output_message = _parse_model_output(response) self._capture_output(span, response) @@ -885,7 +890,7 @@ async def arun_stream( # Token usage will be set after streaming completes based on final response responses = [] - values = self._owned(_invocation_context(instance, "ainvoke_stream", wrapped)) + values = self._owned(_invocation_context(instance, "ainvoke_stream", wrapped), arguments) with span_scope(span, values): source = wrapped(*args, **kwargs) async with aclosing(aiterate(source, span, values)) as iterator: diff --git a/python/frameworks/langchain/traceai_langchain/_attachments.py b/python/frameworks/langchain/traceai_langchain/_attachments.py index 556826040..c78950964 100644 --- a/python/frameworks/langchain/traceai_langchain/_attachments.py +++ b/python/frameworks/langchain/traceai_langchain/_attachments.py @@ -12,13 +12,18 @@ Remote URLs go to the core's ``RemoteUrl`` policy and are never fetched; provider file IDs and anything else are ignored. -One model call, one owner: while attachments are enabled, ``BaseChatModel`` -``generate`` / ``agenerate`` / ``stream`` / ``astream`` run under the owner mark +One model call, one owner: while attachments are enabled, a ``BaseChatModel`` +``generate`` / ``agenerate`` / ``stream`` / ``astream`` call in which this module +observes at least one input media part runs under the owner mark (``media_capture.owned_call``), so a provider instrumentor beneath LangChain (``traceai_openai``, ...) uploads nothing for that call; the LangChain span holds -the occurrence. A DIFFERENT call with the same bytes gets its own occurrence (byte -deduplication is server-side). The mark also carries LangChain's hide flags down: -what the LangChain span may not capture, the provider span does not capture either. +the occurrence. A call with no observed part (no media, or only shapes this module +does not read, such as a provider file ID) is not marked, so the provider span keeps +what only the SDK sees. Residual: a call that mixes an observed part with an +unread one is marked, so the provider skips the unread part too. A DIFFERENT call +with the same bytes gets its own occurrence (byte deduplication is server-side). +The mark also carries LangChain's hide flags down for a marked call: what the +LangChain span may not capture, the provider span does not capture either. This module imports ``fi_instrumentation.attachments``; the tracer imports it only when an enabled attachment manager is passed. @@ -45,8 +50,8 @@ ) from wrapt import FunctionWrapper -__all__ = ("capture_langchain_messages", "langchain_attachment_capabilities", "own_chat_model_calls", - "release_chat_model_calls") +__all__ = ("capture_langchain_messages", "langchain_attachment_capabilities", "observes_langchain_input", + "own_chat_model_calls", "release_chat_model_calls") langchain_attachment_capabilities = MappingProxyType({ "schema_version": 1, @@ -200,9 +205,38 @@ def run_messages(run: Any, direction: str) -> Optional[list]: _NAMESPACE = "langchain" -def _owned(manager): +def _input(name: str, instance: Any, args: Any, kwargs: Any) -> List[Any]: + """The chat messages one owned entry point sends: every batch of generate, or stream's input.""" + if name in ("generate", "agenerate"): + batches = args[0] if args else kwargs.get("messages") + return [message for batch in items(batches) for message in items(batch)] + value = args[0] if args else kwargs.get("input") + if isinstance(value, str): + return [] + # LangChain's own conversion (PromptValue, message sequences, dicts, tuples). + return list(instance._convert_input(value).to_messages()) + + +def observes_langchain_input(name: str, instance: Any, args: Any, kwargs: Any) -> bool: + """True when this module observes at least one input media part of the call (owner-mark gate). + + Never raises; an error means False (no mark), so the provider is never suppressed by mistake. + """ + try: + return next(_parts(_input(name, instance, args, kwargs), "input"), None) is not None + except Exception: + return False + + +def _owner(name: str, manager, instance: Any, args: Any, kwargs: Any): + if not observes_langchain_input(name, instance, args, kwargs): + return None + return call_owner(manager, _NAMESPACE) + + +def _owned(manager, name: str): def wrapper(wrapped, instance, args, kwargs): - owner = call_owner(manager, _NAMESPACE) + owner = _owner(name, manager, instance, args, kwargs) if owner is None: return wrapped(*args, **kwargs) with owner_scope(owner): @@ -212,9 +246,9 @@ def wrapper(wrapped, instance, args, kwargs): return wrapper -def _owned_async(manager): +def _owned_async(manager, name: str): async def wrapper(wrapped, instance, args, kwargs): - owner = call_owner(manager, _NAMESPACE) + owner = _owner(name, manager, instance, args, kwargs) if owner is None: return await wrapped(*args, **kwargs) with owner_scope(owner): @@ -223,7 +257,10 @@ async def wrapper(wrapped, instance, args, kwargs): def own_chat_model_calls(manager) -> List[Tuple[type, str, Any]]: - """Run every chat-model call under the owner mark while ``manager`` can admit. + """Run every chat-model call whose input media is observed here under the owner mark. + + Only while ``manager`` can admit, and only for a call with at least one observed input + part (``observes_langchain_input``). A provider instrumentor beneath LangChain then uploads nothing for that call; the LangChain span owns its media. Returns the patches for ``release_chat_model_calls``. @@ -235,7 +272,7 @@ def own_chat_model_calls(manager) -> List[Tuple[type, str, Any]]: original = BaseChatModel.__dict__.get(name) if original is None: continue - wrapper = _owned_async(manager) if name == "agenerate" else _owned(manager) + wrapper = _owned_async(manager, name) if name == "agenerate" else _owned(manager, name) setattr(BaseChatModel, name, FunctionWrapper(original, wrapper)) patches.append((BaseChatModel, name, original)) return patches diff --git a/python/frameworks/llama_index/traceai_llamaindex/_attachments.py b/python/frameworks/llama_index/traceai_llamaindex/_attachments.py index a0958373a..421906f3a 100644 --- a/python/frameworks/llama_index/traceai_llamaindex/_attachments.py +++ b/python/frameworks/llama_index/traceai_llamaindex/_attachments.py @@ -11,11 +11,16 @@ An ``IOBase`` source is never read, a ``path`` is never opened, and a ``url`` goes to the core's ``RemoteUrl`` policy without being fetched. -One model call, one owner: while attachments are enabled, every LLM's ``chat`` / -``achat`` / ``stream_chat`` / ``astream_chat`` (the calls whose messages are observed -here) runs under the owner mark (``media_capture.owned_call``; a stream only while it +One model call, one owner: while attachments are enabled, an LLM ``chat`` / +``achat`` / ``stream_chat`` / ``astream_chat`` call (the calls whose messages are +observed here) in which this module observes at least one input media part runs +under the owner mark (``media_capture.owned_call``; a stream only while it advances), so a provider instrumentor beneath LlamaIndex (``traceai_openai``, ...) -uploads nothing for that call; the LlamaIndex span holds the occurrence. Completion +uploads nothing for that call; the LlamaIndex span holds the occurrence. A call +with no observed part is not marked: the OpenAI integration inlines a ``path`` +image or document as a data URI before the SDK call, so the provider span captures +it. Residual: a call that mixes an observed part with a ``path`` (or ``IOBase``) +one is marked, so the provider skips that part too. Completion calls are not observed here and are not marked. A DIFFERENT call with the same bytes gets its own occurrence (byte deduplication is server-side). @@ -44,7 +49,8 @@ ) from wrapt import FunctionWrapper -__all__ = ("ChatOwnership", "capture_llamaindex_messages", "llamaindex_attachment_capabilities") +__all__ = ("ChatOwnership", "capture_llamaindex_messages", "llamaindex_attachment_capabilities", + "observes_llamaindex_messages") llamaindex_attachment_capabilities = MappingProxyType({ "schema_version": 1, @@ -120,6 +126,21 @@ def _parts(messages: Sequence[Any], direction: str) -> Iterator[MediaPart]: # IOBase sources and paths: never read or opened. +def observes_llamaindex_messages(messages: Any) -> bool: + """True when this module observes at least one input media part (owner-mark gate). Never raises. + + ``path`` and ``IOBase`` sources are never observed, so a call holding only those is + left to the provider instrumentor. An error means False (no mark): the provider is + never suppressed by mistake. + """ + try: + if not isinstance(messages, (list, tuple)): + return False + return next(_parts(messages, "input"), None) is not None + except Exception: + return False + + def capture_llamaindex_messages(manager, span, messages: Any, *, direction: str, trace_config=None, capture=None) -> Mapping[str, Occurrence]: """Capture typed block media of chat messages; returns ``"{dir}/{msg}/{part}"`` positions. @@ -145,7 +166,10 @@ def capture_llamaindex_messages(manager, span, messages: Any, *, direction: str, class ChatOwnership: - """Run every LLM chat call under the owner mark while ``manager`` can admit. + """Run every LLM chat call whose input media is observed here under the owner mark. + + Only while ``manager`` can admit, and only for a call with at least one observed + input part (``observes_llamaindex_messages``). Patched like ``StreamInstrumentation``: every loaded ``BaseLLM`` / ``MultiModalLLM`` subclass that defines a chat method, plus an ``__init__`` hook for providers imported @@ -199,11 +223,20 @@ def _initialize(self, wrapped: Any, instance: Any, args: Any, kwargs: Any) -> An self._patch_class(type(instance)) return result - def _owner(self): - return call_owner(self._manager, _NAMESPACE) if self._enabled else None + def _owner(self, args: Any, kwargs: Any): + """The mark for a call whose input media this module observes, else None (no mark).""" + if not self._enabled: + return None + try: + messages = args[0] if args else kwargs.get("messages") + except Exception: + return None + if not observes_llamaindex_messages(messages): + return None + return call_owner(self._manager, _NAMESPACE) def _call(self, wrapped: Any, instance: Any, args: Any, kwargs: Any) -> Any: - owner = self._owner() + owner = self._owner(args, kwargs) if owner is None: return wrapped(*args, **kwargs) with owner_scope(owner): @@ -211,7 +244,7 @@ def _call(self, wrapped: Any, instance: Any, args: Any, kwargs: Any) -> Any: return owned_stream(result, owner) async def _call_async(self, wrapped: Any, instance: Any, args: Any, kwargs: Any) -> Any: - owner = self._owner() + owner = self._owner(args, kwargs) if owner is None: return await wrapped(*args, **kwargs) with owner_scope(owner): diff --git a/python/frameworks/pydantic-ai/traceai_pydantic_ai/_attachments.py b/python/frameworks/pydantic-ai/traceai_pydantic_ai/_attachments.py index fcb44e88c..53c461538 100644 --- a/python/frameworks/pydantic-ai/traceai_pydantic_ai/_attachments.py +++ b/python/frameworks/pydantic-ai/traceai_pydantic_ai/_attachments.py @@ -17,11 +17,15 @@ declared ``media_type``. Videos, uploaded provider files and unknown content objects are not captured, and nothing is serialized to look for media. -One model call, one owner: while attachments are enabled, the model request runs -under the owner mark (``media_capture.owned_call``; a streamed request only while -the stream is entered and exited, never the caller's ``async with`` body), so a -provider instrumentor beneath Pydantic AI (``traceai_openai``, ...) uploads nothing -for that call; the model request span holds the occurrence. DIFFERENT calls each +One model call, one owner: while attachments are enabled, a model request in which +this adapter observes at least one input media part runs under the owner mark +(``media_capture.owned_call``; a streamed request only while the stream is entered +and exited, never the caller's ``async with`` body), so a provider instrumentor +beneath Pydantic AI (``traceai_openai``, ...) uploads nothing for that call; the +model request span holds the occurrence. A request with no observed part is not +marked, so the provider span keeps what only the SDK sees. Residual: a request that +mixes an observed part with one this adapter never reads (an uploaded provider +file, ...) is marked, so the provider skips that part too. DIFFERENT calls each get their own occurrence (a tool span's result and the next request that re-sends it are two observations; byte deduplication is server-side). """ @@ -157,6 +161,20 @@ def request(self, span, messages: Any) -> Mapping[str, Occurrence]: return EMPTY return self._capture(span, _request_parts(messages), "input") + def observes_request(self, messages: Any) -> bool: + """True when this adapter observes at least one input media part of the request. + + Only then does the request run under the owner mark. A part this adapter never reads + (a local path, a provider file, ...) is left to the provider instrumentor. Never raises; + an error means False (no mark), so the provider is never suppressed by mistake. + """ + try: + if not isinstance(messages, (list, tuple)): + return False + return next(_request_parts(messages), None) is not None + except Exception: + return False + def response(self, span, response: Any) -> Mapping[str, Occurrence]: # Historical Model.request returns (ModelResponse, Usage). response = response[0] if isinstance(response, tuple) and response else response diff --git a/python/frameworks/pydantic-ai/traceai_pydantic_ai/_execution_wrappers.py b/python/frameworks/pydantic-ai/traceai_pydantic_ai/_execution_wrappers.py index f368b4498..501a54d30 100644 --- a/python/frameworks/pydantic-ai/traceai_pydantic_ai/_execution_wrappers.py +++ b/python/frameworks/pydantic-ai/traceai_pydantic_ai/_execution_wrappers.py @@ -129,11 +129,17 @@ async def __aexit__(self, exc_type, exc_value, traceback): return await self.manager.__aexit__(exc_type, exc_value, traceback) -def _call_owner(media): - """The owner mark for one model request while attachments are enabled, else None.""" +def _call_owner(media, messages): + """The owner mark for one model request whose input media this adapter observes, else None. + + None while attachments are off, and None for a request with no part this adapter + observes, so the provider instrumentor beneath still captures what only it can see. + """ if media is None: return None try: + if not media.observes_request(messages): + return None from fi_instrumentation.attachments.media_capture import call_owner return call_owner(media.manager) @@ -177,7 +183,7 @@ async def stream_wrapper(self, *args, **kwargs): with _model_request_span(self, tracer, current=False) as span: # Input only: streamed output is not captured. _capture(media, "request", span, _request_messages(args, kwargs)) - owner = _call_owner(media) + owner = _call_owner(media, _request_messages(args, kwargs)) with _owner_scope(owner): manager = original(self, *args, **kwargs) async with _ModelStreamContext(manager, self, span, owner) as response: @@ -190,7 +196,7 @@ async def wrapper(self, *args, **kwargs): with _model_request_span(self, tracer) as span: # Input media is captured before the provider call; this span owns its own occurrences. _capture(media, "request", span, _request_messages(args, kwargs)) - with _owner_scope(_call_owner(media)): + with _owner_scope(_call_owner(media, _request_messages(args, kwargs))): response = await original(self, *args, **kwargs) _record_model_usage(span, response) # Only a successful response: an error never reaches this line. diff --git a/python/tests/attachments_frameworks/test_owner_gate.py b/python/tests/attachments_frameworks/test_owner_gate.py new file mode 100644 index 000000000..466e678ad --- /dev/null +++ b/python/tests/attachments_frameworks/test_owner_gate.py @@ -0,0 +1,181 @@ +"""The owner mark is set only for a call in which the adapter observes at least one input media part. + +A call with no observed part (text only, an Agno ``filepath``, a LlamaIndex ``path``, a provider +file ID) runs unmarked, so a provider instrumentor beneath the framework still captures what only +the SDK sees. Every gate fails open: an error means no mark, never a suppressed provider. +""" + +import pytest + +from conftest import b64, png +from fi_instrumentation import TraceConfig +from fi_instrumentation.attachments import media_capture as mc + +IMAGE = png(seed=41) +URI = "data:image/png;base64," + b64(IMAGE) + + +# ------------------------------------------------------------------ LangChain +@pytest.fixture +def langchain_probe(spans): + pytest.importorskip("langchain_core") + pytest.importorskip("traceai_langchain") + from langchain_core.language_models.chat_models import BaseChatModel + from langchain_core.messages import AIMessage + from langchain_core.outputs import ChatGeneration, ChatResult + from traceai_langchain import LangChainInstrumentor + + seen = [] + + class Probe(BaseChatModel): + """Records whether the provider call beneath LangChain runs under the owner mark.""" + + @property + def _llm_type(self) -> str: + return "owner-probe" + + def _generate(self, messages, stop=None, run_manager=None, **kwargs): + seen.append(mc.owned_elsewhere()) + return ChatResult(generations=[ChatGeneration(message=AIMessage(content="ok"))]) + + tracer_provider, _ = spans + state = {} + + def install(manager): + LangChainInstrumentor().instrument(tracer_provider=tracer_provider, attachments=manager, + config=TraceConfig()) + state["on"] = True + return Probe(), seen + + yield install + if state.get("on"): + LangChainInstrumentor().uninstrument() + + +def test_langchain_marks_only_a_call_with_an_observed_part(f01, langchain_probe): + from langchain_core.messages import HumanMessage + + model, seen = langchain_probe(f01.manager()) + image = HumanMessage(content=[{"type": "text", "text": "what"}, {"type": "image_url", "image_url": {"url": URI}}]) + provider_file = HumanMessage(content=[{"type": "file", "file": {"file_id": "file-synthetic"}}]) + model.invoke("text only") + model.invoke([HumanMessage(content="text only")]) + model.invoke([provider_file]) + model.invoke([image]) + for _ in model.stream("text only"): + pass + for _ in model.stream([image]): + pass + assert seen == [False, False, False, True, False, True] + + +def test_langchain_gate_fails_open(): + pytest.importorskip("traceai_langchain") + from traceai_langchain._attachments import observes_langchain_input + + class Broken: + def _convert_input(self, value): + raise RuntimeError("cannot convert") + + assert observes_langchain_input("stream", Broken(), ([{"role": "user"}],), {}) is False + assert observes_langchain_input("generate", None, (object(),), {}) is False + assert observes_langchain_input("generate", None, (), {}) is False + + +# ------------------------------------------------------------------ LlamaIndex +@pytest.fixture +def llamaindex_probe(spans): + pytest.importorskip("llama_index.core") + pytest.importorskip("traceai_llamaindex") + from llama_index.core.base.llms.types import ChatMessage, ChatResponse, CompletionResponse, LLMMetadata + from llama_index.core.llms.callbacks import llm_chat_callback, llm_completion_callback + from llama_index.core.llms.custom import CustomLLM + from traceai_llamaindex import LlamaIndexInstrumentor + + seen = [] + + class Probe(CustomLLM): + @property + def metadata(self) -> LLMMetadata: + return LLMMetadata(model_name="owner-probe", is_chat_model=True) + + @llm_chat_callback() + def chat(self, messages, **kwargs): + seen.append(mc.owned_elsewhere()) + return ChatResponse(message=ChatMessage(role="assistant", content="ok")) + + @llm_completion_callback() + def complete(self, prompt, formatted=False, **kwargs): + return CompletionResponse(text="ok") + + @llm_completion_callback() + def stream_complete(self, prompt, formatted=False, **kwargs): + yield CompletionResponse(text="ok", delta="ok") + + tracer_provider, _ = spans + state = {} + + def install(manager): + LlamaIndexInstrumentor().instrument(tracer_provider=tracer_provider, attachments=manager, + config=TraceConfig()) + state["on"] = True + return Probe(), seen + + yield install + if state.get("on"): + LlamaIndexInstrumentor().uninstrument() + + +def test_llamaindex_marks_only_a_call_with_an_observed_part(f01, llamaindex_probe, tmp_path): + from llama_index.core.base.llms.types import ChatMessage, ImageBlock, TextBlock + + llm, seen = llamaindex_probe(f01.manager()) + path = tmp_path / "source.png" + path.write_bytes(IMAGE) + llm.chat([ChatMessage(role="user", content="text only")]) + llm.chat([ChatMessage(role="user", blocks=[TextBlock(text="what"), + ImageBlock(path=path, image_mimetype="image/png")])]) + llm.chat([ChatMessage(role="user", blocks=[ImageBlock(image=IMAGE, image_mimetype="image/png")])]) + llm.chat(messages=[ChatMessage(role="user", blocks=[ImageBlock(image=IMAGE, image_mimetype="image/png")])]) + assert seen == [False, False, True, True] + + +def test_llamaindex_gate_fails_open(): + pytest.importorskip("traceai_llamaindex") + from traceai_llamaindex._attachments import observes_llamaindex_messages + + assert observes_llamaindex_messages(None) is False + assert observes_llamaindex_messages("text") is False + assert observes_llamaindex_messages([object()]) is False + + +# ------------------------------------------------------------------ Agno +def test_agno_gate_skips_a_filepath_only_call(tmp_path): + pytest.importorskip("agno") + pytest.importorskip("traceai_agno") + from agno.media import Image + from agno.models.message import Message + from traceai_agno._attachments import observes_agno_input + + path = tmp_path / "source.png" + path.write_bytes(IMAGE) + assert observes_agno_input([Message(role="user", content="text only")]) is False + assert observes_agno_input([Message(role="user", content="x", images=[Image(filepath=str(path))])]) is False + assert observes_agno_input([Message(role="user", content="x", images=[Image(content=IMAGE, format="png")])]) + assert observes_agno_input(None) is False and observes_agno_input([object()]) is False + + +# ------------------------------------------------------------------ Pydantic AI +def test_pydantic_ai_gate_marks_only_a_request_with_an_observed_part(): + pytest.importorskip("pydantic_ai") + pytest.importorskip("traceai_pydantic_ai") + from pydantic_ai import BinaryContent + from pydantic_ai.messages import ModelRequest, UserPromptPart + from traceai_pydantic_ai._attachments import PydanticAIMedia + + media = PydanticAIMedia(None, None) + text = ModelRequest(parts=[UserPromptPart(content="text only")]) + image = ModelRequest(parts=[UserPromptPart(content=["what", BinaryContent(data=IMAGE, media_type="image/png")])]) + assert media.observes_request([text]) is False + assert media.observes_request([text, image]) is True + assert media.observes_request(None) is False and media.observes_request([object()]) is False diff --git a/python/tests/attachments_frameworks/test_provider_and_framework.py b/python/tests/attachments_frameworks/test_provider_and_framework.py index 1ff9bf755..a50f1935a 100644 --- a/python/tests/attachments_frameworks/test_provider_and_framework.py +++ b/python/tests/attachments_frameworks/test_provider_and_framework.py @@ -6,6 +6,10 @@ part is uploaded exactly once, under the framework span. DIFFERENT calls still each own their occurrence, even with identical bytes (byte deduplication is server-side). With attachments off for the framework nothing is marked and the provider captures as before. + +The mark is set only for a call in which the adapter observes at least one input media part. +Media the adapter never reads (an Agno ``filepath``, a LlamaIndex ``path``) is inlined by the +SDK integration as a data URI, so ``traceai_openai`` must still capture it on the provider span. """ import asyncio @@ -16,12 +20,13 @@ pytest.importorskip("openai") pytest.importorskip("traceai_openai") -from conftest import assert_uris_resolve, b64, flush, png, refs # noqa: E402 +from conftest import PDF, assert_uris_resolve, b64, flush, png, refs # noqa: E402 from fi_instrumentation import TraceConfig # noqa: E402 from traceai_openai import OpenAIInstrumentor # noqa: E402 IMAGE = png(seed=31) URI = "data:image/png;base64," + b64(IMAGE) +PATH_IMAGE = png(seed=32) # only ever read from disk by the framework's SDK integration OPENAI_SCOPE = "traceai_openai" @@ -186,6 +191,17 @@ def open_stream(self): next(stream) return stream + def path_call(self, path, kind, with_content=False): + """One chat call with a ``path`` block (never read by the adapter), optionally after a content block.""" + from llama_index.core.base.llms.types import ChatMessage, DocumentBlock, ImageBlock, TextBlock + + blocks = [TextBlock(text="what")] + if with_content: + blocks.append(ImageBlock(image=IMAGE, image_mimetype="image/png")) + blocks.append(ImageBlock(path=path, image_mimetype="image/png") if kind == "image" + else DocumentBlock(path=path, document_mimetype="application/pdf")) + self._llm().chat([ChatMessage(role="user", blocks=blocks)]) + class Agno: name = "agno" @@ -229,6 +245,17 @@ def call(self, mode="sync"): for _ in self._agent().run("what is this", images=images, stream=True): pass + def path_call(self, path, kind, with_content=False): + """One model call with a ``filepath`` source (never read by the adapter), optionally after content.""" + from agno.media import File, Image + + if kind == "image": + images = ([Image(content=IMAGE, format="png")] if with_content else []) + [Image(filepath=str(path))] + self._agent().run("what is this", images=images) + else: + images = [Image(content=IMAGE, format="png")] if with_content else None + self._agent().run("what is this", images=images, files=[File(filepath=str(path))]) + class PydanticAI: name = "pydantic_ai" @@ -408,6 +435,78 @@ def test_the_callers_stream_body_is_not_marked(f01, provider, spans, openai_on, assert sorted(is_provider_span(s) for s in found) == [False, True] +# ------------------------------------------------------------------ media only the SDK reads +PATH_SOURCES = [("agno", "image"), ("agno", "file"), ("llamaindex", "image"), ("llamaindex", "document")] + + +def sent(provider, data): + return any(path.endswith("/chat/completions") and b64(data) in json.dumps(body) + for path, body, raw in provider.requests) + + +def _path(tmp_path, kind): + data = PATH_IMAGE if kind == "image" else PDF + path = tmp_path / ("source.png" if kind == "image" else "source.pdf") + path.write_bytes(data) + return path, data + + +@pytest.mark.parametrize("framework, kind", PATH_SOURCES, indirect=["framework"]) +def test_a_source_only_the_sdk_reads_is_still_captured_by_the_provider(f01, provider, spans, openai_on, + framework, kind, tmp_path): + """The adapter never opens a path, the SDK integration inlines it: no mark, ONE upload, on the provider span. + + With the mark set on every call (16311a37..1080f58d) this was 0: nobody captured it. + """ + _, exporter = spans + manager = f01.manager() + framework.instrument(manager) + openai_on(manager) + path, data = _path(tmp_path, kind) + framework.path_call(path, kind) + flush(manager) + assert sent(provider, data), "the SDK integration did not inline the path source" + assert f01.payloads == [data], len(f01.payloads) + (owner,) = owners(exporter) + assert is_provider_span(owner), owner.name + (ref,) = refs(owner) + (upload,) = f01.uploads + assert upload["occurrence_id"] == ref["occurrence_id"] + assert upload["association"]["parent"]["span_id"] == format(owner.context.span_id, "016x") + assert_uris_resolve(exporter) + + +@pytest.mark.parametrize("framework, kind", PATH_SOURCES, indirect=["framework"]) +def test_the_framework_alone_never_reads_a_path_source(f01, provider, spans, framework, kind, tmp_path): + """Negative control for the row above: without traceai_openai the path source is uploaded by nobody.""" + _, exporter = spans + manager = f01.manager() + framework.instrument(manager) + path, data = _path(tmp_path, kind) + framework.path_call(path, kind) + flush(manager) + assert sent(provider, data) + assert f01.payloads == [] and owners(exporter) == [] + + +@pytest.mark.parametrize("framework", ["agno", "llamaindex"], indirect=True) +def test_residual_a_call_mixing_an_observed_part_and_a_path_part_is_marked(f01, provider, spans, openai_on, + framework, tmp_path): + """Disclosed residual: the adapter observes the content part, so the WHOLE call is marked and the + provider skips the path part too. One upload (the content part), on the framework span.""" + _, exporter = spans + manager = f01.manager() + framework.instrument(manager) + openai_on(manager) + path, data = _path(tmp_path, "image") + framework.path_call(path, "image", with_content=True) + flush(manager) + assert sent(provider, IMAGE) and sent(provider, data) + assert f01.payloads == [IMAGE], len(f01.payloads) + (owner,) = owners(exporter) + assert not is_provider_span(owner), owner.name + + # ------------------------------------------------------------------ the mark carries the framework's deny @pytest.mark.parametrize("framework", ["langchain", "llamaindex", "pydantic_ai"], indirect=True) def test_framework_hide_inputs_is_not_bypassed_through_the_provider(f01, provider, spans, openai_on, framework): From a2a007f2d151714df7a869c058c30e200804f20c Mon Sep 17 00:00:00 2001 From: Nikhil Pareek <8347993+nik13@users.noreply.github.com> Date: Thu, 12 Nov 2026 14:10:21 -0800 Subject: [PATCH 16/17] test(python): one owner per call with traceai_anthropic under LangChain LangChain ChatAnthropic + traceai_anthropic on one manager (sync and stream): one upload, on the framework span; with framework attachments off the Anthropic span owns it. Latest tier only; CI installs traceai_anthropic, langchain-anthropic 1.7.10 and anthropic 2.8.0 (floor pins anthropic 0.41.0). Co-Authored-By: Claude Opus 5.5 --- .../attachments-python-frameworks-tests.yml | 18 ++- .../test_anthropic_under_framework.py | 147 ++++++++++++++++++ 2 files changed, 158 insertions(+), 7 deletions(-) create mode 100644 python/tests/attachments_frameworks/test_anthropic_under_framework.py diff --git a/.github/workflows/attachments-python-frameworks-tests.yml b/.github/workflows/attachments-python-frameworks-tests.yml index 4c6152ea0..86b719034 100644 --- a/.github/workflows/attachments-python-frameworks-tests.yml +++ b/.github/workflows/attachments-python-frameworks-tests.yml @@ -2,8 +2,8 @@ name: Python framework attachments # Real LangChain, LlamaIndex, Agno and Pydantic AI (declared floor and latest) through # the Python framework instrumentors to a TLS loopback F01 attachment server, with and -# without traceai_openai on the same manager. Outbound network is disabled while the -# tests run. +# without traceai_openai (and traceai_anthropic under LangChain) on the same manager. +# Outbound network is disabled while the tests run. on: pull_request: @@ -14,6 +14,7 @@ on: - 'python/frameworks/agno/**' - 'python/frameworks/pydantic-ai/**' - 'python/frameworks/openai/**' + - 'python/frameworks/anthropic/**' - 'python/tests/attachments_frameworks/**' - 'python/tests/attachments_providers/conftest.py' - '.github/workflows/attachments-python-frameworks-tests.yml' @@ -26,6 +27,7 @@ on: - 'python/frameworks/agno/**' - 'python/frameworks/pydantic-ai/**' - 'python/frameworks/openai/**' + - 'python/frameworks/anthropic/**' - 'python/tests/attachments_frameworks/**' - 'python/tests/attachments_providers/conftest.py' - '.github/workflows/attachments-python-frameworks-tests.yml' @@ -49,15 +51,16 @@ jobs: python: '3.10' sdks: >- langchain-core==0.2.43 llama-index-core==0.12.41 agno==2.7.2 - openinference-instrumentation-agno==1.1.0 pydantic-ai-slim[openai]==0.1.0 openai==1.74.0 + openinference-instrumentation-agno==1.1.0 pydantic-ai-slim[openai]==0.1.0 openai==1.74.0 anthropic==0.41.0 fastapi 'httpx<0.28' 'opentelemetry-api>=1.28,<1.39' 'opentelemetry-sdk>=1.28,<1.39' - # Latest releases, including the framework OpenAI integrations for the provider-and-framework tests. + # Latest releases, including the framework OpenAI and Anthropic integrations for the + # provider-and-framework tests. - name: latest python: '3.12' sdks: >- langchain-core==1.8.0 langchain-openai==1.7.4 llama-index-core==0.14.25 llama-index-llms-openai==0.8.4 agno==3.1.12 openinference-instrumentation-agno==1.1.0 - pydantic-ai-slim[openai]==2.67.0 fastapi + pydantic-ai-slim[openai]==2.67.0 langchain-anthropic==1.7.10 anthropic==2.8.0 fastapi steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: @@ -71,10 +74,11 @@ jobs: run: | python -m venv .venv # shellcheck disable=SC2086 - eval .venv/bin/pip install -e python -e python/frameworks/openai -e python/frameworks/langchain \ + eval .venv/bin/pip install -e python -e python/frameworks/openai -e python/frameworks/anthropic \ + -e python/frameworks/langchain \ -e python/frameworks/llama_index -e python/frameworks/agno -e python/frameworks/pydantic-ai \ opentelemetry-sdk "'pytest>=7,<9'" $SDKS - .venv/bin/pip freeze | grep -iE '^(langchain|llama-index-core|llama-index-llms|agno|pydantic-ai|openai|openinference)' || true + .venv/bin/pip freeze | grep -iE '^(langchain|llama-index-core|llama-index-llms|agno|pydantic-ai|openai|anthropic|openinference)' || true - name: Run with outbound network disabled run: | sudo unshare --net -- bash -e -c ' diff --git a/python/tests/attachments_frameworks/test_anthropic_under_framework.py b/python/tests/attachments_frameworks/test_anthropic_under_framework.py new file mode 100644 index 000000000..4a01ea922 --- /dev/null +++ b/python/tests/attachments_frameworks/test_anthropic_under_framework.py @@ -0,0 +1,147 @@ +"""Framework + Anthropic instrumentor on ONE manager: one model call, one owner. + +The owner mark is provider-neutral: ``traceai_anthropic`` makes the same core check as +``traceai_openai``, so a framework call that captured its media is not uploaded a second time +by the Anthropic provider span. Driven through LangChain's ``ChatAnthropic`` against the real +Anthropic SDK (loopback provider). Latest-only: skipped where ``langchain-anthropic`` is not +installed. + +Not covered here: +- Pydantic AI's Anthropic model calls ``client.beta.messages``, which ``traceai_anthropic`` does + not instrument, so that pair has no provider span to double-count; +- llama-index-llms-anthropic 0.12.3 fails before any request against anthropic 2.8.0 + (``Cannot instantiate typing.Union`` building ``MessageParam``), with or without traceAI. +""" + +import json + +import pytest + +pytest.importorskip("anthropic") +pytest.importorskip("traceai_anthropic") + +from conftest import assert_uris_resolve, b64, flush, png, refs # noqa: E402 +from fi_instrumentation import TraceConfig # noqa: E402 + +IMAGE = png(seed=41) +URI = "data:image/png;base64," + b64(IMAGE) +ANTHROPIC_SCOPE = "traceai_anthropic" +MODEL = "claude-haiku-4-5" # the loopback answers any model id + + +@pytest.fixture +def anthropic_on(spans): + from traceai_anthropic import AnthropicInstrumentor + + tracer_provider, _ = spans + state = {} + + def install(manager): + AnthropicInstrumentor().instrument(tracer_provider=tracer_provider, attachments=manager, config=TraceConfig()) + state["on"] = True + + yield install + if state.get("on"): + AnthropicInstrumentor().uninstrument() + + +class LangChain: + def __init__(self, provider, tracer_provider): + pytest.importorskip("traceai_langchain") + pytest.importorskip("langchain_anthropic") + self.provider, self.tracer_provider = provider, tracer_provider + + def instrument(self, manager): + from traceai_langchain import LangChainInstrumentor + + kwargs = {"attachments": manager} if manager is not None else {} + LangChainInstrumentor().instrument(tracer_provider=self.tracer_provider, config=TraceConfig(), **kwargs) + + def uninstrument(self): + from traceai_langchain import LangChainInstrumentor + + LangChainInstrumentor().uninstrument() + + def call(self, mode): + from langchain_anthropic import ChatAnthropic + from langchain_core.messages import HumanMessage + + model = ChatAnthropic(model=MODEL, api_key="sk-ant-synthetic", base_url=self.provider.url, max_retries=0, + max_tokens=16) + messages = [HumanMessage(content=[{"type": "text", "text": "what"}, + {"type": "image_url", "image_url": {"url": URI}}])] + if mode == "sync": + model.invoke(messages) + else: + for _ in model.stream(messages): + pass + + +FRAMEWORKS = {"langchain": LangChain} + + +@pytest.fixture +def framework(request, provider, spans): + tracer_provider, _ = spans + driver = FRAMEWORKS[request.param](provider, tracer_provider) + state = {} + original = driver.instrument + + def instrument(manager): + original(manager) + state["on"] = True + + driver.instrument = instrument + yield driver + if state.get("on"): + driver.uninstrument() + + +def sent_image(provider): + return any(path.endswith("/messages") and b64(IMAGE) in json.dumps(body) for path, body, raw in provider.requests) + + +def owners(exporter): + return [s for s in exporter.get_finished_spans() if refs(s)] + + +def is_provider_span(span): + return getattr(span.instrumentation_scope, "name", "") == ANTHROPIC_SCOPE + + +@pytest.mark.parametrize("framework", list(FRAMEWORKS), indirect=True) +@pytest.mark.parametrize("mode", ["sync", "stream"]) +def test_framework_and_anthropic_one_call_uploads_each_part_once(f01, provider, spans, anthropic_on, framework, mode): + """Without the mark this is 2 (framework span + Anthropic span).""" + _, exporter = spans + manager = f01.manager() + framework.instrument(manager) + anthropic_on(manager) + framework.call(mode) + flush(manager) + assert sent_image(provider), "the framework did not send the image to the provider" + assert any(is_provider_span(s) for s in exporter.get_finished_spans()), "no Anthropic span: not a 2-layer call" + assert f01.payloads == [IMAGE], len(f01.payloads) + (owner,) = owners(exporter) + assert not is_provider_span(owner), owner.name + (ref,) = refs(owner) + (upload,) = f01.uploads + assert upload["occurrence_id"] == ref["occurrence_id"] + assert upload["association"]["parent"]["span_id"] == format(owner.context.span_id, "016x") + assert_uris_resolve(exporter) + + +@pytest.mark.parametrize("framework", list(FRAMEWORKS), indirect=True) +def test_framework_attachments_off_leaves_the_anthropic_span_the_owner(f01, provider, spans, anthropic_on, framework): + """Control: no framework manager, no mark, so the Anthropic provider span captures the part.""" + _, exporter = spans + manager = f01.manager() + framework.instrument(None) + anthropic_on(manager) + framework.call("sync") + flush(manager) + assert sent_image(provider) + assert f01.payloads == [IMAGE], len(f01.payloads) + (owner,) = owners(exporter) + assert is_provider_span(owner), owner.name + assert_uris_resolve(exporter) From 66c4371098604f59d89beb4063a303867e31d141 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek <8347993+nik13@users.noreply.github.com> Date: Thu, 12 Nov 2026 15:48:31 -0800 Subject: [PATCH 17/17] test(python): pin each provider-side owner check and the Pydantic AI gate VM mutation run on a2a007f2 left two survivors: - dropping the owner check from Observation alone (local_capture_policy still sets requested=False, so no row changed); each check now has its own assertion; - Pydantic AI _call_owner marking a request it does not observe; now asserted directly (text-only request -> no mark). Co-Authored-By: Claude Opus 5.5 --- .../tests/attachments_frameworks/test_owner.py | 13 +++++++++++++ .../attachments_frameworks/test_owner_gate.py | 17 +++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/python/tests/attachments_frameworks/test_owner.py b/python/tests/attachments_frameworks/test_owner.py index 7821244c0..d1d763e00 100644 --- a/python/tests/attachments_frameworks/test_owner.py +++ b/python/tests/attachments_frameworks/test_owner.py @@ -36,6 +36,19 @@ def test_call_owner_is_gated_on_an_enabled_manager_and_an_unowned_call(f01): otel_context.detach(token) +def test_each_provider_side_check_honours_the_mark(f01, spans): + """A provider instrumentor skips an owned call twice over: its Observation is unusable AND its + local capture policy is unrequested. Each check is pinned on its own (one alone would hide the other).""" + tracer_provider, _ = spans + manager = f01.manager() + with tracer_provider.get_tracer("t").start_as_current_span("provider") as span: + assert mc.Observation(manager, span, "openai").usable + assert mc.local_capture_policy(direction="input").requested + with mc.owner_scope(mc.CallOwner("fw")): + assert not mc.Observation(manager, span, "openai").usable + assert mc.local_capture_policy(direction="input").requested is False + + def test_owned_call_marks_only_the_enclosed_block(): assert not mc.owned_elsewhere() with mc.owner_scope(mc.CallOwner("fw")): diff --git a/python/tests/attachments_frameworks/test_owner_gate.py b/python/tests/attachments_frameworks/test_owner_gate.py index 466e678ad..96d6b6682 100644 --- a/python/tests/attachments_frameworks/test_owner_gate.py +++ b/python/tests/attachments_frameworks/test_owner_gate.py @@ -179,3 +179,20 @@ def test_pydantic_ai_gate_marks_only_a_request_with_an_observed_part(): assert media.observes_request([text]) is False assert media.observes_request([text, image]) is True assert media.observes_request(None) is False and media.observes_request([object()]) is False + + +def test_pydantic_ai_call_owner_follows_the_gate(f01): + """The wrapper marks only a request the adapter observes; a text-only request runs unmarked.""" + pytest.importorskip("pydantic_ai") + pytest.importorskip("traceai_pydantic_ai") + from pydantic_ai import BinaryContent + from pydantic_ai.messages import ModelRequest, UserPromptPart + from traceai_pydantic_ai._attachments import PydanticAIMedia + from traceai_pydantic_ai._execution_wrappers import _call_owner + + media = PydanticAIMedia(f01.manager(), None) + text = ModelRequest(parts=[UserPromptPart(content="text only")]) + image = ModelRequest(parts=[UserPromptPart(content=["what", BinaryContent(data=IMAGE, media_type="image/png")])]) + assert _call_owner(media, [text]) is None + assert isinstance(_call_owner(media, [text, image]), mc.CallOwner) + assert _call_owner(None, [image]) is None