diff --git a/.github/workflows/attachments-python-frameworks-tests.yml b/.github/workflows/attachments-python-frameworks-tests.yml new file mode 100644 index 000000000..86b719034 --- /dev/null +++ b/.github/workflows/attachments-python-frameworks-tests.yml @@ -0,0 +1,89 @@ +name: Python framework attachments + +# Real LangChain, LlamaIndex, Agno and Pydantic AI (declared floor and latest) through +# the Python framework instrumentors to a TLS loopback F01 attachment server, with and +# without traceai_openai (and traceai_anthropic under LangChain) on the same manager. +# Outbound network is disabled while the tests run. + +on: + pull_request: + paths: + - 'python/fi_instrumentation/attachments/**' + - 'python/frameworks/langchain/**' + - 'python/frameworks/llama_index/**' + - 'python/frameworks/agno/**' + - 'python/frameworks/pydantic-ai/**' + - 'python/frameworks/openai/**' + - 'python/frameworks/anthropic/**' + - 'python/tests/attachments_frameworks/**' + - 'python/tests/attachments_providers/conftest.py' + - '.github/workflows/attachments-python-frameworks-tests.yml' + push: + branches: [dev] + paths: + - 'python/fi_instrumentation/attachments/**' + - 'python/frameworks/langchain/**' + - 'python/frameworks/llama_index/**' + - 'python/frameworks/agno/**' + - 'python/frameworks/pydantic-ai/**' + - 'python/frameworks/openai/**' + - 'python/frameworks/anthropic/**' + - 'python/tests/attachments_frameworks/**' + - 'python/tests/attachments_providers/conftest.py' + - '.github/workflows/attachments-python-frameworks-tests.yml' + workflow_dispatch: + +permissions: + contents: read + +jobs: + frameworks: + name: Framework attachments (${{ matrix.name }}) + runs-on: ubuntu-24.04 + timeout-minutes: 25 + strategy: + fail-fast: false + matrix: + include: + # Declared floors (ChatMessage blocks need llama-index-core 0.12.3+). pydantic-ai 0.1.0 imports + # opentelemetry._events, which OpenTelemetry API 1.39+ no longer ships; Agno imports fastapi. + - name: floor + python: '3.10' + sdks: >- + langchain-core==0.2.43 llama-index-core==0.12.41 agno==2.7.2 + openinference-instrumentation-agno==1.1.0 pydantic-ai-slim[openai]==0.1.0 openai==1.74.0 anthropic==0.41.0 + fastapi 'httpx<0.28' 'opentelemetry-api>=1.28,<1.39' 'opentelemetry-sdk>=1.28,<1.39' + # Latest releases, including the framework OpenAI and Anthropic integrations for the + # provider-and-framework tests. + - name: latest + python: '3.12' + sdks: >- + langchain-core==1.8.0 langchain-openai==1.7.4 llama-index-core==0.14.25 + llama-index-llms-openai==0.8.4 agno==3.1.12 openinference-instrumentation-agno==1.1.0 + pydantic-ai-slim[openai]==2.67.0 langchain-anthropic==1.7.10 anthropic==2.8.0 fastapi + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: false + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: ${{ matrix.python }} + - name: Install source integrations and the selected frameworks + env: + SDKS: ${{ matrix.sdks }} + run: | + python -m venv .venv + # shellcheck disable=SC2086 + eval .venv/bin/pip install -e python -e python/frameworks/openai -e python/frameworks/anthropic \ + -e python/frameworks/langchain \ + -e python/frameworks/llama_index -e python/frameworks/agno -e python/frameworks/pydantic-ai \ + opentelemetry-sdk "'pytest>=7,<9'" $SDKS + .venv/bin/pip freeze | grep -iE '^(langchain|llama-index-core|llama-index-llms|agno|pydantic-ai|openai|anthropic|openinference)' || true + - name: Run with outbound network disabled + run: | + sudo unshare --net -- bash -e -c ' + ip link set lo up + cd "$2/python" + AGNO_TELEMETRY=false PYTHONPATH=. "$1" -m pytest tests/attachments_frameworks -q -p no:cacheprovider \ + -o addopts="" -o faulthandler_timeout=120 + ' _ "$PWD/.venv/bin/python" "$PWD" diff --git a/python/fi_instrumentation/attachments/media_capture.py b/python/fi_instrumentation/attachments/media_capture.py index a2fd637c4..18db7b6d6 100644 --- a/python/fi_instrumentation/attachments/media_capture.py +++ b/python/fi_instrumentation/attachments/media_capture.py @@ -13,22 +13,46 @@ - One registry per manager remembers admitted (span, position) pairs, so a second observer of the same span and position gets the FIRST occurrence back and uploads nothing. +- A per-call ``CapturePolicy`` set in the OpenTelemetry context under + ``CAPTURE_POLICY_KEY`` (``traceai.with_attachment_capture_policy``) can only + add restrictions. A context marked under ``OWNER_KEY`` + (``traceai.with_attachment_ownership``) belongs to an adapter that captures + the media itself, so nothing is captured here for that call. +- One model call, one owner. A framework adapter (LangChain, LlamaIndex, Agno, + Pydantic AI) that captures a model call's media runs the downstream provider + SDK call under ``owned_call`` (a ``CallOwner`` under ``OWNER_KEY``), so a + provider instrumentor beneath it (``traceai_openai``, ``traceai_anthropic``, + ...) uploads nothing for that call. Inside its own mark the adapter's + ``capture_parts`` still captures. With attachments off nothing is marked. + The adapter marks only a call in which it observes at least one input + media part: a source it never reads (a local path the SDK inlines) is left + to the provider instrumentor. +- Across DIFFERENT calls every observing span gets its OWN occurrence: + identical bytes in two calls, or at two positions, are two occurrences; + byte deduplication is server-side. A ``fi-attachment:`` reference is only + ever projected onto the span whose ``fi.attachment.refs`` carries that + occurrence. Every public function is non-throwing and returns ``None``/empty on denial. """ import base64 import binascii +import collections.abc +import contextlib import dataclasses import re import threading import weakref from collections import OrderedDict -from typing import Any, Callable, Iterable, Mapping, NamedTuple, Optional, Tuple +from types import MappingProxyType +from typing import Any, Callable, Iterable, Mapping, NamedTuple, Optional, Sequence, Tuple + +from opentelemetry import context as otel_context from .policy import CapturePolicy from .stream import StreamOptions -from .types import Bytes, CorrelationHandle +from .types import Bytes, CorrelationHandle, RemoteUrl KiB = 1024 MiB = 1024 * KiB @@ -41,6 +65,11 @@ STREAM_RESERVATIONS = (4 * MiB, 2 * MiB, 1 * MiB, 512 * KiB, 256 * KiB) IMAGE_MIMES = ("image/png", "image/jpeg", "image/webp") FILE_MIMES = ("application/pdf", "application/octet-stream") +DOCUMENT_MIMES = ("application/pdf",) +AUDIO_MIMES = ("audio/wav", "audio/mpeg", "audio/ogg", "audio/flac", "audio/webm") +# Framework boundaries hand over complete inline parts (mirrors @traceai/framework-attachments). +MAX_FRAMEWORK_BYTES = 1 * MiB +MAX_FRAMEWORK_PARTS = 64 TOOL_ID = re.compile(r"[a-zA-Z0-9_-]{1,128}") _DATA_URI = re.compile(r"data:([a-z]+/[a-z0-9.+-]+);base64") _BASE64 = re.compile(r"[A-Za-z0-9+/]*={0,2}") @@ -50,6 +79,13 @@ # mulaw have no F01 MIME, so they are never captured. SPEECH_FORMATS = {"mp3": "audio/mpeg", "opus": "audio/ogg", "ogg": "audio/ogg", "flac": "audio/flac", "wav": "audio/wav"} +# Context keys shared BY NAME with the auto module (``traceai``) and any adapter, like the +# TypeScript ``Symbol.for`` keys: plain strings, so no package has to import another. +CAPTURE_POLICY_KEY = "traceai.auto.attachment_policy.v1" +OWNER_KEY = "traceai.auto.attachment_owner.v1" +# CapturePolicy facts that grant capture; every other fact only restricts. +_PERMISSIONS = frozenset(("capture_content", "record_inputs", "record_outputs", "requested")) +_DENY_ALL = CapturePolicy(capture_content=False, requested=False) class Occurrence(NamedTuple): @@ -93,6 +129,40 @@ def _restricted(config, name: str) -> bool: return True +def merge_capture_policies(*policies: Any) -> CapturePolicy: + """Deny-wins intersection: a grant needs every policy, a restriction needs any one. + + ``None`` adds nothing. Any other value that is not a ``CapturePolicy`` denies. + """ + present = [policy for policy in policies if policy is not None] + if any(not isinstance(policy, CapturePolicy) for policy in present): + return _DENY_ALL + if not present: + return CapturePolicy() + values = {} + for item in dataclasses.fields(CapturePolicy): + facts = [getattr(policy, item.name) for policy in present] + values[item.name] = all(facts) if item.name in _PERMISSIONS else any(facts) + return CapturePolicy(**values) + + +def context_capture_policy(context=None) -> Any: + """The per-call policy in the (current) OpenTelemetry context; None when unset.""" + try: + return otel_context.get_value(CAPTURE_POLICY_KEY, context) + except Exception: + # An unreadable restriction is a restriction, never permission. + return _DENY_ALL + + +def owned_elsewhere(context=None) -> bool: + """True when another adapter marked this call's media as its own.""" + try: + return otel_context.get_value(OWNER_KEY, context) is not None + except Exception: + return True + + def local_capture_policy(trace_config=None, *, direction: str, image: bool = False, error_detail: bool = False, base: Optional[CapturePolicy] = None) -> CapturePolicy: """Fold the instrumentor's local TraceConfig into a deny-only CapturePolicy. @@ -103,6 +173,7 @@ def local_capture_policy(trace_config=None, *, direction: str, image: bool = Fal ancestry, so a tool screenshot is still treated as model input. """ base = base if isinstance(base, CapturePolicy) else CapturePolicy() + base = merge_capture_policies(base, context_capture_policy()) names = _INPUT_HIDE if direction == "input" else _OUTPUT_HIDE hidden = any(_restricted(trace_config, name) for name in names) or _restricted(trace_config, "pii_redaction") if image and direction == "input" and _restricted(trace_config, "hide_input_images"): @@ -116,6 +187,8 @@ def local_capture_policy(trace_config=None, *, direction: str, image: bool = Fal changes["model_input"] = True if error_detail: changes["error_detail"] = True + if owned_elsewhere(): + changes["requested"] = False return dataclasses.replace(base, **changes) @@ -159,19 +232,45 @@ def put(self, manager, key: str, position: str, occurrence: Occurrence) -> None: REGISTRY = _Registry() +def _attach_admitted(ticket, data: bytes, mime: str, fields: dict) -> Optional[Occurrence]: + """Attach ADMITTED bytes as a new occurrence of the ticket's span. Never raises. + + No cross-span reuse: an occurrence belongs to the span that observed it + (F01 resolve checks the parent), so every observer uploads its own. + """ + try: + if not ticket.admitted: + return None + handle = ticket.attach(Bytes(data), **fields) + return Occurrence(handle.occurrence_id, mime) if handle.admission == "queued" else None + except Exception: + try: + ticket.release() + except Exception: + pass + return None + + def encoded_length(encoded: Any, max_bytes: int = MAX_INLINE_BYTES) -> int: """Exact decoded length from the string's shape only (no decode); 0 when out of bounds.""" - if not isinstance(encoded, str): + if not isinstance(encoded, (str, bytes)): return 0 length = len(encoded) if not length or length % 4 or length > 4 * -(-max_bytes // 3): return 0 - return (length // 4) * 3 - (2 if encoded.endswith("==") else 1 if encoded.endswith("=") else 0) + pad = "=" if isinstance(encoded, str) else b"=" + return (length // 4) * 3 - (2 if encoded.endswith(pad * 2) else 1 if encoded.endswith(pad) else 0) def split_inline(value: Any, mime_types: Tuple[str, ...], raw: bool, max_bytes: int = MAX_INLINE_BYTES) -> Optional[Tuple[str, str, int]]: """Structural parse: (mime, encoded, exact decoded length) or None. Bounded before slicing.""" + if raw and isinstance(value, bytes) and 0 < len(value) <= 4 * -(-max_bytes // 3): + # Base64 held as ASCII bytes (LlamaIndex blocks). Bounded before the copy. + try: + value = value.decode("ascii") + except UnicodeDecodeError: + return None if not isinstance(value, str): return None mime, start = mime_types[0], 0 @@ -212,8 +311,12 @@ def attach_inline(manager, span, read: Callable[[], Any], mime_types: Tuple[str, direction: str, purpose: Optional[str], capture: CapturePolicy, message_index: Optional[int] = None, part_index: Optional[int] = None, role: Optional[str] = None, tool_call_id: Optional[str] = None, - max_bytes: int = MAX_INLINE_BYTES) -> Optional[Occurrence]: - """Preflight, then read once, decode strictly and attach. Never raises.""" + max_bytes: int = MAX_INLINE_BYTES, verify: bool = False) -> Optional[Occurrence]: + """Preflight, then read once, decode strictly and attach. Never raises. + + ``verify`` also requires the decoded container (magic bytes) to match the + MIME it was admitted under. + """ try: state = {} @@ -248,11 +351,75 @@ def preflight(mime): if not ticket.admitted: return None data = strict_decode(encoded, ticket.max_bytes, max_bytes) - if data is None: + if data is None or (verify and sniff_media(data[:16]) != mime): ticket.release() return None - handle = ticket.attach(Bytes(data), **_fields(message_index, part_index, role, tool_call_id)) - return Occurrence(handle.occurrence_id, mime) if handle.admission == "queued" else None + return _attach_admitted(ticket, data, mime, _fields(message_index, part_index, role, tool_call_id)) + except Exception: + return None + + +def sniff_media(head: Any) -> Optional[str]: + """Image, PDF or audio MIME from magic bytes the application already holds; None when unknown.""" + if not isinstance(head, (bytes, bytearray, memoryview)): + return None + head = bytes(head[:16]) + if head[:8] == b"\x89PNG\r\n\x1a\n": + return "image/png" + if head[:3] == b"\xff\xd8\xff": + return "image/jpeg" + if len(head) >= 12 and head[:4] == b"RIFF" and head[8:12] == b"WEBP": + return "image/webp" + if head[:5] == b"%PDF-": + return "application/pdf" + return sniff_audio(head) + + +def attach_bytes(manager, span, data: Any, mime_types: Tuple[str, ...], *, direction: str, + purpose: Optional[str], capture: CapturePolicy, message_index: Optional[int] = None, + part_index: Optional[int] = None, role: Optional[str] = None, + tool_call_id: Optional[str] = None, max_bytes: int = MAX_FRAMEWORK_BYTES) -> Optional[Occurrence]: + """Complete in-memory bytes a framework already holds (never a stream, path or URL). Never raises. + + ``data`` may be a zero-argument callable returning the bytes: it is called only + after a preflight at ``max_bytes`` passed every privacy, correlation and sampling + step, so a denied part is never read. The bytes are then admitted at their exact + length under the declared type; the magic bytes must name one of ``mime_types``, + re-admitted under that exact type. + """ + try: + if not mime_types: + return None + if callable(data): + probe = manager.preflight(span, mime_type=mime_types[0], direction=direction, purpose=purpose, + capture_policy=capture, max_bytes=max_bytes) + # too_large and queue_full come after every privacy, correlation and sampling step, + # so reading and sizing exactly is still allowed for them. + if not probe.admitted and probe.reason not in ("too_large", "queue_full"): + return None + if probe.admitted: + probe.release() + data = data() + if not isinstance(data, (bytes, bytearray, memoryview)): + return None + length = data.nbytes if isinstance(data, memoryview) else len(data) + if not 0 < length <= max_bytes: + return None + options = dict(direction=direction, purpose=purpose, capture_policy=capture, max_bytes=length) + ticket = manager.preflight(span, mime_type=mime_types[0], **options) + if not ticket.admitted: + return None + mime = sniff_media(data[:16]) + if mime is None or mime not in mime_types: + ticket.release() + return None + if mime != mime_types[0]: + ticket.release() + ticket = manager.preflight(span, mime_type=mime, **options) + if not ticket.admitted: + return None + payload = data if isinstance(data, bytes) else bytes(data) + return _attach_admitted(ticket, payload, mime, _fields(message_index, part_index, role, tool_call_id)) except Exception: return None @@ -262,7 +429,7 @@ class Observation: def __init__(self, manager, span, namespace: str): self.manager, self.span, self.namespace = manager, span, namespace - self.key = span_key(span) if active(manager) else None + self.key = span_key(span) if active(manager) and not owned_elsewhere() else None self.captured: "OrderedDict[str, Occurrence]" = OrderedDict() self._visited = 0 @@ -343,9 +510,8 @@ def attach_audio(manager, span, data: Any, *, direction: str, purpose: Optional[ ticket = manager.preflight(span, mime_type=mime, **options) if not ticket.admitted: return None - handle = ticket.attach(Bytes(bytes(data) if not isinstance(data, bytes) else data), - **_fields(message_index, part_index, role)) - return Occurrence(handle.occurrence_id, mime) if handle.admission == "queued" else None + return _attach_admitted(ticket, bytes(data) if not isinstance(data, bytes) else data, mime, + _fields(message_index, part_index, role)) except Exception: return None @@ -455,3 +621,247 @@ def request_files(files: Any) -> Iterable[Tuple[int, Any]]: yield index, content except Exception: return + + +# ------------------------------------------------------------------ framework parts +class MediaPart(NamedTuple): + """One typed media part a framework binding found at an original-value boundary. + + ``kind`` is ``"inline"`` (base64 str/ASCII bytes, or a data URI when ``raw`` is + False), ``"bytes"`` (complete raw bytes) or ``"url"`` (a remote reference that + is handed to the core's ``RemoteUrl`` policy and never fetched here). + ``read`` is called at most once, and only after admission. + """ + + kind: str + read: Callable[[], Any] + mime_types: Tuple[str, ...] + purpose: str + message_index: int + part_index: int + role: str = "unknown" + raw: bool = True + tool_call_id: Optional[str] = None + + +def intersect(*policies: CapturePolicy) -> CapturePolicy: + """Deny-wins intersection: any layer that hides content, mutes or marks provenance wins.""" + policies = tuple(p for p in policies if isinstance(p, CapturePolicy)) or (CapturePolicy(),) + allow = ("capture_content", "record_inputs", "record_outputs", "requested") + mark = ("muted", "temporary_chat", "model_input", "contains_input", "contains_output", "error_detail", + "new_recording") + values = {name: all(getattr(p, name) for p in policies) for name in allow} + values.update({name: any(getattr(p, name) for p in policies) for name in mark}) + return CapturePolicy(**values) + + +class CallOwner: + """The ``OWNER_KEY`` value a framework adapter sets around ONE model call whose media it captures. + + Any value other than None marks the call as owned (``owned_elsewhere``), so provider + instrumentors beneath the adapter skip it. ``namespace`` is set only by an adapter that + captures INSIDE its own mark (LangChain, LlamaIndex callbacks): ``capture_parts`` with the + same namespace treats the mark as its own and still captures. An adapter that captures + before marking (Agno, Pydantic AI) passes None, so a nested request of the same framework + (a wrapper or fallback model) is owned too. + """ + + __slots__ = ("namespace",) + + def __init__(self, namespace: Optional[str] = None) -> None: + self.namespace = namespace + + +def call_owner(manager, namespace: Optional[str] = None) -> Optional[CallOwner]: + """The owner mark for one model call this adapter captures, or None to leave the context as is. + + None when ``manager`` cannot admit (attachments off: the base code path, nothing marked) + or when the call is already owned (an outer adapter, ``traceai.with_attachment_ownership``). + """ + try: + if not active(manager) or owned_elsewhere(): + return None + return CallOwner(namespace) + except Exception: + return None + + +@contextlib.contextmanager +def owner_scope(owner: Optional[CallOwner]): + """Attach ``owner`` for the enclosed synchronous step or awaited operation, never across a yield.""" + if owner is None: + yield + return + token = otel_context.attach(otel_context.set_value(OWNER_KEY, owner)) + try: + yield + finally: + otel_context.detach(token) + + +def owned_call(manager, namespace: Optional[str] = None): + """Run the enclosed downstream provider SDK call as the owner of its media (see ``call_owner``).""" + return owner_scope(call_owner(manager, namespace)) + + +class _OwnedStream: + """A returned generator advanced (and closed) under the owner mark, restored before every yield. + + Implements the full generator protocol, so ``collections.abc.Generator`` checks still pass; + any other attribute is the wrapped generator's. + """ + + __slots__ = ("_stream", "_owner") + + def __init__(self, stream: Any, owner: CallOwner) -> None: + self._stream, self._owner = stream, owner + + def __getattr__(self, name: str) -> Any: + return getattr(self._stream, name) + + def __iter__(self) -> "_OwnedStream": + return self + + def __next__(self) -> Any: + with owner_scope(self._owner): + return next(self._stream) + + def send(self, value: Any) -> Any: + with owner_scope(self._owner): + return self._stream.send(value) + + def throw(self, *args: Any) -> Any: + with owner_scope(self._owner): + return self._stream.throw(*args) + + def close(self) -> Any: + with owner_scope(self._owner): + return self._stream.close() + + +class _OwnedAsyncStream: + """The async-generator form of ``_OwnedStream``: each awaited step runs under the mark.""" + + __slots__ = ("_stream", "_owner") + + def __init__(self, stream: Any, owner: CallOwner) -> None: + self._stream, self._owner = stream, owner + + def __getattr__(self, name: str) -> Any: + return getattr(self._stream, name) + + def __aiter__(self) -> "_OwnedAsyncStream": + return self + + async def __anext__(self) -> Any: + with owner_scope(self._owner): + return await self._stream.__anext__() + + async def asend(self, value: Any) -> Any: + with owner_scope(self._owner): + return await self._stream.asend(value) + + async def athrow(self, *args: Any) -> Any: + with owner_scope(self._owner): + return await self._stream.athrow(*args) + + async def aclose(self) -> Any: + with owner_scope(self._owner): + return await self._stream.aclose() + + +def owned_stream(result: Any, owner: Optional[CallOwner]) -> Any: + """``result`` itself, or a generator / async generator proxy advanced under ``owner``. + + A lazy SDK stream makes its provider request on the first advance, after the call that + returned it; the mark covers every advance but is never left attached across a yield, so + the caller's code between chunks (and an abandoned stream) is never marked. + """ + if owner is None: + return result + # Async first: a stream proxy (LlamaIndex's) can implement both protocols. + if isinstance(result, collections.abc.AsyncGenerator): + return _OwnedAsyncStream(result, owner) + if isinstance(result, collections.abc.Generator): + return _OwnedStream(result, owner) + return result + + +def _own_mark(namespace: str) -> bool: + try: + owner = otel_context.get_value(OWNER_KEY) + except Exception: + return False + return isinstance(owner, CallOwner) and owner.namespace is not None and owner.namespace == namespace + + +def capture_parts(manager, span, namespace: str, parts: Iterable[MediaPart], *, direction: str, + trace_configs: Sequence[Any] = (), capture: Optional[CapturePolicy] = None, + max_bytes: int = MAX_FRAMEWORK_BYTES, + policy_directions: Sequence[str] = ()) -> Mapping[str, Occurrence]: + """Capture typed framework parts on ``span``; returns admitted ``"{dir}/{msg}/{part}"`` positions. + + Every local ``trace_configs`` entry (FI TraceConfig, OpenInference TraceConfig, ...) + folds in deny-only for every direction in ``policy_directions`` (default: the upload + ``direction``); the strictest wins. A tool span's result is that span's OUTPUT and a + later model INPUT, so it passes ``("input", "output")``: any input or output hide flag + denies it. At most ``MAX_FRAMEWORK_PARTS`` positions per span. Never raises, never + walks arbitrary objects, never fetches or opens anything. + """ + if _own_mark(namespace): + # This adapter owns the call: the mark is for the provider beneath it, not for itself. + token = otel_context.attach(otel_context.set_value(OWNER_KEY, None)) + try: + return capture_parts(manager, span, namespace, parts, direction=direction, + trace_configs=trace_configs, capture=capture, max_bytes=max_bytes, + policy_directions=policy_directions) + finally: + otel_context.detach(token) + observation = Observation(manager, span, namespace) + if not observation.usable: + return EMPTY + try: + directions = tuple(policy_directions) or (direction,) + configs = tuple(trace_configs) or (None,) + visited = 0 + for part in parts: + visited += 1 + if visited > MAX_FRAMEWORK_PARTS or len(observation.captured) >= MAX_FRAMEWORK_PARTS: + break + if not isinstance(part, MediaPart) or not part.mime_types: + continue + image = part.mime_types[0].startswith("image/") + policy = intersect(*(local_capture_policy(config, direction=fold, base=capture, image=image) + for config in configs for fold in directions)) + fields = dict(direction=direction, purpose=part.purpose, capture=policy, + message_index=part.message_index, part_index=part.part_index, role=part.role, + tool_call_id=part.tool_call_id) + position = "%s/%d/%d" % (direction, part.message_index, part.part_index) + if part.kind == "inline": + observation.visit(position, lambda p=part, f=fields: attach_inline( + manager, span, p.read, p.mime_types, raw=p.raw, max_bytes=max_bytes, verify=True, **f)) + elif part.kind == "bytes": + observation.visit(position, lambda p=part, f=fields: attach_bytes( + manager, span, p.read, p.mime_types, max_bytes=max_bytes, **f)) + elif part.kind == "url": + observation.visit(position, lambda p=part, f=fields: _remote(manager, span, p, f)) + except Exception: + pass # Instrumentation must not throw or log source-bearing errors. + return MappingProxyType(dict(observation.captured)) if observation.captured else EMPTY + + +def looks_base64(data: Any) -> bool: + """Bytes a framework holds as base64 ASCII (vs raw container bytes); decided from the head only.""" + if not isinstance(data, (bytes, bytearray)) or not data or len(data) % 4: + return False + return all(48 <= c <= 57 or 65 <= c <= 90 or 97 <= c <= 122 or c in (43, 47, 61) for c in bytes(data[:64])) + + +def _remote(manager, span, part: MediaPart, fields: dict) -> Optional[Occurrence]: + """A remote reference goes to the core's RemoteUrl policy. Nothing here resolves or fetches it.""" + url = part.read() + if not isinstance(url, str) or not url.startswith(("https://", "http://")) or len(url) > 2048: + return None + handle = manager.attach(span, RemoteUrl(url), mime_type=part.mime_types[0], direction=fields["direction"], + purpose=fields["purpose"], capture_policy=fields["capture"]) + return Occurrence(handle.occurrence_id, part.mime_types[0]) if handle.admission == "queued" else None diff --git a/python/frameworks/agno/traceai_agno/_attachments.py b/python/frameworks/agno/traceai_agno/_attachments.py new file mode 100644 index 000000000..27182b93e --- /dev/null +++ b/python/frameworks/agno/traceai_agno/_attachments.py @@ -0,0 +1,175 @@ +"""Agno typed media through the shared, provider-owned attachment manager. + +Observed at the model boundary only (``Model.invoke`` and friends): + +- input: ``Message.images`` / ``audio`` / ``files`` of every message (tool + messages keep their ``tool_call_id``); +- output: ``ModelResponse.images`` / ``audio`` / ``audios`` / ``files`` of a + non-streaming call. + +Only ``content`` held as bytes is captured, and its magic bytes must match an +admitted type. ``filepath`` is never opened, ``url`` goes to the core's +``RemoteUrl`` policy without being fetched, ``external`` provider objects and +media references are ignored. Videos are not captured. + +Privacy is the INTERSECTION of the FI TraceConfig and the OpenInference +TraceConfig the instrumentor runs with (either one hiding wins), the manager's +own policy, and the per-call capture policy. + +One model call, one owner: while attachments are enabled, a model call in which +this adapter observes at least one input media part runs under the owner mark +(``media_capture.OWNER_KEY``, a stream only while it advances), so a provider +instrumentor beneath Agno (``traceai_openai``, ...) uploads nothing for that call; +the Agno span holds the occurrence. A call with no observed part is not marked: +Agno inlines a ``filepath`` image or file as a data URI before the SDK call, so the +provider span captures it. Residual: a call that mixes an observed part with a +``filepath`` one is marked, so the provider skips the ``filepath`` part too. A DIFFERENT +call with the same bytes gets its own occurrence (byte deduplication is +server-side). +""" + +from types import MappingProxyType +from typing import Any, Iterator, Mapping, Optional, Sequence + +from fi_instrumentation.attachments.media_capture import ( + AUDIO_MIMES, + DOCUMENT_MIMES, + EMPTY, + IMAGE_MIMES, + MAX_FRAMEWORK_BYTES, + MAX_FRAMEWORK_PARTS, + MediaPart, + Occurrence, + capture_parts, + looks_base64, +) + +__all__ = ("capture_agno_input", "capture_agno_output", "agno_attachment_capabilities", "observes_agno_input") + +agno_attachment_capabilities = MappingProxyType({ + "schema_version": 1, + "operations": ("model.invoke.input", "model.invoke.output", "model.invoke_stream.input"), + "sources": ("content_bytes",), + "unsupported": ("filepath", "remote_url_fetch", "external_file", "media_reference", "video", + "streamed_output"), + "max_part_bytes": MAX_FRAMEWORK_BYTES, + "max_parts_per_span": MAX_FRAMEWORK_PARTS, +}) + +_ROLES = ("user", "assistant", "tool", "system") +_AUDIO_FORMATS = {"wav": "audio/wav", "mp3": "audio/mpeg", "mpeg": "audio/mpeg", "ogg": "audio/ogg", + "flac": "audio/flac", "webm": "audio/webm"} +_IMAGE_FORMATS = {"png": "image/png", "jpeg": "image/jpeg", "jpg": "image/jpeg", "webp": "image/webp"} +# Message/ModelResponse field -> MIME family. Order fixes part indexes. +_INPUT_FIELDS = (("images", IMAGE_MIMES), ("audio", AUDIO_MIMES), ("files", DOCUMENT_MIMES)) +_OUTPUT_FIELDS = (("images", IMAGE_MIMES), ("audio", AUDIO_MIMES), ("audios", AUDIO_MIMES), + ("files", DOCUMENT_MIMES)) + + +def _field(value: Any, name: str) -> Any: + """A declared model field only: never a property, resolver or ``__getattr__``.""" + try: + return vars(value).get(name) + except TypeError: + return None + + +def _ordered(mime: Any, family: tuple) -> Optional[tuple]: + if mime is None: + return family + if isinstance(mime, str) and mime in family: + return (mime,) + tuple(m for m in family if m != mime) + return None + + +def _declared(media: Any, family: tuple) -> Optional[tuple]: + mime = _field(media, "mime_type") + if mime is None: + fmt = _field(media, "format") + if isinstance(fmt, str): + mime = (_IMAGE_FORMATS if family is IMAGE_MIMES else _AUDIO_FORMATS if family is AUDIO_MIMES + else {"pdf": "application/pdf"}).get(fmt.lower(), "unsupported/format") + return _ordered(mime, family) + + +def _media_parts(values: Any, family: tuple, purpose: str, message_index: int, start: int, role: str, + tool_call_id: Optional[str]) -> Iterator[MediaPart]: + values = [values] if values is not None and not isinstance(values, (list, tuple)) else list(values or ()) + for offset, media in enumerate(values[:MAX_FRAMEWORK_PARTS]): + mimes = _declared(media, family) + if not mimes: + continue + content = _field(media, "content") + url = _field(media, "url") + index = start + offset + if isinstance(content, (bytes, bytearray)): + kind = "inline" if looks_base64(content) else "bytes" + yield MediaPart(kind, lambda c=content: c, mimes, purpose, message_index, index, role=role, + tool_call_id=tool_call_id) + elif content is None and isinstance(url, str) and _field(media, "filepath") is None: + yield MediaPart("url", lambda u=url: u, mimes, purpose, message_index, index, role=role, + tool_call_id=tool_call_id) + # filepath, external objects, media references, str content: never read or resolved. + + +def _input_parts(messages: Sequence[Any]) -> Iterator[MediaPart]: + for message_index, message in enumerate(list(messages)[:MAX_FRAMEWORK_PARTS]): + role = _field(message, "role") + role = role if role in _ROLES else "unknown" + tool_call_id = _field(message, "tool_call_id") if role == "tool" else None + tool_call_id = tool_call_id if isinstance(tool_call_id, str) else None + start = 0 + for name, family in _INPUT_FIELDS: + values = _field(message, name) + purpose = ("tool_result" if role == "tool" else "document" if family is DOCUMENT_MIMES + else "model_input") + parts = list(_media_parts(values, family, purpose, message_index, start, role, tool_call_id)) + yield from parts + start += len(values) if isinstance(values, (list, tuple)) else (1 if values is not None else 0) + + +def _output_parts(response: Any) -> Iterator[MediaPart]: + start = 0 + for name, family in _OUTPUT_FIELDS: + values = _field(response, name) + yield from _media_parts(values, family, "model_output", 0, start, "assistant", None) + start += len(values) if isinstance(values, (list, tuple)) else (1 if values is not None else 0) + + +def capture_agno_input(manager, span, messages: Any, *, trace_configs: Sequence[Any] = (), + capture=None) -> Mapping[str, Occurrence]: + """Capture typed media of the model call's input messages. Never raises.""" + try: + if not isinstance(messages, (list, tuple)): + return EMPTY + return capture_parts(manager, span, "agno", _input_parts(messages), direction="input", + trace_configs=trace_configs, capture=capture) + except Exception: + return EMPTY + + +def observes_agno_input(messages: Any) -> bool: + """True when this adapter observes at least one input media part (owner-mark gate). Never raises. + + A ``filepath`` part is never observed, so a call holding only those is left to the + provider instrumentor. An error means False (no mark): the provider is never + suppressed by mistake. + """ + try: + if not isinstance(messages, (list, tuple)): + return False + return next(_input_parts(messages), None) is not None + except Exception: + return False + + +def capture_agno_output(manager, span, response: Any, *, trace_configs: Sequence[Any] = (), + capture=None) -> Mapping[str, Occurrence]: + """Capture generated media of a non-streaming ``ModelResponse``. Never raises.""" + try: + if response is None: + return EMPTY + return capture_parts(manager, span, "agno", _output_parts(response), direction="output", + trace_configs=trace_configs, capture=capture) + except Exception: + return EMPTY diff --git a/python/frameworks/agno/traceai_agno/_instrumentor.py b/python/frameworks/agno/traceai_agno/_instrumentor.py index 266f922a7..673b903c0 100644 --- a/python/frameworks/agno/traceai_agno/_instrumentor.py +++ b/python/frameworks/agno/traceai_agno/_instrumentor.py @@ -36,6 +36,7 @@ class AgnoInstrumentorWrapper: dict[str, str], str | None, TraceConfig | None, + object | None, ] | None ) @@ -59,6 +60,7 @@ def instrument( *, project_name: str | None = None, config: TraceConfig | None = None, + attachments: object | None = None, ) -> AgnoInstrumentorWrapper: """Use an existing provider OR create an OTLP/HTTP protobuf exporter. @@ -66,6 +68,9 @@ def instrument( globally. Endpoint configuration owns a private provider. With no arguments, an already configured global provider is required. ``config`` is OpenInference's TraceConfig, not fi_instrumentation's. + ``attachments`` is the provider-owned attachment manager + (``register(...).attachment_manager``); media is captured only when it is + enabled, under both the FI and the OpenInference content policy. """ with self._lock: configuration = ( @@ -74,6 +79,7 @@ def instrument( dict(otlp_headers or {}), project_name, config, + attachments, ) if self._agno_instrumentor is not None and not self.is_instrumented: raise RuntimeError( @@ -88,6 +94,7 @@ def instrument( otlp_headers, project_name, config, + attachments, ) ): return self @@ -151,6 +158,7 @@ def instrument( delegate, tracer_provider=provider, config=config, + attachments=attachments, raise_exception_on_conflict=True, ) if not delegate.is_instrumented_by_opentelemetry: @@ -204,6 +212,7 @@ def configure_agno_tracing( project_name: str | None = None, *, config: TraceConfig | None = None, + attachments: object | None = None, ) -> AgnoInstrumentorWrapper: """Configure Agno using an existing provider or a complete OTLP traces URL. @@ -219,6 +228,7 @@ def configure_agno_tracing( otlp_headers=otlp_headers, project_name=project_name, config=config, + attachments=attachments, ) diff --git a/python/frameworks/agno/traceai_agno/_wrappers/_install.py b/python/frameworks/agno/traceai_agno/_wrappers/_install.py index 42678cd36..cd0ed6987 100644 --- a/python/frameworks/agno/traceai_agno/_wrappers/_install.py +++ b/python/frameworks/agno/traceai_agno/_wrappers/_install.py @@ -39,6 +39,17 @@ def _wrap_methods( wrap_function_wrapper(target, name, wrapper) +def _attachment_manager(manager: Any) -> Any: + """An enabled shared manager, else None (a disabled or pre-attachments core is a no-op).""" + if manager is None: + return None + try: + from fi_instrumentation.attachments.media_capture import active + except ImportError: + return None + return manager if active(manager) else None + + def _instrument(self: Any, **kwargs: Any) -> None: from agno.agent import _run as agent_run from agno.knowledge.knowledge import Knowledge @@ -75,7 +86,12 @@ def _instrument(self: Any, **kwargs: Any) -> None: ), ) run = _RunWrapper(self._tracer) - model = _ModelWrapper(self._tracer, hide_inputs=bool(config.hide_inputs)) + model = _ModelWrapper( + self._tracer, + hide_inputs=bool(config.hide_inputs), + attachments=_attachment_manager(kwargs.get("attachments")), + trace_configs=(fi_config, config), + ) tool = _FunctionCallWrapper(self._tracer, hide_inputs=bool(config.hide_inputs)) workflow = _WorkflowWrapper(self._tracer) execute = _WorkflowExecuteWrapper(self._tracer) diff --git a/python/frameworks/agno/traceai_agno/_wrappers/_model_wrapper.py b/python/frameworks/agno/traceai_agno/_wrappers/_model_wrapper.py index f69d923eb..283f3fb7a 100644 --- a/python/frameworks/agno/traceai_agno/_wrappers/_model_wrapper.py +++ b/python/frameworks/agno/traceai_agno/_wrappers/_model_wrapper.py @@ -617,9 +617,68 @@ def _usage_attributes(metrics: Any) -> Iterator[Tuple[str, Any]]: class _ModelWrapper: - def __init__(self, tracer: trace_api.Tracer, *, hide_inputs: bool = False) -> None: + def __init__( + self, + tracer: trace_api.Tracer, + *, + hide_inputs: bool = False, + attachments: Any = None, + trace_configs: Tuple[Any, ...] = (), + ) -> None: self._tracer = tracer self._hide_inputs = hide_inputs + # The shared, provider-owned attachment manager (already checked active), or None. + # trace_configs are the FI and OpenInference configs: either one hiding wins. + self._attachments = attachments + self._trace_configs = trace_configs + + def _capture_input(self, span: Any, arguments: Mapping[str, Any]) -> None: + # Input media is captured before the provider call; this span owns its own occurrences. + if self._attachments is None: + return + try: + from traceai_agno._attachments import capture_agno_input + + capture_agno_input( + self._attachments, span, arguments.get("messages"), + trace_configs=self._trace_configs, + ) + except Exception: + pass + + def _owned(self, values: Dict[str, Any], arguments: Mapping[str, Any]) -> Dict[str, Any]: + """``values`` plus the owner mark when this call's input media is observed here. + + span_scope attaches it only around the downstream call and each stream advance, + so the provider instrumentor beneath Agno skips this call's media. Attachments + off, a call with no input part this adapter observes (only ``filepath`` media, + which the SDK inlines and the provider captures), or a call already owned + further out: ``values`` unchanged. + """ + if self._attachments is None: + return values + try: + from fi_instrumentation.attachments.media_capture import OWNER_KEY, call_owner + from traceai_agno._attachments import observes_agno_input + + if not observes_agno_input(arguments.get("messages")): + return values + owner = call_owner(self._attachments) + except Exception: + return values + return values if owner is None else {**values, OWNER_KEY: owner} + + def _capture_output(self, span: Any, response: Any) -> None: + if self._attachments is None: + return + try: + from traceai_agno._attachments import capture_agno_output + + capture_agno_output( + self._attachments, span, response, trace_configs=self._trace_configs + ) + except Exception: + pass def run( self, @@ -657,10 +716,12 @@ def run( span.set_attribute(LLM_PROVIDER, model.provider) if llm_system := _get_llm_system(model): span.set_attribute(LLM_SYSTEM, llm_system) + self._capture_input(span, arguments) - with span_scope(span, _invocation_context(instance, "invoke", wrapped)): + with span_scope(span, self._owned(_invocation_context(instance, "invoke", wrapped), arguments)): response = wrapped(*args, **kwargs) output_message = _parse_model_output(response) + self._capture_output(span, response) span.set_attributes(dict(_output_value_and_mime_type(output_message))) span.set_attributes( @@ -708,10 +769,11 @@ def run_stream( span.set_attribute(LLM_PROVIDER, model.provider) if llm_system := _get_llm_system(model): span.set_attribute(LLM_SYSTEM, llm_system) + self._capture_input(span, arguments) # Token usage will be set after streaming completes based on final response responses = [] - values = _invocation_context(instance, "invoke_stream", wrapped) + values = self._owned(_invocation_context(instance, "invoke_stream", wrapped), arguments) with span_scope(span, values): source = iter(wrapped(*args, **kwargs)) with closing(iterate(source, span, values)) as iterator: @@ -769,10 +831,12 @@ async def arun( span.set_attribute(LLM_PROVIDER, model.provider) if llm_system := _get_llm_system(model): span.set_attribute(LLM_SYSTEM, llm_system) + self._capture_input(span, arguments) - with span_scope(span, _invocation_context(instance, "ainvoke", wrapped)): + with span_scope(span, self._owned(_invocation_context(instance, "ainvoke", wrapped), arguments)): response = await wrapped(*args, **kwargs) output_message = _parse_model_output(response) + self._capture_output(span, response) span.set_attributes( dict(_usage_attributes(getattr(response, "response_usage", None))) @@ -822,10 +886,11 @@ async def arun_stream( span.set_attribute(LLM_PROVIDER, model.provider) if llm_system := _get_llm_system(model): span.set_attribute(LLM_SYSTEM, llm_system) + self._capture_input(span, arguments) # Token usage will be set after streaming completes based on final response responses = [] - values = _invocation_context(instance, "ainvoke_stream", wrapped) + values = self._owned(_invocation_context(instance, "ainvoke_stream", wrapped), arguments) with span_scope(span, values): source = wrapped(*args, **kwargs) async with aclosing(aiterate(source, span, values)) as iterator: diff --git a/python/frameworks/langchain/traceai_langchain/__init__.py b/python/frameworks/langchain/traceai_langchain/__init__.py index df30b52be..8d79b6df9 100644 --- a/python/frameworks/langchain/traceai_langchain/__init__.py +++ b/python/frameworks/langchain/traceai_langchain/__init__.py @@ -35,6 +35,23 @@ _MODULE = "langchain_core" +def _attachment_manager(kwargs: Any) -> Any: + """The shared manager passed as ``attachments`` (or auto's ``attachment_manager``), if it can admit. + + A missing, disabled or pre-attachments core leaves tracing exactly as before. + """ + manager = kwargs.get("attachments") + if manager is None: + manager = kwargs.get("attachment_manager") + if manager is None: + return None + try: + from fi_instrumentation.attachments.media_capture import active + except ImportError: + return None + return manager if active(manager) else None + + class LangChainInstrumentor(BaseInstrumentor): # type: ignore """ An instrumentor for LangChain @@ -63,7 +80,18 @@ def _instrument(self, **kwargs: Any) -> None: trace_api.get_tracer(__name__, __version__, tracer_provider), config=config, ) - self._tracer: Optional[LangChainFiTracer] = LangChainFiTracer(self.fi_tracer) + attachments = _attachment_manager(kwargs) + self._tracer: Optional[LangChainFiTracer] = LangChainFiTracer( + self.fi_tracer, + attachments=attachments, + trace_config=config, + ) + # One model call, one owner: only while attachments are enabled (else nothing is patched). + self._owner_patches: List[Any] = [] + if attachments is not None: + from traceai_langchain._attachments import own_chat_model_calls + + self._owner_patches = own_chat_model_calls(attachments) self._original_callback_manager_init = BaseCallbackManager.__init__ wrap_function_wrapper( module="langchain_core.callbacks", @@ -82,6 +110,11 @@ def _instrument(self, **kwargs: Any) -> None: self._original_protect = None def _uninstrument(self, **kwargs: Any) -> None: + if getattr(self, "_owner_patches", None): + from traceai_langchain._attachments import release_chat_model_calls + + release_chat_model_calls(self._owner_patches) + self._owner_patches = [] langchain_core.callbacks.BaseCallbackManager.__init__ = self._original_callback_manager_init # type: ignore self._original_callback_manager_init = None # type: ignore if self._protect_class is not None: diff --git a/python/frameworks/langchain/traceai_langchain/_attachments.py b/python/frameworks/langchain/traceai_langchain/_attachments.py new file mode 100644 index 000000000..c78950964 --- /dev/null +++ b/python/frameworks/langchain/traceai_langchain/_attachments.py @@ -0,0 +1,283 @@ +"""LangChain typed message media through the shared, provider-owned attachment manager. + +Mirrors ``@traceai/framework-attachments`` (dialect ``langchain``). Only the typed +content arrays of chat-model messages are observed, in the serialized form the +tracer receives (``dumpd`` messages under ``kwargs.content``): + +- OpenAI-style ``image_url`` / ``file`` / ``input_audio`` parts with inline data; +- LangChain 0.3 standard blocks (``source_type: "base64"``); +- LangChain 1.x content blocks (``base64`` + ``mime_type``); +- Anthropic-style ``source: {"type": "base64"}`` blocks. + +Remote URLs go to the core's ``RemoteUrl`` policy and are never fetched; provider +file IDs and anything else are ignored. + +One model call, one owner: while attachments are enabled, a ``BaseChatModel`` +``generate`` / ``agenerate`` / ``stream`` / ``astream`` call in which this module +observes at least one input media part runs under the owner mark +(``media_capture.owned_call``), so a provider instrumentor beneath LangChain +(``traceai_openai``, ...) uploads nothing for that call; the LangChain span holds +the occurrence. A call with no observed part (no media, or only shapes this module +does not read, such as a provider file ID) is not marked, so the provider span keeps +what only the SDK sees. Residual: a call that mixes an observed part with an +unread one is marked, so the provider skips the unread part too. A DIFFERENT call +with the same bytes gets its own occurrence (byte deduplication is server-side). +The mark also carries LangChain's hide flags down for a marked call: what the +LangChain span may not capture, the provider span does not capture either. + +This module imports ``fi_instrumentation.attachments``; the tracer imports it only +when an enabled attachment manager is passed. +""" + +from types import MappingProxyType +from typing import Any, Iterator, List, Mapping, Optional, Tuple + +from fi_instrumentation.attachments.media_capture import ( + AUDIO_MIMES, + DOCUMENT_MIMES, + EMPTY, + IMAGE_MIMES, + MAX_FRAMEWORK_BYTES, + MAX_FRAMEWORK_PARTS, + MediaPart, + Occurrence, + call_owner, + capture_parts, + get, + items, + owned_stream, + owner_scope, +) +from wrapt import FunctionWrapper + +__all__ = ("capture_langchain_messages", "langchain_attachment_capabilities", "observes_langchain_input", + "own_chat_model_calls", "release_chat_model_calls") + +langchain_attachment_capabilities = MappingProxyType({ + "schema_version": 1, + "operations": ("chat_model.input", "chat_model.output", "tool_message.input"), + "sources": ("inline_base64", "data_uri"), + "unsupported": ("remote_url_fetch", "provider_file", "local_path", "streamed_output", "audio_output"), + "max_part_bytes": MAX_FRAMEWORK_BYTES, + "max_parts_per_span": MAX_FRAMEWORK_PARTS, +}) + +_ROLES = {"HumanMessage": "user", "AIMessage": "assistant", "SystemMessage": "system", "ToolMessage": "tool", + "HumanMessageChunk": "user", "AIMessageChunk": "assistant", "ToolMessageChunk": "tool"} +_AUDIO_FORMATS = {"wav": "audio/wav", "mp3": "audio/mpeg"} + + +def _family(mime: Any, default: tuple) -> Optional[tuple]: + """The admitted MIME list for a declared type, the declared type first; None when unsupported.""" + if mime is None: + return default + if not isinstance(mime, str): + return None + for family in (IMAGE_MIMES, DOCUMENT_MIMES, AUDIO_MIMES): + if mime in family: + return (mime,) + tuple(m for m in family if m != mime) + return None + + +def _purpose(mimes: tuple, direction: str, role: str) -> str: + if direction == "output": + return "model_output" + if role == "tool": + return "tool_result" + return "document" if mimes[0] == "application/pdf" else "model_input" + + +def _part(block: Any, message_index: int, part_index: int, role: str, direction: str, + tool_call_id: Optional[str]) -> Optional[MediaPart]: + kind = get(block, "type") + found = None # (source kind, reader, mimes, raw) + if kind == "image_url": + image = get(block, "image_url") + url = image if isinstance(image, str) else get(image, "url") + if isinstance(url, str) and url.startswith("data:"): + found = ("inline", lambda: url, IMAGE_MIMES, False) + elif isinstance(url, str): + found = ("url", lambda: url, IMAGE_MIMES, True) + elif kind == "input_audio": + audio = get(block, "input_audio") + mimes = _family(_AUDIO_FORMATS.get(get(audio, "format")), ()) + if mimes: + found = ("inline", lambda: get(audio, "data"), mimes, True) + elif kind == "file" and isinstance(get(block, "file"), Mapping): + data = get(get(block, "file"), "file_data") + if isinstance(data, str) and data.startswith("data:"): + found = ("inline", lambda: data, DOCUMENT_MIMES, False) + elif kind in ("image", "audio", "file"): + default = {"image": IMAGE_MIMES, "audio": AUDIO_MIMES, "file": DOCUMENT_MIMES}[kind] + source = get(block, "source") + if isinstance(source, Mapping): + # Anthropic-style block. + if get(source, "type") == "base64": + mimes = _family(get(source, "media_type"), default) + if mimes: + found = ("inline", lambda: get(source, "data"), mimes, True) + elif get(source, "type") == "url" and isinstance(get(source, "url"), str): + found = ("url", lambda: get(source, "url"), default, True) + elif get(block, "source_type") == "base64": + mimes = _family(get(block, "mime_type"), default) # LangChain 0.3 standard block + if mimes: + found = ("inline", lambda: get(block, "data"), mimes, True) + elif get(block, "source_type") == "url" and isinstance(get(block, "url"), str): + found = ("url", lambda: get(block, "url"), default, True) + elif get(block, "base64") is not None: + mimes = _family(get(block, "mime_type"), default) # LangChain 1.x content block + if mimes: + found = ("inline", lambda: get(block, "base64"), mimes, True) + elif isinstance(get(block, "url"), str): + found = ("url", lambda: get(block, "url"), default, True) + if found is None: + return None + source_kind, read, mimes, raw = found + return MediaPart(source_kind, read, mimes, _purpose(mimes, direction, role), message_index, part_index, + role=role, raw=raw, tool_call_id=tool_call_id) + + +def _message(message: Any) -> tuple: + """(role, content, tool_call_id) of a serialized (``dumpd``) or plain message.""" + kwargs = get(message, "kwargs") + if isinstance(kwargs, Mapping): + ident = get(message, "id") + name = ident[-1] if isinstance(ident, list) and ident and isinstance(ident[-1], str) else "" + role = _ROLES.get(name, "unknown") + if name.startswith("ChatMessage"): + native = get(kwargs, "role") + role = native if native in ("user", "assistant", "tool", "system") else "unknown" + return role, get(kwargs, "content"), get(kwargs, "tool_call_id") + role = get(message, "type") + role = {"human": "user", "ai": "assistant", "system": "system", "tool": "tool"}.get(role, "unknown") + return role, get(message, "content"), get(message, "tool_call_id") + + +def _parts(messages: List[Any], direction: str) -> Iterator[MediaPart]: + for message_index, message in enumerate(items(messages)): + role, content, tool_call_id = _message(message) + tool_call_id = tool_call_id if role == "tool" and isinstance(tool_call_id, str) else None + for part_index, block in enumerate(items(content)): + if isinstance(block, Mapping): + part = _part(block, message_index, part_index, role, direction, tool_call_id) + if part is not None: + yield part + + +def capture_langchain_messages(manager, span, messages: Any, *, direction: str, trace_config=None, + capture=None) -> Mapping[str, Occurrence]: + """Capture typed media of one chat-model run's first message batch or generations. + + ``messages`` is a list of serialized messages. Returns admitted positions keyed + ``"{direction}/{message_index}/{part_index}"``; empty when no manager, disabled, + unsampled or denied. Never raises and never changes ``messages``. + """ + try: + if not isinstance(messages, list): + return EMPTY + return capture_parts(manager, span, "langchain", _parts(messages, direction), direction=direction, + trace_configs=(trace_config,), capture=capture) + except Exception: + return EMPTY + + +def run_messages(run: Any, direction: str) -> Optional[list]: + """The first message batch (input) or the first generation's messages (output) of a run.""" + try: + if direction == "input": + batches = get(get(run, "inputs"), "messages") + first = batches[0] if isinstance(batches, list) and batches else None + return first if isinstance(first, list) else None + generations = get(get(run, "outputs"), "generations") + first = generations[0] if isinstance(generations, list) and generations else None + if not isinstance(first, list): + return None + return [message for message in (get(g, "message") for g in first[:MAX_FRAMEWORK_PARTS]) + if message is not None] + except Exception: + return None + + +# ------------------------------------------------------------------ one call, one owner +# BaseChatModel entry points that run one or more chat-model calls (invoke/batch go through +# generate/agenerate). The tracer captures inside them, so the mark carries this namespace. +_OWNED_METHODS = ("generate", "agenerate", "stream", "astream") +_NAMESPACE = "langchain" + + +def _input(name: str, instance: Any, args: Any, kwargs: Any) -> List[Any]: + """The chat messages one owned entry point sends: every batch of generate, or stream's input.""" + if name in ("generate", "agenerate"): + batches = args[0] if args else kwargs.get("messages") + return [message for batch in items(batches) for message in items(batch)] + value = args[0] if args else kwargs.get("input") + if isinstance(value, str): + return [] + # LangChain's own conversion (PromptValue, message sequences, dicts, tuples). + return list(instance._convert_input(value).to_messages()) + + +def observes_langchain_input(name: str, instance: Any, args: Any, kwargs: Any) -> bool: + """True when this module observes at least one input media part of the call (owner-mark gate). + + Never raises; an error means False (no mark), so the provider is never suppressed by mistake. + """ + try: + return next(_parts(_input(name, instance, args, kwargs), "input"), None) is not None + except Exception: + return False + + +def _owner(name: str, manager, instance: Any, args: Any, kwargs: Any): + if not observes_langchain_input(name, instance, args, kwargs): + return None + return call_owner(manager, _NAMESPACE) + + +def _owned(manager, name: str): + def wrapper(wrapped, instance, args, kwargs): + owner = _owner(name, manager, instance, args, kwargs) + if owner is None: + return wrapped(*args, **kwargs) + with owner_scope(owner): + result = wrapped(*args, **kwargs) + # stream/astream return a lazy generator: mark each advance, never across a yield. + return owned_stream(result, owner) + return wrapper + + +def _owned_async(manager, name: str): + async def wrapper(wrapped, instance, args, kwargs): + owner = _owner(name, manager, instance, args, kwargs) + if owner is None: + return await wrapped(*args, **kwargs) + with owner_scope(owner): + return await wrapped(*args, **kwargs) + return wrapper + + +def own_chat_model_calls(manager) -> List[Tuple[type, str, Any]]: + """Run every chat-model call whose input media is observed here under the owner mark. + + Only while ``manager`` can admit, and only for a call with at least one observed input + part (``observes_langchain_input``). + + A provider instrumentor beneath LangChain then uploads nothing for that call; the + LangChain span owns its media. Returns the patches for ``release_chat_model_calls``. + """ + from langchain_core.language_models.chat_models import BaseChatModel + + patches = [] + for name in _OWNED_METHODS: + original = BaseChatModel.__dict__.get(name) + if original is None: + continue + wrapper = _owned_async(manager, name) if name == "agenerate" else _owned(manager, name) + setattr(BaseChatModel, name, FunctionWrapper(original, wrapper)) + patches.append((BaseChatModel, name, original)) + return patches + + +def release_chat_model_calls(patches: List[Tuple[type, str, Any]]) -> None: + for cls, name, original in reversed(patches): + setattr(cls, name, original) diff --git a/python/frameworks/langchain/traceai_langchain/_tracer.py b/python/frameworks/langchain/traceai_langchain/_tracer.py index e7d4a7781..d3c99ae1a 100644 --- a/python/frameworks/langchain/traceai_langchain/_tracer.py +++ b/python/frameworks/langchain/traceai_langchain/_tracer.py @@ -178,8 +178,18 @@ def _send_span_data_to_api( except Exception as e: logger.exception(f"Error sending span data to API: {e}") - def __init__(self, fi_tracer: "FITracer", *args: Any, **kwargs: Any) -> None: + def __init__( + self, + fi_tracer: "FITracer", + *args: Any, + attachments: Any = None, + trace_config: Optional[TraceConfig] = None, + **kwargs: Any, + ) -> None: super().__init__(*args, **kwargs) + # The shared, provider-owned attachment manager (already checked active), or None. + self._attachments = attachments + self._attachment_config = trace_config if TYPE_CHECKING: # check that `run_map` still exists in parent class assert self.run_map @@ -239,6 +249,9 @@ def _start_trace(self, run: Run) -> None: # token = context_api.attach(context) with self._lock: self._spans_by_run[run.id] = span + # Chat-model input media is captured BEFORE the provider call, so a provider + # instrumentor sharing the manager reuses this upload instead of a second one. + self._capture_media(span, run, "input") # Send initial span data with input input_data = None @@ -279,6 +292,9 @@ def _end_trace(self, run: Run) -> None: is_interrupt=is_interrupt, error=error, ) + # Never from a failed run: its outputs can carry provider error bodies. + if run.error is None: + self._capture_media(span, run, "output") # Send final span data with both input and output output_data = None @@ -306,6 +322,22 @@ def _end_trace(self, run: Run) -> None: def _persist_run(self, run: Run) -> None: pass + def _capture_media(self, span: Span, run: Run, direction: str) -> None: + if self._attachments is None or run.run_type != "llm": + return + try: + from traceai_langchain._attachments import capture_langchain_messages, run_messages + + messages = run_messages(run, direction) + if messages: + capture_langchain_messages( + self._attachments, span, messages, direction=direction, + trace_config=self._attachment_config, + ) + except Exception: + # Media observation must never affect the run or log source-bearing data. + pass + def _handle_error( self, error: BaseException, diff --git a/python/frameworks/llama_index/traceai_llamaindex/__init__.py b/python/frameworks/llama_index/traceai_llamaindex/__init__.py index 1ca5eb3f7..b5af16e25 100644 --- a/python/frameworks/llama_index/traceai_llamaindex/__init__.py +++ b/python/frameworks/llama_index/traceai_llamaindex/__init__.py @@ -20,6 +20,23 @@ logger.addHandler(logging.NullHandler()) +def _attachment_manager(kwargs: Any) -> Any: + """The shared manager passed as ``attachments`` (or auto's ``attachment_manager``), if it can admit. + + A missing, disabled or pre-attachments core leaves tracing exactly as before. + """ + manager = kwargs.get("attachments") + if manager is None: + manager = kwargs.get("attachment_manager") + if manager is None: + return None + try: + from fi_instrumentation.attachments.media_capture import active + except ImportError: + return None + return manager if active(manager) else None + + class LlamaIndexInstrumentor(BaseInstrumentor): # type: ignore """ An instrumentor for LlamaIndex @@ -70,6 +87,7 @@ def _instrument(self, **kwargs: Any) -> None: self._event_handler = None self._workflow_run_patches = [] self._stream_instrumentation = None + self._chat_ownership = None if self._use_legacy_callback_handler: import llama_index.core @@ -83,7 +101,10 @@ def _instrument(self, **kwargs: Any) -> None: from ._handler import EventHandler, instrument_workflow_runs from ._stream import StreamInstrumentation - self._event_handler = EventHandler(tracer=tracer, config=config) + attachments = _attachment_manager(kwargs) + self._event_handler = EventHandler( + tracer=tracer, config=config, attachments=attachments + ) self._span_handler = self._event_handler._span_handler dispatcher = get_dispatcher() for span_handler in dispatcher.span_handlers: @@ -99,6 +120,12 @@ def _instrument(self, **kwargs: Any) -> None: self._workflow_run_patches = instrument_workflow_runs() self._stream_instrumentation = StreamInstrumentation() self._stream_instrumentation.install() + if attachments is not None: + # One model call, one owner: only while attachments are enabled. + from ._attachments import ChatOwnership + + self._chat_ownership = ChatOwnership(attachments) + self._chat_ownership.install() if Protect is not None: self._original_protect = Protect.protect @@ -123,6 +150,9 @@ def _uninstrument(self, **kwargs: Any) -> None: from ._handler import uninstrument_workflow_runs + if getattr(self, "_chat_ownership", None) is not None: + self._chat_ownership.uninstall() + self._chat_ownership = None if self._stream_instrumentation is not None: self._stream_instrumentation.uninstall() self._stream_instrumentation = None diff --git a/python/frameworks/llama_index/traceai_llamaindex/_attachments.py b/python/frameworks/llama_index/traceai_llamaindex/_attachments.py new file mode 100644 index 000000000..421906f3a --- /dev/null +++ b/python/frameworks/llama_index/traceai_llamaindex/_attachments.py @@ -0,0 +1,268 @@ +"""LlamaIndex typed message media through the shared, provider-owned attachment manager. + +Mirrors ``@traceai/framework-attachments`` (dialect ``llamaindex``). Only +``ChatMessage.blocks`` (llama-index-core 0.12.3+) of chat-model input and of the +non-streaming chat response are observed: + +- ``ImageBlock.image``, ``AudioBlock.audio`` and ``DocumentBlock.data`` held as + bytes (LlamaIndex stores them base64-encoded; raw bytes are accepted too); +- the magic bytes must match the admitted type. + +An ``IOBase`` source is never read, a ``path`` is never opened, and a ``url`` goes +to the core's ``RemoteUrl`` policy without being fetched. + +One model call, one owner: while attachments are enabled, an LLM ``chat`` / +``achat`` / ``stream_chat`` / ``astream_chat`` call (the calls whose messages are +observed here) in which this module observes at least one input media part runs +under the owner mark (``media_capture.owned_call``; a stream only while it +advances), so a provider instrumentor beneath LlamaIndex (``traceai_openai``, ...) +uploads nothing for that call; the LlamaIndex span holds the occurrence. A call +with no observed part is not marked: the OpenAI integration inlines a ``path`` +image or document as a data URI before the SDK call, so the provider span captures +it. Residual: a call that mixes an observed part with a ``path`` (or ``IOBase``) +one is marked, so the provider skips that part too. Completion +calls are not observed here and are not marked. A DIFFERENT call with the same bytes +gets its own occurrence (byte deduplication is server-side). + +This module imports ``fi_instrumentation.attachments``; the handler imports it only +when an enabled attachment manager is passed. +""" + +from threading import RLock +from types import MappingProxyType +from typing import Any, Dict, Iterator, Mapping, Optional, Sequence, Tuple + +from fi_instrumentation.attachments.media_capture import ( + AUDIO_MIMES, + DOCUMENT_MIMES, + EMPTY, + IMAGE_MIMES, + MAX_FRAMEWORK_BYTES, + MAX_FRAMEWORK_PARTS, + MediaPart, + Occurrence, + call_owner, + capture_parts, + looks_base64, + owned_stream, + owner_scope, +) +from wrapt import FunctionWrapper + +__all__ = ("ChatOwnership", "capture_llamaindex_messages", "llamaindex_attachment_capabilities", + "observes_llamaindex_messages") + +llamaindex_attachment_capabilities = MappingProxyType({ + "schema_version": 1, + "operations": ("llm.chat.input", "llm.chat.output"), + "sources": ("block_bytes",), + "unsupported": ("io_source", "local_path", "remote_url_fetch", "video"), + "min_core": "0.12.3", + "max_part_bytes": MAX_FRAMEWORK_BYTES, + "max_parts_per_span": MAX_FRAMEWORK_PARTS, +}) + +_ROLES = ("user", "assistant", "tool", "system") +_AUDIO_FORMATS = {"wav": "audio/wav", "mp3": "audio/mpeg", "ogg": "audio/ogg", "flac": "audio/flac", + "webm": "audio/webm"} +# block_type -> (data field, MIME field, default family) +_BLOCKS = {"image": ("image", "image_mimetype", IMAGE_MIMES), "audio": ("audio", None, AUDIO_MIMES), + "document": ("data", "document_mimetype", DOCUMENT_MIMES)} + + +def _field(block: Any, name: str) -> Any: + """A declared model field only: never a property, resolver or ``__getattr__``.""" + try: + values = vars(block) + except TypeError: + return None + return values.get(name) + + +def _ordered(mime: Any, family: tuple) -> Optional[tuple]: + if mime is None: + return family + if isinstance(mime, str) and mime in family: + return (mime,) + tuple(m for m in family if m != mime) + return None + + +def _role(message: Any) -> str: + role = _field(message, "role") + role = getattr(role, "value", role) + return role if role in _ROLES else "unknown" + + +def _parts(messages: Sequence[Any], direction: str) -> Iterator[MediaPart]: + for message_index, message in enumerate(list(messages)[:MAX_FRAMEWORK_PARTS]): + role = _role(message) + blocks = _field(message, "blocks") + if not isinstance(blocks, list): + continue + for part_index, block in enumerate(blocks[:MAX_FRAMEWORK_PARTS]): + spec = _BLOCKS.get(_field(block, "block_type")) + if spec is None: + continue + data_field, mime_field, family = spec + if mime_field is not None: + mimes = _ordered(_field(block, mime_field), family) + else: + fmt = _field(block, "format") + mimes = _ordered(_AUDIO_FORMATS.get(fmt), family) if fmt is not None else family + if not mimes: + continue + purpose = ("model_output" if direction == "output" else "tool_result" if role == "tool" + else "document" if family is DOCUMENT_MIMES else "model_input") + data = _field(block, data_field) + url = _field(block, "url") + if isinstance(data, (bytes, bytearray)): + # Base64 ASCII when it parses strictly; otherwise complete raw bytes. + kind = "inline" if looks_base64(data) else "bytes" + yield MediaPart(kind, lambda d=data: d, mimes, purpose, message_index, part_index, role=role, + raw=True) + elif data is None and url is not None and _field(block, "path") is None: + yield MediaPart("url", lambda u=url: str(u), mimes, purpose, message_index, part_index, + role=role) + # IOBase sources and paths: never read or opened. + + +def observes_llamaindex_messages(messages: Any) -> bool: + """True when this module observes at least one input media part (owner-mark gate). Never raises. + + ``path`` and ``IOBase`` sources are never observed, so a call holding only those is + left to the provider instrumentor. An error means False (no mark): the provider is + never suppressed by mistake. + """ + try: + if not isinstance(messages, (list, tuple)): + return False + return next(_parts(messages, "input"), None) is not None + except Exception: + return False + + +def capture_llamaindex_messages(manager, span, messages: Any, *, direction: str, trace_config=None, + capture=None) -> Mapping[str, Occurrence]: + """Capture typed block media of chat messages; returns ``"{dir}/{msg}/{part}"`` positions. + + Never raises and never changes ``messages``. + """ + try: + if not isinstance(messages, (list, tuple)): + return EMPTY + return capture_parts(manager, span, "llamaindex", _parts(messages, direction), direction=direction, + trace_configs=(trace_config,), capture=capture) + except Exception: + return EMPTY + + +# ------------------------------------------------------------------ one call, one owner +# The LLM methods whose messages this module observes (LLMChatStart/End events). Completion +# methods are not observed, so they are never marked. +_SYNC_METHODS = ("chat", "stream_chat") +_ASYNC_METHODS = ("achat", "astream_chat") +_NAMESPACE = "llamaindex" +_MISSING = object() + + +class ChatOwnership: + """Run every LLM chat call whose input media is observed here under the owner mark. + + Only while ``manager`` can admit, and only for a call with at least one observed + input part (``observes_llamaindex_messages``). + + Patched like ``StreamInstrumentation``: every loaded ``BaseLLM`` / ``MultiModalLLM`` + subclass that defines a chat method, plus an ``__init__`` hook for providers imported + later. A returned stream is marked only while it advances. Install after and uninstall + before ``StreamInstrumentation``. + """ + + def __init__(self, manager: Any) -> None: + self._manager = manager + self._patches: Dict[Tuple[type, str], Any] = {} + self._lock = RLock() + self._enabled = True + + def install(self) -> None: + from llama_index.core.base.llms.base import BaseLLM + from llama_index.core.multi_modal_llms import MultiModalLLM + + seen = set() + + def visit(cls: type) -> None: + if cls in seen: + return + seen.add(cls) + self._patch_class(cls) + for child in cls.__subclasses__(): + visit(child) + + for base in (BaseLLM, MultiModalLLM): + visit(base) + self._patch(base, "__init__", self._initialize) + + def _patch_class(self, cls: type) -> None: + for base in reversed(cls.__mro__): + for name in _SYNC_METHODS + _ASYNC_METHODS: + attr = base.__dict__.get(name) + if attr is not None and not getattr(attr, "__isabstractmethod__", False): + self._patch(base, name, self._call_async if name in _ASYNC_METHODS else self._call) + + def _patch(self, cls: type, name: str, wrapper: Any) -> None: + with self._lock: + if not self._enabled or (cls, name) in self._patches: + return + attr = getattr(cls, name) + self._patches[cls, name] = cls.__dict__.get(name, _MISSING) + setattr(cls, name, FunctionWrapper(attr, wrapper)) + + def _initialize(self, wrapped: Any, instance: Any, args: Any, kwargs: Any) -> Any: + # After the inner hooks, so a later provider class is patched in install order too. + result = wrapped(*args, **kwargs) + if self._enabled: + self._patch_class(type(instance)) + return result + + def _owner(self, args: Any, kwargs: Any): + """The mark for a call whose input media this module observes, else None (no mark).""" + if not self._enabled: + return None + try: + messages = args[0] if args else kwargs.get("messages") + except Exception: + return None + if not observes_llamaindex_messages(messages): + return None + return call_owner(self._manager, _NAMESPACE) + + def _call(self, wrapped: Any, instance: Any, args: Any, kwargs: Any) -> Any: + owner = self._owner(args, kwargs) + if owner is None: + return wrapped(*args, **kwargs) + with owner_scope(owner): + result = wrapped(*args, **kwargs) + return owned_stream(result, owner) + + async def _call_async(self, wrapped: Any, instance: Any, args: Any, kwargs: Any) -> Any: + owner = self._owner(args, kwargs) + if owner is None: + return await wrapped(*args, **kwargs) + with owner_scope(owner): + result = await wrapped(*args, **kwargs) + return owned_stream(result, owner) + + def uninstall(self) -> None: + self._enabled = False + with self._lock: + for (cls, name), original in reversed(list(self._patches.items())): + attr = cls.__dict__.get(name) + if ( + not isinstance(attr, FunctionWrapper) + or getattr(attr._self_wrapper, "__self__", None) is not self + ): + continue # Do not overwrite another instrumentor's patch. + if original is _MISSING: + delattr(cls, name) + else: + setattr(cls, name, original) + self._patches.clear() diff --git a/python/frameworks/llama_index/traceai_llamaindex/_handler.py b/python/frameworks/llama_index/traceai_llamaindex/_handler.py index 24e624a7e..2748145f2 100644 --- a/python/frameworks/llama_index/traceai_llamaindex/_handler.py +++ b/python/frameworks/llama_index/traceai_llamaindex/_handler.py @@ -241,6 +241,7 @@ class Config: _cancel_reason: Optional[str] = PrivateAttr() _workflow_run: bool = PrivateAttr() _run_handler: Any = PrivateAttr() + _attachments: Any = PrivateAttr() def __init__( self, @@ -248,9 +249,12 @@ def __init__( config: Optional[TraceConfig] = None, span_kind: Optional[str] = None, parent: Optional["_Span"] = None, + attachments: Any = None, **kwargs: Any, ) -> None: super().__init__(**kwargs) + # The shared, provider-owned attachment manager (already checked active), or None. + self._attachments = attachments self._config = config if config is not None else TraceConfig() self._otel_span = otel_span self._active = otel_span.is_recording() @@ -650,6 +654,8 @@ def _(self, event: LLMChatStartEvent) -> None: GEN_AI_INPUT_MESSAGES, *event.messages, ) + # Input media is captured before the provider call; this span owns its own occurrences. + self._capture_media(event.messages, "input") self[INPUT_VALUE] = safe_json_dumps(event.messages) self[INPUT_VALUE] = safe_json_dumps(_to_dict(event)) @@ -667,6 +673,7 @@ def _(self, event: LLMChatEndEvent) -> None: GEN_AI_OUTPUT_MESSAGES, response.message, ) + self._capture_media([response.message], "output") self[OUTPUT_VALUE] = safe_json_dumps( { "message": _to_dict(response.message), @@ -823,6 +830,20 @@ def _process_messages( f"{prefix}.{i}.{MESSAGE_CONTENTS}.{index}.{MESSAGE_CONTENT_AUDIO}" ] = (block.audio or block.path or block.url) + def _capture_media(self, messages: Any, direction: str) -> None: + if self._attachments is None or not self._active: + return + try: + from ._attachments import capture_llamaindex_messages + + capture_llamaindex_messages( + self._attachments, self._otel_span, list(messages or ()), + direction=direction, trace_config=self._config, + ) + except Exception: + # Media observation must never affect the call or log source-bearing data. + pass + def _process_query_type(self, query: Optional[QueryType]) -> None: if query is None: return @@ -956,9 +977,13 @@ class _SpanHandler(BaseSpanHandler[_Span], extra="allow"): _otel_tracer: Tracer = PrivateAttr() _config: TraceConfig = PrivateAttr() _export_queue: _ExportQueue = PrivateAttr() + _attachments: Any = PrivateAttr() - def __init__(self, tracer: Tracer, config: Optional[TraceConfig] = None) -> None: + def __init__( + self, tracer: Tracer, config: Optional[TraceConfig] = None, attachments: Any = None + ) -> None: super().__init__() + self._attachments = attachments self._config = config if config is not None else TraceConfig() self._otel_tracer = tracer self._export_queue = _ExportQueue() @@ -994,6 +1019,7 @@ def new_span( config=self._config, span_kind=_init_span_kind(instance), parent=parent, + attachments=self._attachments, id_=id_, parent_id=parent_span_id, ) @@ -1234,9 +1260,11 @@ def uninstrument_workflow_runs(patched: Iterable[Tuple[type, str]]) -> None: class EventHandler(BaseEventHandler, extra="allow"): _span_handler: _SpanHandler = PrivateAttr() - def __init__(self, tracer: Tracer, config: Optional[TraceConfig] = None) -> None: + def __init__( + self, tracer: Tracer, config: Optional[TraceConfig] = None, attachments: Any = None + ) -> None: super().__init__() - self._span_handler = _SpanHandler(tracer=tracer, config=config) + self._span_handler = _SpanHandler(tracer=tracer, config=config, attachments=attachments) def handle(self, event: BaseEvent, **kwargs: Any) -> Any: if context_api.get_value(_SUPPRESS_INSTRUMENTATION_KEY): diff --git a/python/frameworks/pydantic-ai/traceai_pydantic_ai/_attachments.py b/python/frameworks/pydantic-ai/traceai_pydantic_ai/_attachments.py new file mode 100644 index 000000000..53c461538 --- /dev/null +++ b/python/frameworks/pydantic-ai/traceai_pydantic_ai/_attachments.py @@ -0,0 +1,188 @@ +"""Pydantic AI typed media through the shared, provider-owned attachment manager. + +Observed at SDK execution boundaries only: + +- model request span, input: ``BinaryContent`` in ``UserPromptPart.content``, + in ``ToolReturnPart.content`` (with ``tool_call_id``) and in re-sent + ``FilePart`` history; ``ImageUrl`` / ``AudioUrl`` / ``DocumentUrl`` go to the + core's ``RemoteUrl`` policy and are never fetched or resolved here; +- model request span, output: ``FilePart`` content of a successful, + non-streaming response (never an error body); +- tool call span: ``BinaryContent`` a tool returned. It is the tool span's + OUTPUT and the next request's model INPUT, so the local input AND output + hide flags both apply (deny-wins, as in the TypeScript MCP binding); it is + uploaded as model input (``direction="input"``, ``purpose="tool_result"``). + +``BinaryContent.data`` must be complete bytes whose magic bytes match its +declared ``media_type``. Videos, uploaded provider files and unknown content +objects are not captured, and nothing is serialized to look for media. + +One model call, one owner: while attachments are enabled, a model request in which +this adapter observes at least one input media part runs under the owner mark +(``media_capture.owned_call``; a streamed request only while the stream is entered +and exited, never the caller's ``async with`` body), so a provider instrumentor +beneath Pydantic AI (``traceai_openai``, ...) uploads nothing for that call; the +model request span holds the occurrence. A request with no observed part is not +marked, so the provider span keeps what only the SDK sees. Residual: a request that +mixes an observed part with one this adapter never reads (an uploaded provider +file, ...) is marked, so the provider skips that part too. DIFFERENT calls each +get their own occurrence (a tool span's result and the next request that re-sends +it are two observations; byte deduplication is server-side). +""" + +import dataclasses +from types import MappingProxyType +from typing import Any, Iterator, Mapping, Optional + +from fi_instrumentation.attachments.media_capture import ( + AUDIO_MIMES, + DOCUMENT_MIMES, + EMPTY, + IMAGE_MIMES, + MAX_FRAMEWORK_BYTES, + MAX_FRAMEWORK_PARTS, + MediaPart, + Occurrence, + capture_parts, +) + +__all__ = ("PydanticAIMedia", "pydantic_ai_attachment_capabilities") + +pydantic_ai_attachment_capabilities = MappingProxyType({ + "schema_version": 1, + "operations": ("model.request.input", "model.request.output", "tool.call.result", + "model.request_stream.input"), + "sources": ("binary_content",), + "unsupported": ("remote_url_fetch", "uploaded_file", "video", "streamed_output", "error_body"), + "max_part_bytes": MAX_FRAMEWORK_BYTES, + "max_parts_per_span": MAX_FRAMEWORK_PARTS, +}) + +_URL_FAMILIES = {"image-url": IMAGE_MIMES, "audio-url": AUDIO_MIMES, "document-url": DOCUMENT_MIMES} + + +def _field(value: Any, name: str) -> Any: + """A declared field only: never a property, resolver or ``__getattr__``.""" + try: + return vars(value).get(name) + except TypeError: + pass + try: + if dataclasses.is_dataclass(value) and not isinstance(value, type) and \ + any(f.name == name for f in dataclasses.fields(value)): + return getattr(value, name) + except Exception: + pass + return None + + +def _family(mime: Any) -> Optional[tuple]: + for family in (IMAGE_MIMES, AUDIO_MIMES, DOCUMENT_MIMES): + if mime in family: + return (mime,) + tuple(m for m in family if m != mime) + return None + + +def _content_parts(content: Any, message_index: int, start: int, role: str, purpose: str, + tool_call_id: Optional[str]) -> Iterator[MediaPart]: + values = content if isinstance(content, (list, tuple)) else [content] + index = start + for value in values[:MAX_FRAMEWORK_PARTS]: + kind = _field(value, "kind") + if kind == "binary": + data = _field(value, "data") + mimes = _family(_field(value, "media_type")) + if mimes and isinstance(data, (bytes, bytearray)): + part_purpose = "document" if purpose == "model_input" and mimes[0] in DOCUMENT_MIMES else purpose + yield MediaPart("bytes", lambda d=data: d, mimes, part_purpose, message_index, index, role=role, + tool_call_id=tool_call_id) + index += 1 + elif kind in _URL_FAMILIES: + url = _field(value, "url") + if isinstance(url, str): + yield MediaPart("url", lambda u=url: u, _URL_FAMILIES[kind], purpose, message_index, index, + role=role, tool_call_id=tool_call_id) + index += 1 + elif kind is not None: + index += 1 # video-url, uploaded files, ...: never captured + + +def _request_parts(messages: Any) -> Iterator[MediaPart]: + for message_index, message in enumerate(list(messages)[:MAX_FRAMEWORK_PARTS]): + index = 0 + for part in list(_field(message, "parts") or ())[:MAX_FRAMEWORK_PARTS]: + part_kind = _field(part, "part_kind") + if part_kind == "user-prompt": + content = _field(part, "content") + if isinstance(content, str): + continue + parts = list(_content_parts(content, message_index, index, "user", "model_input", None)) + elif part_kind == "tool-return": + tool_call_id = _field(part, "tool_call_id") + parts = list(_content_parts(_field(part, "content"), message_index, index, "tool", "tool_result", + tool_call_id if isinstance(tool_call_id, str) else None)) + elif part_kind == "file": + # A generated file re-sent as history is model input now. + parts = list(_content_parts(_field(part, "content"), message_index, index, "assistant", + "model_input", None)) + else: + continue + yield from parts + index += max(len(parts), 1) + + +def _response_parts(response: Any) -> Iterator[MediaPart]: + index = 0 + for part in list(_field(response, "parts") or ())[:MAX_FRAMEWORK_PARTS]: + if _field(part, "part_kind") == "file": + yield from _content_parts(_field(part, "content"), 0, index, "assistant", "model_output", None) + index += 1 + + +class PydanticAIMedia: + """The instrumentor's binding: manager plus the caller's TraceConfig (deny-only).""" + + __slots__ = ("manager", "trace_config") + + def __init__(self, manager: Any, trace_config: Any) -> None: + self.manager, self.trace_config = manager, trace_config + + def _capture(self, span, parts, direction: str, policy_directions=()) -> Mapping[str, Occurrence]: + if span is None: + return EMPTY + try: + return capture_parts(self.manager, span, "pydantic_ai", parts, direction=direction, + trace_configs=(self.trace_config,), policy_directions=policy_directions) + except Exception: + return EMPTY + + def request(self, span, messages: Any) -> Mapping[str, Occurrence]: + if not isinstance(messages, (list, tuple)): + return EMPTY + return self._capture(span, _request_parts(messages), "input") + + def observes_request(self, messages: Any) -> bool: + """True when this adapter observes at least one input media part of the request. + + Only then does the request run under the owner mark. A part this adapter never reads + (a local path, a provider file, ...) is left to the provider instrumentor. Never raises; + an error means False (no mark), so the provider is never suppressed by mistake. + """ + try: + if not isinstance(messages, (list, tuple)): + return False + return next(_request_parts(messages), None) is not None + except Exception: + return False + + def response(self, span, response: Any) -> Mapping[str, Occurrence]: + # Historical Model.request returns (ModelResponse, Usage). + response = response[0] if isinstance(response, tuple) and response else response + return self._capture(span, _response_parts(response), "output") + + def tool_result(self, span, result: Any, tool_call_id: Any) -> Mapping[str, Occurrence]: + tool_call_id = tool_call_id if isinstance(tool_call_id, str) else None + # The tool span redacts this value under the OUTPUT hide flags (TOOL_RESULT -> OUTPUT_VALUE), + # and it is a later model input: fold both directions so either side's hide flag denies. + return self._capture(span, _content_parts(result, 0, 0, "tool", "tool_result", tool_call_id), "input", + policy_directions=("input", "output")) diff --git a/python/frameworks/pydantic-ai/traceai_pydantic_ai/_execution_wrappers.py b/python/frameworks/pydantic-ai/traceai_pydantic_ai/_execution_wrappers.py index e39bdded9..501a54d30 100644 --- a/python/frameworks/pydantic-ai/traceai_pydantic_ai/_execution_wrappers.py +++ b/python/frameworks/pydantic-ai/traceai_pydantic_ai/_execution_wrappers.py @@ -99,23 +99,26 @@ class _ModelStreamContext: The span may outlive either task, but a ContextVar token may not cross them. """ - def __init__(self, manager, model, span): + def __init__(self, manager, model, span, owner=None): self.manager = manager self.model = model self.span = span + # The attachment owner mark (one call, one owner): entered and exited, never the caller's body. + self.owner = owner @contextmanager def _current(self): - if self.span is None: - yield - return - token = _ACTIVE_MODEL_REQUEST.set((self.model, _ACTIVE_USAGE.get())) - try: - with use_span(self.span, end_on_exit=False, record_exception=False, - set_status_on_exception=False): + with _owner_scope(self.owner): + if self.span is None: yield - finally: - _ACTIVE_MODEL_REQUEST.reset(token) + return + token = _ACTIVE_MODEL_REQUEST.set((self.model, _ACTIVE_USAGE.get())) + try: + with use_span(self.span, end_on_exit=False, record_exception=False, + set_status_on_exception=False): + yield + finally: + _ACTIVE_MODEL_REQUEST.reset(token) async def __aenter__(self): with self._current(): @@ -126,14 +129,64 @@ async def __aexit__(self, exc_type, exc_value, traceback): return await self.manager.__aexit__(exc_type, exc_value, traceback) -def wrap_model_request(original, tracer, streaming=False): +def _call_owner(media, messages): + """The owner mark for one model request whose input media this adapter observes, else None. + + None while attachments are off, and None for a request with no part this adapter + observes, so the provider instrumentor beneath still captures what only it can see. + """ + if media is None: + return None + try: + if not media.observes_request(messages): + return None + from fi_instrumentation.attachments.media_capture import call_owner + + return call_owner(media.manager) + except Exception: + return None + + +@contextmanager +def _owner_scope(owner): + """Run the enclosed provider SDK call as the owner of its media, so the provider + instrumentor beneath Pydantic AI skips it. No mark: the context is unchanged.""" + if owner is None: + yield + return + from fi_instrumentation.attachments.media_capture import owner_scope + + with owner_scope(owner): + yield + + +def _request_messages(args, kwargs): + return args[0] if args else kwargs.get("messages") + + +def _capture(media, method, span, *values): + # Media observation must never affect the call or log source-bearing data. + if media is None or span is None: + return + try: + getattr(media, method)(span, *values) + except Exception: + pass + + +def wrap_model_request(original, tracer, streaming=False, media=None): + """``media`` is the instrumentor's attachment binding (``PydanticAIMedia``) or None.""" if streaming: @functools.wraps(original) @asynccontextmanager async def stream_wrapper(self, *args, **kwargs): with _model_request_span(self, tracer, current=False) as span: - manager = original(self, *args, **kwargs) - async with _ModelStreamContext(manager, self, span) as response: + # Input only: streamed output is not captured. + _capture(media, "request", span, _request_messages(args, kwargs)) + owner = _call_owner(media, _request_messages(args, kwargs)) + with _owner_scope(owner): + manager = original(self, *args, **kwargs) + async with _ModelStreamContext(manager, self, span, owner) as response: yield response _record_model_usage(span, response) return stream_wrapper @@ -141,13 +194,18 @@ async def stream_wrapper(self, *args, **kwargs): @functools.wraps(original) async def wrapper(self, *args, **kwargs): with _model_request_span(self, tracer) as span: - response = await original(self, *args, **kwargs) + # Input media is captured before the provider call; this span owns its own occurrences. + _capture(media, "request", span, _request_messages(args, kwargs)) + with _owner_scope(_call_owner(media, _request_messages(args, kwargs))): + response = await original(self, *args, **kwargs) _record_model_usage(span, response) + # Only a successful response: an error never reaches this line. + _capture(media, "response", span, response) return response return wrapper -def wrap_tool_call(original, tracer, legacy=False): +def wrap_tool_call(original, tracer, legacy=False, media=None): signature = inspect.signature(original) @functools.wraps(original) @@ -162,6 +220,7 @@ async def wrapper(self, *args, **kwargs): description = getattr(self, "description", None) arguments = getattr(bound.get("message"), "args", None) retry = getattr(self, "current_retry", None) + tool_call_id = getattr(bound.get("message"), "tool_call_id", None) else: name = bound.get("name", "unknown") tool = bound.get("tool") @@ -169,6 +228,7 @@ async def wrapper(self, *args, **kwargs): description = getattr(definition, "description", None) arguments = bound.get("tool_args") retry = getattr(bound.get("ctx"), "retry", None) + tool_call_id = getattr(bound.get("ctx"), "tool_call_id", None) attributes = { A.SPAN_KIND: PydanticAISpanKind.TOOL_CALL.value, A.GEN_AI_TOOL_NAME: name, @@ -207,6 +267,7 @@ async def wrapper(self, *args, **kwargs): span.set_status(Status(StatusCode.ERROR, message)) else: span.set_status(Status(StatusCode.OK)) + _capture(media, "tool_result", span, content, tool_call_id) return result finally: span.set_attribute(A.TOOL_DURATION_MS, (time.perf_counter() - start) * 1000) diff --git a/python/frameworks/pydantic-ai/traceai_pydantic_ai/_instrumentor.py b/python/frameworks/pydantic-ai/traceai_pydantic_ai/_instrumentor.py index 5be76fb43..6facf1a1f 100644 --- a/python/frameworks/pydantic-ai/traceai_pydantic_ai/_instrumentor.py +++ b/python/frameworks/pydantic-ai/traceai_pydantic_ai/_instrumentor.py @@ -22,6 +22,27 @@ _instruments = ("pydantic-ai >= 0.0.1",) +def _attachment_binding(kwargs, config): + """The shared manager passed as ``attachments`` (or auto's ``attachment_manager``), if it can admit. + + A missing, disabled or pre-attachments core leaves tracing exactly as before. + """ + manager = kwargs.get("attachments") + if manager is None: + manager = kwargs.get("attachment_manager") + if manager is None: + return None + try: + from fi_instrumentation.attachments.media_capture import active + except ImportError: + return None + if not active(manager): + return None + from ._attachments import PydanticAIMedia + + return PydanticAIMedia(manager, config) + + class PydanticAIInstrumentor(BaseInstrumentor): """OpenTelemetry Instrumentor for Pydantic AI. @@ -67,6 +88,7 @@ def __init__(self): self._execution_patches = [] self._legacy_usage_original = None self._use_builtin: bool = False + self._media = None def instrumentation_dependencies(self) -> Collection[str]: """Return the instrumentation dependencies.""" @@ -105,6 +127,7 @@ def _instrument(self, **kwargs: Any) -> None: # Retain the option for compatibility, but never bypass FI's policy. self._use_builtin = kwargs.get("use_builtin", False) + self._media = _attachment_binding(kwargs, config) # Try to import and patch Pydantic AI try: @@ -198,7 +221,8 @@ def patch_model(cls): if callable(original) and not getattr(original, "__isabstractmethod__", False): self._install_execution_patch( cls, name, wrap_model_request( - original, self._tracer, streaming=name == "request_stream")) + original, self._tracer, streaming=name == "request_stream", + media=self._media)) for subclass in cls.__subclasses__(): patch_model(subclass) @@ -223,11 +247,12 @@ def init_subclass(cls, **kwargs): from pydantic_ai.tools import Tool if isinstance(Tool, type) and hasattr(Tool, "run"): self._install_execution_patch( - Tool, "run", wrap_tool_call(Tool.run, self._tracer, legacy=True)) + Tool, "run", wrap_tool_call(Tool.run, self._tracer, legacy=True, media=self._media)) else: if isinstance(FunctionToolset, type): self._install_execution_patch( - FunctionToolset, "call_tool", wrap_tool_call(FunctionToolset.call_tool, self._tracer)) + FunctionToolset, "call_tool", + wrap_tool_call(FunctionToolset.call_tool, self._tracer, media=self._media)) def _install_execution_patch(self, cls, name, replacement): original = cls.__dict__.get(name) @@ -394,6 +419,7 @@ def _uninstrument(self, **kwargs: Any) -> None: self._restore_execution_methods() self._is_instrumented = False self._tracer = None + self._media = None logger.info("Pydantic AI instrumentation disabled") @@ -476,6 +502,7 @@ def instrument_pydantic_ai( tracer_provider: Optional[Any] = None, use_builtin: bool = False, config: Optional[TraceConfig] = None, + attachments: Optional[Any] = None, ) -> PydanticAIInstrumentor: """Convenience function to instrument Pydantic AI. @@ -483,6 +510,8 @@ def instrument_pydantic_ai( tracer_provider: Optional OpenTelemetry tracer provider use_builtin: Deprecated compatibility flag; uses FI-aware wrappers config: Content privacy policy; defaults resolve from the environment + attachments: The provider-owned attachment manager + (``register(...).attachment_manager``); media is captured only when enabled Returns: The instrumentor instance @@ -503,6 +532,8 @@ def instrument_pydantic_ai( kwargs["use_builtin"] = use_builtin if config is not None: kwargs["config"] = config + if attachments is not None: + kwargs["attachments"] = attachments instrumentor.instrument(**kwargs) return instrumentor diff --git a/python/tests/attachments_frameworks/conftest.py b/python/tests/attachments_frameworks/conftest.py new file mode 100644 index 000000000..f9e7d293b --- /dev/null +++ b/python/tests/attachments_frameworks/conftest.py @@ -0,0 +1,75 @@ +"""Framework attachment tests reuse the provider lane's loopback harness (synthetic data only). + +- ``f01``: TLS loopback F01 attachment server; ``provider``: plain-HTTP loopback that + answers like the OpenAI API, so real framework + real provider SDK paths run end to end; +- ``spans``: an OpenTelemetry provider with an in-memory exporter. +""" + +import importlib.util +import re +import sys +from pathlib import Path + +import pytest + +pytest.importorskip("fi_instrumentation.attachments.media_capture") + +sys.path.insert(0, str(Path(__file__).parent)) + +_spec = importlib.util.spec_from_file_location( + "_attachments_provider_harness", Path(__file__).parent.parent / "attachments_providers" / "conftest.py") +_harness = importlib.util.module_from_spec(_spec) +sys.modules[_spec.name] = _harness +_spec.loader.exec_module(_harness) + +# Fixtures (pytest registers them from this module's namespace) and helpers. +clean_env = _harness.clean_env +f01 = _harness.f01 +provider = _harness.provider +spans = _harness.spans +PDF = _harness.PDF +b64 = _harness.b64 +flush = _harness.flush +png = _harness.png +refs = _harness.refs +wait_uploads = _harness.wait_uploads +wav = _harness.wav + + +def jpeg(seed=0) -> bytes: + """Bytes with a JPEG signature; only the container magic is checked by the SDK side.""" + return b"\xff\xd8\xff\xe0" + bytes((seed + i) % 256 for i in range(60)) + b"\xff\xd9" + + +def content_puts(f01): + """Content PUT bodies (actual uploaded bytes) in arrival order.""" + return f01.payloads + + +def by_name(exporter, fragment): + return [span for span in exporter.get_finished_spans() if fragment in span.name] + + +def all_refs(exporter): + found = [] + for span in exporter.get_finished_spans(): + found.extend((span.name, ref) for ref in refs(span)) + return found + + +_URI = re.compile(r"fi-attachment:([A-Za-z0-9_-]+)") + + +def assert_uris_resolve(exporter): + """F01 resolve checks the expected parent: every ``fi-attachment:`` a span carries + must be an occurrence in THAT span's own ``fi.attachment.refs``. Returns the URI count.""" + seen = 0 + for span in exporter.get_finished_spans(): + own = {ref["occurrence_id"] for ref in refs(span)} + for name, value in (span.attributes or {}).items(): + values = value if isinstance(value, (list, tuple)) else (value,) + for item in values: + for occurrence in _URI.findall(item) if isinstance(item, str) else (): + seen += 1 + assert occurrence in own, (span.name, name, occurrence, sorted(own)) + return seen diff --git a/python/tests/attachments_frameworks/test_agno_attachments.py b/python/tests/attachments_frameworks/test_agno_attachments.py new file mode 100644 index 000000000..b09ac0ed7 --- /dev/null +++ b/python/tests/attachments_frameworks/test_agno_attachments.py @@ -0,0 +1,139 @@ +"""Real Agno agent + real OpenAI SDK (loopback) -> traceai_agno -> loopback F01. + +Privacy is the intersection of the FI policy and the OpenInference TraceConfig. +""" + +import pytest + +pytest.importorskip("agno") +pytest.importorskip("openinference.instrumentation.agno") +pytest.importorskip("traceai_agno") + +from conftest import PDF, b64, flush, png, refs, wav # noqa: E402 +from agno.agent import Agent # noqa: E402 +from agno.media import Audio, File, Image # noqa: E402 +from agno.models.openai import OpenAIChat # noqa: E402 +from fi_instrumentation.attachments import AttachmentConfig, create_attachment_manager # noqa: E402 +from openinference.instrumentation import TraceConfig as OITraceConfig # noqa: E402 +from traceai_agno import AgnoInstrumentorWrapper, configure_agno_tracing # noqa: E402 + +IMAGE = png(seed=51) + + +@pytest.fixture(autouse=True) +def no_telemetry(monkeypatch): + monkeypatch.setenv("AGNO_TELEMETRY", "false") + for key in ("OPENINFERENCE_HIDE_INPUTS", "OPENINFERENCE_HIDE_INPUT_IMAGES", "OPENINFERENCE_HIDE_OUTPUTS"): + monkeypatch.delenv(key, raising=False) + + +@pytest.fixture +def instrument(spans): + tracer_provider, exporter = spans + + def install(manager, config=None): + configure_agno_tracing(tracer_provider=tracer_provider, attachments=manager, config=config) + return exporter + + yield install + AgnoInstrumentorWrapper().uninstrument() + + +def agent(provider): + model = OpenAIChat(id="gpt-4o-mini", api_key="sk-synthetic", base_url=provider.url + "/v1", max_retries=0) + try: + return Agent(model=model, telemetry=False) + except TypeError: # older Agno has no telemetry switch; AGNO_TELEMETRY=false covers it + return Agent(model=model) + + +def model_spans(exporter): + return [s for s in exporter.get_finished_spans() if s.name.endswith((".invoke", ".ainvoke"))] + + +def test_image_bytes_reach_f01_on_the_model_span(f01, provider, instrument): + manager = f01.manager() + exporter = instrument(manager) + agent(provider).run("what is this", images=[Image(content=IMAGE, format="png")]) + flush(manager) + assert f01.payloads == [IMAGE] + (body,) = f01.uploads + assert body["mime_type"] == "image/png" and body["direction"] == "input" and body["purpose"] == "model_input" + (span,) = model_spans(exporter) + assert [r["occurrence_id"] for r in refs(span)] == [body["occurrence_id"]] + + +def test_async_run_and_audio(f01, provider, instrument): + import asyncio + + manager = f01.manager() + instrument(manager) + asyncio.run(agent(provider).arun("listen", audio=[Audio(content=wav(), format="wav")])) + flush(manager) + assert f01.payloads == [wav()] + + +def test_file_bytes_pdf(f01, provider, instrument): + manager = f01.manager() + instrument(manager) + try: + agent(provider).run("read", files=[File(content=PDF, mime_type="application/pdf", filename="a.pdf")]) + except Exception: + pass # the OpenAI chat formatter may reject files on some versions; capture happens before the call + flush(manager) + assert f01.payloads == [PDF] + + +@pytest.mark.parametrize("oi", [OITraceConfig(hide_input_images=True), OITraceConfig(hide_inputs=True), + OITraceConfig(hide_input_messages=True)]) +def test_openinference_restriction_alone_denies(f01, provider, instrument, oi): + manager = f01.manager() + instrument(manager, config=oi) + agent(provider).run("what is this", images=[Image(content=IMAGE, format="png")]) + flush(manager) + assert f01.after_bootstrap() == [] + + +@pytest.mark.parametrize("env", ["FI_HIDE_INPUT_IMAGES", "FI_HIDE_INPUTS", "FI_PII_REDACTION"]) +def test_fi_restriction_alone_denies_even_with_permissive_oi_config(f01, provider, instrument, monkeypatch, env): + monkeypatch.setenv(env, "true") + manager = f01.manager() + instrument(manager, config=OITraceConfig(hide_input_images=False, hide_inputs=False)) + agent(provider).run("what is this", images=[Image(content=IMAGE, format="png")]) + flush(manager) + assert f01.after_bootstrap() == [] + + +def test_openinference_env_restriction_denies(f01, provider, instrument, monkeypatch): + monkeypatch.setenv("OPENINFERENCE_HIDE_INPUT_IMAGES", "true") + manager = f01.manager() + instrument(manager) + agent(provider).run("what is this", images=[Image(content=IMAGE, format="png")]) + flush(manager) + assert f01.after_bootstrap() == [] + + +def test_url_and_filepath_are_never_read(f01, provider, instrument, tmp_path): + manager = f01.manager() + instrument(manager) + path = tmp_path / "secret.png" + path.write_bytes(IMAGE) + run = agent(provider) + for image in (Image(url="http://127.0.0.1:9/never.png"), Image(filepath=str(path))): + try: + run.run("what is this", images=[image]) + except Exception: + pass # Agno itself may try to load the source for the provider; the instrumentation never does + flush(manager) + assert f01.payloads == [] + + +def test_mismatched_bytes_and_disabled_manager(f01, provider, instrument): + instrument(create_attachment_manager(config=AttachmentConfig(enabled=False))) + agent(provider).run("x", images=[Image(content=IMAGE, format="png")]) + AgnoInstrumentorWrapper().uninstrument() + manager = f01.manager() + instrument(manager) + agent(provider).run("x", images=[Image(content=b"GIF89a not a png at all", format="png")]) + flush(manager) + assert f01.payloads == [] diff --git a/python/tests/attachments_frameworks/test_anthropic_under_framework.py b/python/tests/attachments_frameworks/test_anthropic_under_framework.py new file mode 100644 index 000000000..4a01ea922 --- /dev/null +++ b/python/tests/attachments_frameworks/test_anthropic_under_framework.py @@ -0,0 +1,147 @@ +"""Framework + Anthropic instrumentor on ONE manager: one model call, one owner. + +The owner mark is provider-neutral: ``traceai_anthropic`` makes the same core check as +``traceai_openai``, so a framework call that captured its media is not uploaded a second time +by the Anthropic provider span. Driven through LangChain's ``ChatAnthropic`` against the real +Anthropic SDK (loopback provider). Latest-only: skipped where ``langchain-anthropic`` is not +installed. + +Not covered here: +- Pydantic AI's Anthropic model calls ``client.beta.messages``, which ``traceai_anthropic`` does + not instrument, so that pair has no provider span to double-count; +- llama-index-llms-anthropic 0.12.3 fails before any request against anthropic 2.8.0 + (``Cannot instantiate typing.Union`` building ``MessageParam``), with or without traceAI. +""" + +import json + +import pytest + +pytest.importorskip("anthropic") +pytest.importorskip("traceai_anthropic") + +from conftest import assert_uris_resolve, b64, flush, png, refs # noqa: E402 +from fi_instrumentation import TraceConfig # noqa: E402 + +IMAGE = png(seed=41) +URI = "data:image/png;base64," + b64(IMAGE) +ANTHROPIC_SCOPE = "traceai_anthropic" +MODEL = "claude-haiku-4-5" # the loopback answers any model id + + +@pytest.fixture +def anthropic_on(spans): + from traceai_anthropic import AnthropicInstrumentor + + tracer_provider, _ = spans + state = {} + + def install(manager): + AnthropicInstrumentor().instrument(tracer_provider=tracer_provider, attachments=manager, config=TraceConfig()) + state["on"] = True + + yield install + if state.get("on"): + AnthropicInstrumentor().uninstrument() + + +class LangChain: + def __init__(self, provider, tracer_provider): + pytest.importorskip("traceai_langchain") + pytest.importorskip("langchain_anthropic") + self.provider, self.tracer_provider = provider, tracer_provider + + def instrument(self, manager): + from traceai_langchain import LangChainInstrumentor + + kwargs = {"attachments": manager} if manager is not None else {} + LangChainInstrumentor().instrument(tracer_provider=self.tracer_provider, config=TraceConfig(), **kwargs) + + def uninstrument(self): + from traceai_langchain import LangChainInstrumentor + + LangChainInstrumentor().uninstrument() + + def call(self, mode): + from langchain_anthropic import ChatAnthropic + from langchain_core.messages import HumanMessage + + model = ChatAnthropic(model=MODEL, api_key="sk-ant-synthetic", base_url=self.provider.url, max_retries=0, + max_tokens=16) + messages = [HumanMessage(content=[{"type": "text", "text": "what"}, + {"type": "image_url", "image_url": {"url": URI}}])] + if mode == "sync": + model.invoke(messages) + else: + for _ in model.stream(messages): + pass + + +FRAMEWORKS = {"langchain": LangChain} + + +@pytest.fixture +def framework(request, provider, spans): + tracer_provider, _ = spans + driver = FRAMEWORKS[request.param](provider, tracer_provider) + state = {} + original = driver.instrument + + def instrument(manager): + original(manager) + state["on"] = True + + driver.instrument = instrument + yield driver + if state.get("on"): + driver.uninstrument() + + +def sent_image(provider): + return any(path.endswith("/messages") and b64(IMAGE) in json.dumps(body) for path, body, raw in provider.requests) + + +def owners(exporter): + return [s for s in exporter.get_finished_spans() if refs(s)] + + +def is_provider_span(span): + return getattr(span.instrumentation_scope, "name", "") == ANTHROPIC_SCOPE + + +@pytest.mark.parametrize("framework", list(FRAMEWORKS), indirect=True) +@pytest.mark.parametrize("mode", ["sync", "stream"]) +def test_framework_and_anthropic_one_call_uploads_each_part_once(f01, provider, spans, anthropic_on, framework, mode): + """Without the mark this is 2 (framework span + Anthropic span).""" + _, exporter = spans + manager = f01.manager() + framework.instrument(manager) + anthropic_on(manager) + framework.call(mode) + flush(manager) + assert sent_image(provider), "the framework did not send the image to the provider" + assert any(is_provider_span(s) for s in exporter.get_finished_spans()), "no Anthropic span: not a 2-layer call" + assert f01.payloads == [IMAGE], len(f01.payloads) + (owner,) = owners(exporter) + assert not is_provider_span(owner), owner.name + (ref,) = refs(owner) + (upload,) = f01.uploads + assert upload["occurrence_id"] == ref["occurrence_id"] + assert upload["association"]["parent"]["span_id"] == format(owner.context.span_id, "016x") + assert_uris_resolve(exporter) + + +@pytest.mark.parametrize("framework", list(FRAMEWORKS), indirect=True) +def test_framework_attachments_off_leaves_the_anthropic_span_the_owner(f01, provider, spans, anthropic_on, framework): + """Control: no framework manager, no mark, so the Anthropic provider span captures the part.""" + _, exporter = spans + manager = f01.manager() + framework.instrument(None) + anthropic_on(manager) + framework.call("sync") + flush(manager) + assert sent_image(provider) + assert f01.payloads == [IMAGE], len(f01.payloads) + (owner,) = owners(exporter) + assert is_provider_span(owner), owner.name + assert_uris_resolve(exporter) diff --git a/python/tests/attachments_frameworks/test_exit_probe.py b/python/tests/attachments_frameworks/test_exit_probe.py new file mode 100644 index 000000000..58f004389 --- /dev/null +++ b/python/tests/attachments_frameworks/test_exit_probe.py @@ -0,0 +1,112 @@ +"""Lifecycle: a framework-instrumented process exits promptly with attachments enabled, disabled, after fork.""" + +import os +import subprocess +import sys +import textwrap +import time + +import pytest + +PROBE = textwrap.dedent(""" + import base64, os, ssl, sys, threading + sys.path.insert(0, os.environ["PROBE_DIR"]) + from conftest import png + from fi_instrumentation.attachments import AttachmentConfig, AttachmentManager, create_attachment_manager + from fi_instrumentation.attachments.protocol import HTTPTransport + from opentelemetry.sdk.trace import TracerProvider + + enabled = os.environ["PROBE_ENABLED"] == "1" + framework = os.environ["PROBE_FRAMEWORK"] + def attachment_threads(): + return {t.ident for t in threading.enumerate() if t.name.startswith("fi-attachment")} + before = attachment_threads() + if enabled: + transport = HTTPTransport(os.environ["PROBE_F01"], {"X-API-Key": "SYNTHETIC_KEY"}, + ssl_context=ssl.create_default_context(cafile=os.environ["PROBE_CERT"])) + manager = AttachmentManager(config=AttachmentConfig(enabled=True), project_name="synthetic", + transport=transport) + assert manager.ready(5) + else: + manager = create_attachment_manager(config=AttachmentConfig(enabled=False)) + image = png(seed=71) + + if framework == "langchain": + from langchain_core.language_models.fake_chat_models import FakeListChatModel + from langchain_core.messages import HumanMessage + from traceai_langchain import LangChainInstrumentor + LangChainInstrumentor().instrument(tracer_provider=TracerProvider(), attachments=manager) + model = FakeListChatModel(responses=["ok", "ok"]) + def call(): + model.invoke([HumanMessage(content=[{"type": "image", "source_type": "base64", + "data": base64.b64encode(image).decode(), + "mime_type": "image/png"}])]) + else: + from pydantic_ai import Agent, BinaryContent + from pydantic_ai.messages import ModelResponse, TextPart + from pydantic_ai.models.function import FunctionModel + from traceai_pydantic_ai import PydanticAIInstrumentor + PydanticAIInstrumentor().instrument(tracer_provider=TracerProvider(), attachments=manager, + skip_dep_check=True) + agent = Agent(FunctionModel(lambda m, i: ModelResponse(parts=[TextPart("ok")]))) + def call(): + agent.run_sync(["x", BinaryContent(data=image, media_type="image/png")]) + + call() + if not enabled: + # Frameworks may start their own threads (event loops, executors); attachments must not. + assert attachment_threads() == before, "disabled attachments started a thread" + if os.environ.get("PROBE_FORK") == "1" and hasattr(os, "fork"): + pid = os.fork() + if pid == 0: + call() + os._exit(0) + os.waitpid(pid, 0) + print("PROBE_DONE", flush=True) + # No explicit flush or shutdown: interpreter exit must not hang. +""") + + +def _pydantic_ai_major(): + from importlib.metadata import PackageNotFoundError, version + + for name in ("pydantic-ai-slim", "pydantic-ai"): + try: + return int(version(name).split(".")[0]) + except (PackageNotFoundError, ValueError): + continue + return 0 + + +def _available(framework): + module = {"langchain": "traceai_langchain", "pydantic_ai": "traceai_pydantic_ai"}[framework] + try: + __import__(module) + __import__("langchain_core" if framework == "langchain" else "pydantic_ai") + return True + except ImportError: + return False + + +@pytest.mark.parametrize("framework", ["langchain", "pydantic_ai"]) +@pytest.mark.parametrize("enabled,fork", [(True, False), (False, False), (True, True)]) +def test_process_exits_promptly(f01, tmp_path, framework, enabled, fork): + if not _available(framework): + pytest.skip(framework + " not installed") + if fork and framework == "pydantic_ai" and _pydantic_ai_major() < 1: + # pydantic-ai 0.x run_sync hangs in a forked child with NO instrumentation at all (seen with 0.1.0); + # the fork row is still covered there by LangChain. + pytest.skip("pydantic-ai 0.x run_sync is not fork-safe on its own") + script = tmp_path / "probe.py" + script.write_text(PROBE) + env = dict(os.environ, PROBE_DIR=os.path.dirname(__file__), PROBE_ENABLED="1" if enabled else "0", + PROBE_F01=f01.endpoint, PROBE_CERT=str(f01.cert), PROBE_FORK="1" if fork else "0", + PROBE_FRAMEWORK=framework) + started = time.monotonic() + result = subprocess.run([sys.executable, str(script)], env=env, capture_output=True, text=True, timeout=60) + elapsed = time.monotonic() - started + assert result.returncode == 0, result.stderr[-2000:] + assert "PROBE_DONE" in result.stdout + assert elapsed < 30, elapsed + if not enabled: + assert f01.state["requests"] == [] diff --git a/python/tests/attachments_frameworks/test_langchain_attachments.py b/python/tests/attachments_frameworks/test_langchain_attachments.py new file mode 100644 index 000000000..3f7267c91 --- /dev/null +++ b/python/tests/attachments_frameworks/test_langchain_attachments.py @@ -0,0 +1,227 @@ +"""Real langchain-core callbacks -> traceai_langchain -> loopback F01 (typed message media).""" + +from typing import Any, List, Optional + +import pytest + +pytest.importorskip("langchain_core") +pytest.importorskip("traceai_langchain") + +from conftest import PDF, assert_uris_resolve, b64, flush, png, refs, wav # noqa: E402 +from fi_instrumentation import TraceConfig # noqa: E402 +from fi_instrumentation.attachments import AttachmentConfig, create_attachment_manager # noqa: E402 +from langchain_core.language_models.chat_models import BaseChatModel # noqa: E402 +from langchain_core.messages import AIMessage, BaseMessage, HumanMessage, ToolMessage # noqa: E402 +from langchain_core.outputs import ChatGeneration, ChatResult # noqa: E402 +from opentelemetry.sdk.trace import TracerProvider # noqa: E402 +from opentelemetry.sdk.trace.export import SimpleSpanProcessor # noqa: E402 +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter # noqa: E402 +from opentelemetry.sdk.trace.sampling import ALWAYS_OFF # noqa: E402 +from traceai_langchain import LangChainInstrumentor # noqa: E402 + +IMAGE = png(seed=21) +OUTPUT = png(seed=22) +TOOL = png(seed=23) + + +class FakeChat(BaseChatModel): + """A real BaseChatModel: LangChain's own callback manager drives the tracer.""" + + reply: Any = "ok" + fail: bool = False + + @property + def _llm_type(self) -> str: + return "fake-media" + + def _generate(self, messages: List[BaseMessage], stop: Optional[List[str]] = None, run_manager=None, + **kwargs: Any) -> ChatResult: + if self.fail: + raise RuntimeError("provider said: secret error body") + return ChatResult(generations=[ChatGeneration(message=AIMessage(content=self.reply))]) + + +@pytest.fixture +def instrument(spans): + tracer_provider, exporter = spans + state = {} + + def install(manager, config=None, provider=None): + LangChainInstrumentor().instrument(tracer_provider=provider or tracer_provider, attachments=manager, + config=config or TraceConfig()) + state["on"] = True + return exporter + + yield install + if state.get("on"): + LangChainInstrumentor().uninstrument() + + +def llm_span(exporter): + found = [s for s in exporter.get_finished_spans() if s.attributes.get("gen_ai.span.kind") == "LLM" + or s.name == "FakeChat"] + assert found, [s.name for s in exporter.get_finished_spans()] + return found[-1] + + +def human(*blocks): + return HumanMessage(content=[{"type": "text", "text": "describe"}, *blocks]) + + +BLOCK_STYLES = { + "openai_image_url": ({"type": "image_url", "image_url": {"url": "data:image/png;base64," + b64(IMAGE)}}, + IMAGE, "image/png", "model_input"), + "lc03_standard": ({"type": "image", "source_type": "base64", "data": b64(IMAGE), "mime_type": "image/png"}, + IMAGE, "image/png", "model_input"), + "lc1_content_block": ({"type": "image", "base64": b64(IMAGE), "mime_type": "image/png"}, + IMAGE, "image/png", "model_input"), + "anthropic_source": ({"type": "image", "source": {"type": "base64", "media_type": "image/png", + "data": b64(IMAGE)}}, IMAGE, "image/png", "model_input"), + "openai_file_pdf": ({"type": "file", "file": {"filename": "a.pdf", + "file_data": "data:application/pdf;base64," + b64(PDF)}}, + PDF, "application/pdf", "document"), + "openai_input_audio": ({"type": "input_audio", "input_audio": {"data": b64(wav()), "format": "wav"}}, + wav(), "audio/wav", "model_input"), +} + + +@pytest.mark.parametrize("style", sorted(BLOCK_STYLES)) +def test_input_block_styles_upload_original_bytes(f01, instrument, style): + block, data, mime, purpose = BLOCK_STYLES[style] + manager = f01.manager() + exporter = instrument(manager) + FakeChat().invoke([human(block)]) + flush(manager) + assert f01.payloads == [data] + (body,) = f01.uploads + assert body["mime_type"] == mime and body["direction"] == "input" and body["purpose"] == purpose + span_refs = refs(llm_span(exporter)) + assert [(r["occurrence_id"], r["mime_type"]) for r in span_refs] == [(body["occurrence_id"], mime)] + + +def test_output_media_and_tool_results(f01, instrument): + manager = f01.manager() + exporter = instrument(manager) + reply = [{"type": "text", "text": "here"}, {"type": "image", "base64": b64(OUTPUT), "mime_type": "image/png"}] + tool = ToolMessage(content=[{"type": "image_url", "image_url": {"url": "data:image/png;base64," + b64(TOOL)}}], + tool_call_id="call_screen_1") + ask = AIMessage(content="", tool_calls=[{"id": "call_screen_1", "name": "screenshot", "args": {}}]) + FakeChat(reply=reply).invoke([human(), ask, tool]) + flush(manager) + assert sorted(f01.payloads) == sorted([TOOL, OUTPUT]) + bodies = {body["direction"]: body for body in f01.uploads} + assert bodies["input"]["purpose"] == "tool_result" and bodies["input"].get("tool_call_id") == "call_screen_1" + assert bodies["output"]["purpose"] == "model_output" + positions = {(r["direction"], r.get("message_index"), r.get("part_index"), r.get("role")) + for r in refs(llm_span(exporter))} + assert ("input", 2, 0, "tool") in positions and ("output", 0, 1, "assistant") in positions + + +def test_failed_run_never_captures_output_and_input_still_follows_policy(f01, instrument): + manager = f01.manager() + instrument(manager) + with pytest.raises(RuntimeError): + FakeChat(fail=True, reply=[{"type": "image", "base64": b64(OUTPUT), "mime_type": "image/png"}]).invoke( + [human({"type": "image_url", "image_url": {"url": "data:image/png;base64," + b64(IMAGE)}})]) + flush(manager) + assert f01.payloads == [IMAGE] + + +@pytest.mark.parametrize("config", [ + TraceConfig(hide_inputs=True), TraceConfig(hide_input_messages=True), TraceConfig(hide_input_images=True), + TraceConfig(hide_input_text=True), TraceConfig(pii_redaction=True), +]) +def test_local_input_restrictions_deny(f01, instrument, config): + manager = f01.manager() + instrument(manager, config=config) + FakeChat().invoke([human({"type": "image", "base64": b64(IMAGE), "mime_type": "image/png"})]) + flush(manager) + assert f01.after_bootstrap() == [] + + +def test_pii_redaction_beats_server_allowance(f01, instrument): + f01.state["policy"]["allow_unredacted_media_with_text_pii"] = True + manager = f01.manager() + instrument(manager, config=TraceConfig(pii_redaction=True)) + FakeChat().invoke([human({"type": "image", "base64": b64(IMAGE), "mime_type": "image/png"})]) + flush(manager) + assert f01.after_bootstrap() == [] + + +def test_hide_outputs_denies_output_only(f01, instrument): + manager = f01.manager() + instrument(manager, config=TraceConfig(hide_outputs=True)) + FakeChat(reply=[{"type": "image", "base64": b64(OUTPUT), "mime_type": "image/png"}]).invoke( + [human({"type": "image", "base64": b64(IMAGE), "mime_type": "image/png"})]) + flush(manager) + assert f01.payloads == [IMAGE] + + +def test_server_policy_audio_only_denies_images(f01, instrument): + f01.state["policy"]["allowed_modalities"] = ["audio"] + manager = f01.manager() + instrument(manager) + FakeChat().invoke([human({"type": "image", "base64": b64(IMAGE), "mime_type": "image/png"}, + {"type": "input_audio", "input_audio": {"data": b64(wav()), "format": "wav"}})]) + flush(manager) + assert f01.payloads == [wav()] + + +def test_disabled_missing_and_unsampled_capture_nothing(f01, instrument, monkeypatch): + block = {"type": "image", "base64": b64(IMAGE), "mime_type": "image/png"} + instrument(create_attachment_manager(config=AttachmentConfig(enabled=False))) + FakeChat().invoke([human(block)]) + LangChainInstrumentor().uninstrument() + instrument(None) + FakeChat().invoke([human(block)]) + LangChainInstrumentor().uninstrument() + manager = f01.manager() + exporter = InMemorySpanExporter() + unsampled = TracerProvider(sampler=ALWAYS_OFF) + unsampled.add_span_processor(SimpleSpanProcessor(exporter)) + instrument(manager, provider=unsampled) + FakeChat().invoke([human(block)]) + flush(manager) + assert f01.after_bootstrap() == [] + + +def test_env_disable_wins(f01, instrument, monkeypatch): + manager = f01.manager() + instrument(manager) + monkeypatch.setenv("FI_ATTACHMENTS_ENABLED", "false") + FakeChat().invoke([human({"type": "image", "base64": b64(IMAGE), "mime_type": "image/png"})]) + flush(manager) + assert f01.after_bootstrap() == [] + + +def test_unsupported_sources_are_ignored(f01, instrument): + manager = f01.manager() + instrument(manager) + FakeChat().invoke([human( + {"type": "image_url", "image_url": {"url": "https://example.invalid/remote.png"}}, + {"type": "image", "source_type": "id", "id": "file-abc"}, + {"type": "file", "file": {"file_id": "file-abc"}}, + {"type": "image", "base64": b64(IMAGE), "mime_type": "image/gif"}, + {"type": "image", "base64": "@@@@", "mime_type": "image/png"}, + {"type": "image", "base64": b64(b"GIF89a-not-png"), "mime_type": "image/png"}, + )]) + flush(manager) + assert f01.payloads == [] + + +def test_same_image_on_two_chat_spans_gives_each_span_its_own_occurrence(f01, instrument): + from langchain_core.runnables import RunnableLambda + + manager = f01.manager() + exporter = instrument(manager) + block = {"type": "image", "base64": b64(IMAGE), "mime_type": "image/png"} + chain = RunnableLambda(lambda _: FakeChat().invoke([human(block)])) | \ + RunnableLambda(lambda _: FakeChat().invoke([human(block)])) + chain.invoke("x") + flush(manager) + assert f01.payloads == [IMAGE, IMAGE] # byte deduplication is server-side + chats = [s for s in exporter.get_finished_spans() if s.name == "FakeChat"] + assert [len(refs(s)) for s in chats] == [1, 1] + owned = {r["occurrence_id"]: format(s.context.span_id, "016x") for s in chats for r in refs(s)} + assert owned == {b["occurrence_id"]: b["association"]["parent"]["span_id"] for b in f01.uploads} + assert_uris_resolve(exporter) diff --git a/python/tests/attachments_frameworks/test_llamaindex_attachments.py b/python/tests/attachments_frameworks/test_llamaindex_attachments.py new file mode 100644 index 000000000..a90af2b0f --- /dev/null +++ b/python/tests/attachments_frameworks/test_llamaindex_attachments.py @@ -0,0 +1,151 @@ +"""Real llama-index-core dispatcher events -> traceai_llamaindex -> loopback F01 (ChatMessage blocks).""" + +import io + +import pytest + +pytest.importorskip("llama_index.core") +pytest.importorskip("traceai_llamaindex") + +from conftest import PDF, b64, flush, png, refs, wav # noqa: E402 +from fi_instrumentation import TraceConfig # noqa: E402 +from fi_instrumentation.attachments import AttachmentConfig, create_attachment_manager # noqa: E402 +from traceai_llamaindex import LlamaIndexInstrumentor # noqa: E402 + +types = pytest.importorskip("llama_index.core.base.llms.types") +if not hasattr(types, "ImageBlock"): + pytest.skip("ChatMessage blocks need llama-index-core 0.12.3+", allow_module_level=True) + +from llama_index.core.base.llms.types import ( # noqa: E402 + ChatMessage, ChatResponse, CompletionResponse, ImageBlock, LLMMetadata, TextBlock, +) +from llama_index.core.llms.callbacks import llm_chat_callback, llm_completion_callback # noqa: E402 +from llama_index.core.llms.custom import CustomLLM # noqa: E402 + +DocumentBlock = getattr(types, "DocumentBlock", None) +AudioBlock = getattr(types, "AudioBlock", None) +IMAGE = png(seed=41) +OUTPUT = png(seed=42) + + +class MediaLLM(CustomLLM): + """A real CustomLLM whose chat() is wrapped by LlamaIndex's own dispatcher callbacks.""" + + reply_blocks: list = [] + + @property + def metadata(self) -> LLMMetadata: + return LLMMetadata(model_name="media-llm", is_chat_model=True) + + @llm_chat_callback() + def chat(self, messages, **kwargs): + return ChatResponse(message=ChatMessage(role="assistant", blocks=list(self.reply_blocks))) + + @llm_completion_callback() + def complete(self, prompt, formatted=False, **kwargs): + return CompletionResponse(text="ok") + + @llm_completion_callback() + def stream_complete(self, prompt, formatted=False, **kwargs): + yield CompletionResponse(text="ok", delta="ok") + + +@pytest.fixture +def instrument(spans): + tracer_provider, exporter = spans + state = {} + + def install(manager, config=None): + LlamaIndexInstrumentor().instrument(tracer_provider=tracer_provider, attachments=manager, + config=config or TraceConfig()) + state["on"] = True + return exporter + + yield install + if state.get("on"): + LlamaIndexInstrumentor().uninstrument() + + +def chat_refs(exporter): + return [ref for span in exporter.get_finished_spans() for ref in refs(span)] + + +def test_input_and_output_blocks(f01, instrument): + manager = f01.manager() + exporter = instrument(manager) + llm = MediaLLM(reply_blocks=[TextBlock(text="here"), ImageBlock(image=OUTPUT, image_mimetype="image/png")]) + blocks = [TextBlock(text="what"), ImageBlock(image=IMAGE, image_mimetype="image/png")] + if AudioBlock is not None: + blocks.append(AudioBlock(audio=wav(), format="wav")) + if DocumentBlock is not None: + blocks.append(DocumentBlock(data=PDF, document_mimetype="application/pdf")) + llm.chat([ChatMessage(role="user", blocks=blocks)]) + flush(manager) + expected = [IMAGE, OUTPUT] + ([wav()] if AudioBlock is not None else []) + \ + ([PDF] if DocumentBlock is not None else []) + assert sorted(f01.payloads) == sorted(expected) + found = {(r["direction"], r.get("message_index"), r.get("part_index"), r["mime_type"]) + for r in chat_refs(exporter)} + assert ("input", 0, 1, "image/png") in found + if AudioBlock is not None: + assert ("input", 0, 2, "audio/wav") in found + assert ("output", 0, 1, "image/png") in found + purposes = {body["mime_type"]: body["purpose"] for body in f01.uploads if body["direction"] == "input"} + assert purposes["image/png"] == "model_input" + if DocumentBlock is not None: + assert purposes["application/pdf"] == "document" + + +def test_url_path_and_stream_sources_are_never_read(f01, instrument, tmp_path): + manager = f01.manager() + instrument(manager) + path = tmp_path / "secret.png" + path.write_bytes(IMAGE) + stream = io.BytesIO(IMAGE) + blocks = [ImageBlock(url="http://127.0.0.1:9/never.png", image_mimetype="image/png"), + ImageBlock(path=path, image_mimetype="image/png")] + try: + blocks.append(ImageBlock(image=stream, image_mimetype="image/png")) + except Exception: + pass # older cores reject IOBase sources at construction + MediaLLM().chat([ChatMessage(role="user", blocks=blocks)]) + flush(manager) + assert f01.payloads == [] + assert stream.tell() in (0, len(IMAGE)) # never read by the instrumentation beyond what LlamaIndex did + + +def test_mismatched_magic_is_rejected(f01, instrument): + manager = f01.manager() + instrument(manager) + MediaLLM().chat([ChatMessage(role="user", blocks=[ImageBlock(image=PDF, image_mimetype="image/png")])]) + flush(manager) + assert f01.payloads == [] + + +@pytest.mark.parametrize("config", [TraceConfig(hide_inputs=True), TraceConfig(hide_input_images=True), + TraceConfig(pii_redaction=True), TraceConfig(hide_input_messages=True)]) +def test_local_restrictions_deny_input(f01, instrument, config): + manager = f01.manager() + instrument(manager, config=config) + MediaLLM().chat([ChatMessage(role="user", blocks=[ImageBlock(image=IMAGE, image_mimetype="image/png")])]) + flush(manager) + assert f01.after_bootstrap() == [] + + +def test_hide_outputs_denies_output(f01, instrument): + manager = f01.manager() + instrument(manager, config=TraceConfig(hide_outputs=True)) + MediaLLM(reply_blocks=[ImageBlock(image=OUTPUT, image_mimetype="image/png")]).chat( + [ChatMessage(role="user", content="hi")]) + flush(manager) + assert f01.after_bootstrap() == [] + + +def test_disabled_and_missing_manager_capture_nothing(f01, instrument): + instrument(create_attachment_manager(config=AttachmentConfig(enabled=False))) + MediaLLM().chat([ChatMessage(role="user", blocks=[ImageBlock(image=IMAGE, image_mimetype="image/png")])]) + LlamaIndexInstrumentor().uninstrument() + instrument(None) + MediaLLM().chat([ChatMessage(role="user", blocks=[ImageBlock(image=IMAGE, image_mimetype="image/png")])]) + f01.manager() + assert f01.after_bootstrap() == [] diff --git a/python/tests/attachments_frameworks/test_occurrences.py b/python/tests/attachments_frameworks/test_occurrences.py new file mode 100644 index 000000000..4650af2d3 --- /dev/null +++ b/python/tests/attachments_frameworks/test_occurrences.py @@ -0,0 +1,323 @@ +"""Shared helper: per-span occurrences, raw bytes, URL policy and part limits (no framework installed). + +F01 resolves an occurrence only under its own parent span, so every observing span owns +its own occurrence: nothing here reuses an occurrence across spans or traces. Two layers of +ONE model call do not both observe it: the framework adapter's owner mark makes the provider +skip (``test_owner.py``, ``test_provider_and_framework.py``); here no mark is set. +""" + +import http.server +import threading + +import pytest +from conftest import b64, flush, jpeg, png, refs, wav +from fi_instrumentation import TraceConfig +from fi_instrumentation.attachments import CapturePolicy +from fi_instrumentation.attachments import media_capture as mc +from opentelemetry import context as otel_context +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.sampling import ALWAYS_OFF + +IMAGE = png(seed=11) +URI = "data:image/png;base64," + b64(IMAGE) + + +def _inline(manager, span, value=URI, policy=None, raw=False, part_index=0): + return mc.attach_inline(manager, span, lambda: value, mc.IMAGE_MIMES, raw=raw, direction="input", + purpose="model_input", capture=policy or mc.local_capture_policy(None, direction="input"), + message_index=0, part_index=part_index, role="user") + + +def _parents(f01): + return [body["association"]["parent"]["span_id"] for body in f01.uploads] + + +def test_two_unmarked_observing_spans_each_own_their_occurrence(f01, spans): + tracer_provider, exporter = spans + manager = f01.manager() + tracer = tracer_provider.get_tracer("t") + with tracer.start_as_current_span("framework") as parent: + first = _inline(manager, parent) + with tracer.start_as_current_span("provider") as child: + second = _inline(manager, child) + flush(manager) + assert first is not None and second is not None and first != second + assert f01.payloads == [IMAGE, IMAGE] # byte deduplication is the server's job + names = {span.name: refs(span) for span in exporter.get_finished_spans()} + assert [r["occurrence_id"] for r in names["framework"]] == [first.occurrence_id] + assert [r["occurrence_id"] for r in names["provider"]] == [second.occurrence_id] + # Upload workers run concurrently, so the order the server sees them in is not fixed. + assert sorted(_parents(f01)) == sorted(["%016x" % parent.get_span_context().span_id, + "%016x" % child.get_span_context().span_id]) + + +def test_raw_bytes_and_inline_base64_at_two_positions_are_two_occurrences(f01, spans): + tracer_provider, exporter = spans + manager = f01.manager() + with tracer_provider.get_tracer("t").start_as_current_span("s") as span: + a = mc.attach_bytes(manager, span, IMAGE, mc.IMAGE_MIMES, direction="input", purpose="model_input", + capture=CapturePolicy(model_input=True), message_index=0, part_index=0) + b = _inline(manager, span, part_index=1) + flush(manager) + assert a is not None and b is not None and a != b + assert f01.payloads == [IMAGE, IMAGE] + (span,) = exporter.get_finished_spans() + assert sorted(r["occurrence_id"] for r in refs(span)) == sorted([a.occurrence_id, b.occurrence_id]) + + +def test_separate_traces_on_one_manager_never_share_an_occurrence(f01, spans): + # LangChain and LlamaIndex do not make their span current, so the provider span is a separate root. + tracer_provider, exporter = spans + manager = f01.manager() + tracer = tracer_provider.get_tracer("t") + with tracer.start_as_current_span("framework") as one: + a = _inline(manager, one) + with tracer.start_as_current_span("provider") as two: + b = _inline(manager, two) + flush(manager) + assert one.get_span_context().trace_id != two.get_span_context().trace_id + assert a is not None and b is not None and a != b and f01.payloads == [IMAGE, IMAGE] + for span, occurrence in zip(exporter.get_finished_spans(), (a, b)): + assert [r["occurrence_id"] for r in refs(span)] == [occurrence.occurrence_id] + + +def test_same_span_and_position_observed_twice_is_one_occurrence(f01, spans): + # The (span, position) registry still dedups ONE span observing ONE position twice. + tracer_provider, exporter = spans + manager = f01.manager() + part = mc.MediaPart("inline", lambda: URI, mc.IMAGE_MIMES, "model_input", 0, 0, raw=False) + with tracer_provider.get_tracer("t").start_as_current_span("s") as span: + first = mc.capture_parts(manager, span, "test", [part], direction="input") + again = mc.capture_parts(manager, span, "test", [part], direction="input") + flush(manager) + assert dict(first) == dict(again) and f01.payloads == [IMAGE] + (span,) = exporter.get_finished_spans() + assert len(refs(span)) == 1 + + +class _FailingStore(list): + """The loopback server fails every content PUT while this is its payload store.""" + + def append(self, item): + raise RuntimeError("synthetic content PUT failure") + + +def test_a_failed_upload_is_not_reused_by_a_later_span(f01, spans): + # The first content PUT fails; the same bytes on a later span upload again under that span. + tracer_provider, exporter = spans + manager = f01.manager() + tracer = tracer_provider.get_tracer("t") + good = f01.state["payloads"] + f01.state["payloads"] = _FailingStore() + with tracer.start_as_current_span("first") as one: + _inline(manager, one) + manager.flush(15) + f01.state["payloads"] = good + with tracer.start_as_current_span("second") as two: + b = _inline(manager, two) + flush(manager) + assert b is not None and f01.payloads == [IMAGE] + second = [s for s in exporter.get_finished_spans() if s.name == "second"][0] + assert [r["occurrence_id"] for r in refs(second)] == [b.occurrence_id] + assert _parents(f01)[-1] == "%016x" % two.get_span_context().span_id + + +def test_a_later_observer_denied_by_its_own_policy_gets_nothing(f01, spans): + tracer_provider, exporter = spans + manager = f01.manager() + tracer = tracer_provider.get_tracer("t") + hidden = mc.local_capture_policy(TraceConfig(hide_input_images=True), direction="input", image=True) + with tracer.start_as_current_span("framework") as parent: + assert _inline(manager, parent) is not None + with tracer.start_as_current_span("provider") as child: + assert _inline(manager, child, policy=hidden) is None + flush(manager) + assert f01.payloads == [IMAGE] + assert refs([s for s in exporter.get_finished_spans() if s.name == "provider"][0]) == [] + + +def test_denied_sources_are_never_read(f01, spans): + tracer_provider, _ = spans + manager = f01.manager() + reads = [] + hidden = mc.local_capture_policy(TraceConfig(hide_inputs=True), direction="input") + with tracer_provider.get_tracer("t").start_as_current_span("s") as span: + assert mc.attach_inline(manager, span, lambda: reads.append(1) or URI, mc.IMAGE_MIMES, raw=False, + direction="input", purpose="model_input", capture=hidden) is None + part = mc.MediaPart("bytes", lambda: reads.append(1) or IMAGE, mc.IMAGE_MIMES, "model_input", 0, 1) + assert mc.capture_parts(manager, span, "test", [part], direction="input", + trace_configs=(TraceConfig(hide_inputs=True),)) == {} + assert reads == [] and f01.after_bootstrap() == [] + + +def test_concurrent_observers_of_identical_bytes_each_own_one(f01, spans): + tracer_provider, exporter = spans + manager = f01.manager() + tracer = tracer_provider.get_tracer("t") + results = [] + with tracer.start_as_current_span("root"): + ctx = otel_context.get_current() + + def observe(): + otel_context.attach(ctx) + with tracer.start_as_current_span("child") as child: + results.append(_inline(manager, child)) + + threads = [threading.Thread(target=observe) for _ in range(8)] + [t.start() for t in threads] + [t.join() for t in threads] + flush(manager) + assert None not in results and len(set(results)) == 8 + assert f01.payloads == [IMAGE] * 8 + children = [s for s in exporter.get_finished_spans() if s.name == "child"] + assert sorted(r["occurrence_id"] for s in children for r in refs(s)) == \ + sorted(o.occurrence_id for o in results) + + +def test_tool_policy_folds_input_and_output_hide_flags(f01, spans): + tracer_provider, _ = spans + manager = f01.manager() + part = mc.MediaPart("bytes", lambda: IMAGE, mc.IMAGE_MIMES, "tool_result", 0, 0, role="tool") + tracer = tracer_provider.get_tracer("t") + for config in (TraceConfig(hide_outputs=True), TraceConfig(hide_output_text=True), + TraceConfig(hide_output_messages=True), TraceConfig(hide_inputs=True), + TraceConfig(hide_input_images=True), TraceConfig(pii_redaction=True)): + with tracer.start_as_current_span("tool") as span: + assert mc.capture_parts(manager, span, "test", [part], direction="input", trace_configs=(config,), + policy_directions=("input", "output")) == {}, config + with tracer.start_as_current_span("tool") as span: + captured = mc.capture_parts(manager, span, "test", [part], direction="input", + trace_configs=(TraceConfig(),), policy_directions=("input", "output")) + flush(manager) + assert len(captured) == 1 and f01.payloads == [IMAGE] + (body,) = f01.uploads + assert body["direction"] == "input" and body["purpose"] == "tool_result" + + +def test_magic_bytes_must_match_and_select_the_exact_type(f01, spans): + tracer_provider, _ = spans + manager = f01.manager() + policy = CapturePolicy(model_input=True) + with tracer_provider.get_tracer("t").start_as_current_span("s") as span: + assert mc.attach_bytes(manager, span, b"not an image at all", mc.IMAGE_MIMES, direction="input", + purpose="model_input", capture=policy) is None + assert mc.attach_bytes(manager, span, wav(), mc.IMAGE_MIMES, direction="input", + purpose="model_input", capture=policy) is None # audio is not an image + found = mc.attach_bytes(manager, span, jpeg(), mc.IMAGE_MIMES, direction="input", purpose="model_input", + capture=policy, part_index=1) + assert found is not None and found.mime_type == "image/jpeg" + # verify=True on inline data: a PNG data URI holding JPEG bytes is rejected. + lying = "data:image/png;base64," + b64(jpeg(seed=3)) + assert mc.attach_inline(manager, span, lambda: lying, mc.IMAGE_MIMES, raw=False, direction="input", + purpose="model_input", capture=policy, verify=True) is None + flush(manager) + assert f01.payloads == [jpeg()] + + +def test_over_cap_part_is_denied_before_any_read(f01, spans): + tracer_provider, _ = spans + manager = f01.manager() + reads = [] + big = b"\x89PNG\r\n\x1a\n" + b"\x00" * (mc.MAX_FRAMEWORK_BYTES + 1) + part = mc.MediaPart("bytes", lambda: reads.append(1) or big, mc.IMAGE_MIMES, "model_input", 0, 0) + with tracer_provider.get_tracer("t").start_as_current_span("s") as span: + assert mc.capture_parts(manager, span, "test", [part], direction="input") == {} + flush(manager) + assert f01.payloads == [] + + +def test_url_parts_go_to_core_policy_and_are_never_fetched(f01, spans): + hits = [] + + class Handler(http.server.BaseHTTPRequestHandler): + def do_GET(self): # noqa: N802 + hits.append(self.path) + self.send_response(200) + self.end_headers() + + def log_message(self, *args): + pass + + server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), Handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + tracer_provider, _ = spans + manager = f01.manager() + urls = ["http://127.0.0.1:%d/private.png" % server.server_port, "http://169.254.169.254/latest", + "file:///etc/passwd"] + parts = [mc.MediaPart("url", lambda u=u: u, mc.IMAGE_MIMES, "model_input", 0, i) for i, u in enumerate(urls)] + with tracer_provider.get_tracer("t").start_as_current_span("s") as span: + assert mc.capture_parts(manager, span, "test", parts, direction="input") == {} + flush(manager) + assert hits == [] and f01.after_bootstrap() == [] + finally: + server.shutdown() + server.server_close() + + +def test_part_limit_per_span(f01, spans): + tracer_provider, _ = spans + manager = f01.manager() + parts = [mc.MediaPart("inline", lambda i=i: b64(png(seed=i)), mc.IMAGE_MIMES, "model_input", 0, i) + for i in range(mc.MAX_FRAMEWORK_PARTS + 6)] + with tracer_provider.get_tracer("t").start_as_current_span("s") as span: + captured = mc.capture_parts(manager, span, "test", parts, direction="input") + flush(manager) + assert len(captured) == mc.MAX_FRAMEWORK_PARTS + assert len(f01.payloads) == mc.MAX_FRAMEWORK_PARTS + + +def test_unsampled_span_captures_nothing(f01): + manager = f01.manager() + tracer = TracerProvider(sampler=ALWAYS_OFF).get_tracer("t") + with tracer.start_as_current_span("s") as span: + assert _inline(manager, span) is None + assert f01.after_bootstrap() == [] + + +def test_intersection_is_deny_wins(): + a = CapturePolicy() + b = CapturePolicy(capture_content=False, muted=True) + both = mc.intersect(a, b) + assert both.capture_content is False and both.muted is True and both.record_inputs is True + assert mc.intersect().capture_content is True + # Any configuration in the list that hides wins (FI and OpenInference configs, for example). + class OIConfig: + hide_inputs = False + hide_input_images = True + folded = mc.intersect(*(mc.local_capture_policy(c, direction="input", image=True) + for c in (TraceConfig(), OIConfig()))) + assert folded.capture_content is False and folded.model_input is True + + +def test_capture_parts_folds_every_config(f01, spans): + class OIConfig: + hide_input_images = True + tracer_provider, _ = spans + manager = f01.manager() + parts = [mc.MediaPart("inline", lambda: URI, mc.IMAGE_MIMES, "model_input", 0, 0, raw=False)] + with tracer_provider.get_tracer("t").start_as_current_span("s") as span: + assert mc.capture_parts(manager, span, "test", parts, direction="input", + trace_configs=(TraceConfig(), OIConfig())) == {} + assert mc.capture_parts(manager, span, "test", parts, direction="input", + trace_configs=(TraceConfig(),)) + flush(manager) + assert f01.payloads == [IMAGE] + + +def test_looks_base64(): + assert mc.looks_base64(b64(IMAGE).encode()) + assert not mc.looks_base64(IMAGE) + assert not mc.looks_base64(b"%PDF-1.4 abc") + assert not mc.looks_base64("not bytes") + + +@pytest.mark.parametrize("value", [None, 7, b"", "data:image/png;base64,@@@@"]) +def test_malformed_sources_capture_nothing(f01, spans, value): + tracer_provider, _ = spans + manager = f01.manager() + with tracer_provider.get_tracer("t").start_as_current_span("s") as span: + assert _inline(manager, span, value=value) is None + flush(manager) + assert f01.payloads == [] diff --git a/python/tests/attachments_frameworks/test_owner.py b/python/tests/attachments_frameworks/test_owner.py new file mode 100644 index 000000000..d1d763e00 --- /dev/null +++ b/python/tests/attachments_frameworks/test_owner.py @@ -0,0 +1,183 @@ +"""One model call, one owner: the core owner-mark helpers (no framework installed).""" + +import asyncio + +from conftest import b64, flush, png, refs +from fi_instrumentation.attachments import AttachmentConfig, create_attachment_manager +from fi_instrumentation.attachments import media_capture as mc +from opentelemetry import context as otel_context + +IMAGE = png(seed=21) +URI = "data:image/png;base64," + b64(IMAGE) + + +def _part(): + return mc.MediaPart("inline", lambda: URI, mc.IMAGE_MIMES, "model_input", 0, 0, role="user", raw=False) + + +def _capture(manager, span, namespace="fw"): + return mc.capture_parts(manager, span, namespace, [_part()], direction="input") + + +def _under(value): + return otel_context.attach(otel_context.set_value(mc.OWNER_KEY, value)) + + +def test_call_owner_is_gated_on_an_enabled_manager_and_an_unowned_call(f01): + assert mc.call_owner(None, "fw") is None + assert mc.call_owner(create_attachment_manager(config=AttachmentConfig(enabled=False)), "fw") is None + manager = f01.manager() + owner = mc.call_owner(manager, "fw") + assert isinstance(owner, mc.CallOwner) and owner.namespace == "fw" + token = _under(True) # an outer adapter or traceai.with_attachment_ownership + try: + assert mc.call_owner(manager, "fw") is None + finally: + otel_context.detach(token) + + +def test_each_provider_side_check_honours_the_mark(f01, spans): + """A provider instrumentor skips an owned call twice over: its Observation is unusable AND its + local capture policy is unrequested. Each check is pinned on its own (one alone would hide the other).""" + tracer_provider, _ = spans + manager = f01.manager() + with tracer_provider.get_tracer("t").start_as_current_span("provider") as span: + assert mc.Observation(manager, span, "openai").usable + assert mc.local_capture_policy(direction="input").requested + with mc.owner_scope(mc.CallOwner("fw")): + assert not mc.Observation(manager, span, "openai").usable + assert mc.local_capture_policy(direction="input").requested is False + + +def test_owned_call_marks_only_the_enclosed_block(): + assert not mc.owned_elsewhere() + with mc.owner_scope(mc.CallOwner("fw")): + assert mc.owned_elsewhere() + assert not mc.owned_elsewhere() + try: + with mc.owner_scope(mc.CallOwner("fw")): + raise RuntimeError("provider failed") + except RuntimeError: + pass + assert not mc.owned_elsewhere() + with mc.owner_scope(None): + assert not mc.owned_elsewhere() + + +def test_capture_parts_captures_inside_its_own_mark_only(f01, spans): + tracer_provider, exporter = spans + manager = f01.manager() + tracer = tracer_provider.get_tracer("t") + results = {} + for label, mark, namespace in (("own", mc.CallOwner("fw"), "fw"), ("other", mc.CallOwner("other"), "fw"), + ("anonymous", mc.CallOwner(None), "fw"), ("user", True, "fw")): + with tracer.start_as_current_span(label) as span: + token = _under(mark) + try: + results[label] = _capture(manager, span, namespace) + # The adapter's capture restores the mark for the provider call that follows. + assert otel_context.get_value(mc.OWNER_KEY) is mark + finally: + otel_context.detach(token) + flush(manager) + assert len(results["own"]) == 1 + assert results["other"] == {} and results["anonymous"] == {} and results["user"] == {} + assert f01.payloads == [IMAGE] + assert [s.name for s in exporter.get_finished_spans() if refs(s)] == ["own"] + + +def test_owned_stream_marks_each_advance_never_the_callers_code(): + seen = [] + + def stream(): + seen.append(mc.owned_elsewhere()) + yield 1 + seen.append(mc.owned_elsewhere()) + yield 2 + + owner = mc.CallOwner("fw") + wrapped = mc.owned_stream(stream(), owner) + caller = [] + for _ in wrapped: + caller.append(mc.owned_elsewhere()) + assert seen == [True, True] and caller == [False, False] + + closed = [] + + def abandoned(): + try: + yield 1 + yield 2 + finally: + closed.append(mc.owned_elsewhere()) + + wrapped = mc.owned_stream(abandoned(), owner) + next(wrapped) + assert not mc.owned_elsewhere() # suspended mid-call: the caller's context is unmarked + wrapped.close() + assert closed == [True] and not mc.owned_elsewhere() + assert mc.owned_stream(stream(), None) is not None + assert mc.owned_stream([1], owner) == [1] # not a generator: returned as is + + +def test_owned_async_stream_marks_each_awaited_step_only(): + seen, caller = [], [] + + async def stream(): + seen.append(mc.owned_elsewhere()) + yield 1 + seen.append(mc.owned_elsewhere()) + yield 2 + + async def run(): + async for _ in mc.owned_stream(stream(), mc.CallOwner("fw")): + caller.append(mc.owned_elsewhere()) + + asyncio.run(run()) + assert seen == [True, True] and caller == [False, False] + + +def test_a_proxy_with_both_protocols_is_treated_as_async(): + """LlamaIndex's async stream proxy also defines send/throw/close: it must still be iterated async.""" + + async def stream(): + yield mc.owned_elsewhere() + + class BothProtocols: + def __init__(self, inner): + self.inner = inner + + def __iter__(self): + return self + + def __next__(self): + raise StopIteration + + def send(self, value): + raise StopIteration + + def throw(self, *args): + raise StopIteration + + def close(self): + pass + + def __aiter__(self): + return self + + def __anext__(self): + return self.inner.__anext__() + + def asend(self, value): + return self.inner.asend(value) + + def athrow(self, *args): + return self.inner.athrow(*args) + + def aclose(self): + return self.inner.aclose() + + async def run(): + return [item async for item in mc.owned_stream(BothProtocols(stream()), mc.CallOwner("fw"))] + + assert asyncio.run(run()) == [True] diff --git a/python/tests/attachments_frameworks/test_owner_gate.py b/python/tests/attachments_frameworks/test_owner_gate.py new file mode 100644 index 000000000..96d6b6682 --- /dev/null +++ b/python/tests/attachments_frameworks/test_owner_gate.py @@ -0,0 +1,198 @@ +"""The owner mark is set only for a call in which the adapter observes at least one input media part. + +A call with no observed part (text only, an Agno ``filepath``, a LlamaIndex ``path``, a provider +file ID) runs unmarked, so a provider instrumentor beneath the framework still captures what only +the SDK sees. Every gate fails open: an error means no mark, never a suppressed provider. +""" + +import pytest + +from conftest import b64, png +from fi_instrumentation import TraceConfig +from fi_instrumentation.attachments import media_capture as mc + +IMAGE = png(seed=41) +URI = "data:image/png;base64," + b64(IMAGE) + + +# ------------------------------------------------------------------ LangChain +@pytest.fixture +def langchain_probe(spans): + pytest.importorskip("langchain_core") + pytest.importorskip("traceai_langchain") + from langchain_core.language_models.chat_models import BaseChatModel + from langchain_core.messages import AIMessage + from langchain_core.outputs import ChatGeneration, ChatResult + from traceai_langchain import LangChainInstrumentor + + seen = [] + + class Probe(BaseChatModel): + """Records whether the provider call beneath LangChain runs under the owner mark.""" + + @property + def _llm_type(self) -> str: + return "owner-probe" + + def _generate(self, messages, stop=None, run_manager=None, **kwargs): + seen.append(mc.owned_elsewhere()) + return ChatResult(generations=[ChatGeneration(message=AIMessage(content="ok"))]) + + tracer_provider, _ = spans + state = {} + + def install(manager): + LangChainInstrumentor().instrument(tracer_provider=tracer_provider, attachments=manager, + config=TraceConfig()) + state["on"] = True + return Probe(), seen + + yield install + if state.get("on"): + LangChainInstrumentor().uninstrument() + + +def test_langchain_marks_only_a_call_with_an_observed_part(f01, langchain_probe): + from langchain_core.messages import HumanMessage + + model, seen = langchain_probe(f01.manager()) + image = HumanMessage(content=[{"type": "text", "text": "what"}, {"type": "image_url", "image_url": {"url": URI}}]) + provider_file = HumanMessage(content=[{"type": "file", "file": {"file_id": "file-synthetic"}}]) + model.invoke("text only") + model.invoke([HumanMessage(content="text only")]) + model.invoke([provider_file]) + model.invoke([image]) + for _ in model.stream("text only"): + pass + for _ in model.stream([image]): + pass + assert seen == [False, False, False, True, False, True] + + +def test_langchain_gate_fails_open(): + pytest.importorskip("traceai_langchain") + from traceai_langchain._attachments import observes_langchain_input + + class Broken: + def _convert_input(self, value): + raise RuntimeError("cannot convert") + + assert observes_langchain_input("stream", Broken(), ([{"role": "user"}],), {}) is False + assert observes_langchain_input("generate", None, (object(),), {}) is False + assert observes_langchain_input("generate", None, (), {}) is False + + +# ------------------------------------------------------------------ LlamaIndex +@pytest.fixture +def llamaindex_probe(spans): + pytest.importorskip("llama_index.core") + pytest.importorskip("traceai_llamaindex") + from llama_index.core.base.llms.types import ChatMessage, ChatResponse, CompletionResponse, LLMMetadata + from llama_index.core.llms.callbacks import llm_chat_callback, llm_completion_callback + from llama_index.core.llms.custom import CustomLLM + from traceai_llamaindex import LlamaIndexInstrumentor + + seen = [] + + class Probe(CustomLLM): + @property + def metadata(self) -> LLMMetadata: + return LLMMetadata(model_name="owner-probe", is_chat_model=True) + + @llm_chat_callback() + def chat(self, messages, **kwargs): + seen.append(mc.owned_elsewhere()) + return ChatResponse(message=ChatMessage(role="assistant", content="ok")) + + @llm_completion_callback() + def complete(self, prompt, formatted=False, **kwargs): + return CompletionResponse(text="ok") + + @llm_completion_callback() + def stream_complete(self, prompt, formatted=False, **kwargs): + yield CompletionResponse(text="ok", delta="ok") + + tracer_provider, _ = spans + state = {} + + def install(manager): + LlamaIndexInstrumentor().instrument(tracer_provider=tracer_provider, attachments=manager, + config=TraceConfig()) + state["on"] = True + return Probe(), seen + + yield install + if state.get("on"): + LlamaIndexInstrumentor().uninstrument() + + +def test_llamaindex_marks_only_a_call_with_an_observed_part(f01, llamaindex_probe, tmp_path): + from llama_index.core.base.llms.types import ChatMessage, ImageBlock, TextBlock + + llm, seen = llamaindex_probe(f01.manager()) + path = tmp_path / "source.png" + path.write_bytes(IMAGE) + llm.chat([ChatMessage(role="user", content="text only")]) + llm.chat([ChatMessage(role="user", blocks=[TextBlock(text="what"), + ImageBlock(path=path, image_mimetype="image/png")])]) + llm.chat([ChatMessage(role="user", blocks=[ImageBlock(image=IMAGE, image_mimetype="image/png")])]) + llm.chat(messages=[ChatMessage(role="user", blocks=[ImageBlock(image=IMAGE, image_mimetype="image/png")])]) + assert seen == [False, False, True, True] + + +def test_llamaindex_gate_fails_open(): + pytest.importorskip("traceai_llamaindex") + from traceai_llamaindex._attachments import observes_llamaindex_messages + + assert observes_llamaindex_messages(None) is False + assert observes_llamaindex_messages("text") is False + assert observes_llamaindex_messages([object()]) is False + + +# ------------------------------------------------------------------ Agno +def test_agno_gate_skips_a_filepath_only_call(tmp_path): + pytest.importorskip("agno") + pytest.importorskip("traceai_agno") + from agno.media import Image + from agno.models.message import Message + from traceai_agno._attachments import observes_agno_input + + path = tmp_path / "source.png" + path.write_bytes(IMAGE) + assert observes_agno_input([Message(role="user", content="text only")]) is False + assert observes_agno_input([Message(role="user", content="x", images=[Image(filepath=str(path))])]) is False + assert observes_agno_input([Message(role="user", content="x", images=[Image(content=IMAGE, format="png")])]) + assert observes_agno_input(None) is False and observes_agno_input([object()]) is False + + +# ------------------------------------------------------------------ Pydantic AI +def test_pydantic_ai_gate_marks_only_a_request_with_an_observed_part(): + pytest.importorskip("pydantic_ai") + pytest.importorskip("traceai_pydantic_ai") + from pydantic_ai import BinaryContent + from pydantic_ai.messages import ModelRequest, UserPromptPart + from traceai_pydantic_ai._attachments import PydanticAIMedia + + media = PydanticAIMedia(None, None) + text = ModelRequest(parts=[UserPromptPart(content="text only")]) + image = ModelRequest(parts=[UserPromptPart(content=["what", BinaryContent(data=IMAGE, media_type="image/png")])]) + assert media.observes_request([text]) is False + assert media.observes_request([text, image]) is True + assert media.observes_request(None) is False and media.observes_request([object()]) is False + + +def test_pydantic_ai_call_owner_follows_the_gate(f01): + """The wrapper marks only a request the adapter observes; a text-only request runs unmarked.""" + pytest.importorskip("pydantic_ai") + pytest.importorskip("traceai_pydantic_ai") + from pydantic_ai import BinaryContent + from pydantic_ai.messages import ModelRequest, UserPromptPart + from traceai_pydantic_ai._attachments import PydanticAIMedia + from traceai_pydantic_ai._execution_wrappers import _call_owner + + media = PydanticAIMedia(f01.manager(), None) + text = ModelRequest(parts=[UserPromptPart(content="text only")]) + image = ModelRequest(parts=[UserPromptPart(content=["what", BinaryContent(data=IMAGE, media_type="image/png")])]) + assert _call_owner(media, [text]) is None + assert isinstance(_call_owner(media, [text, image]), mc.CallOwner) + assert _call_owner(None, [image]) is None diff --git a/python/tests/attachments_frameworks/test_provider_and_framework.py b/python/tests/attachments_frameworks/test_provider_and_framework.py new file mode 100644 index 000000000..a50f1935a --- /dev/null +++ b/python/tests/attachments_frameworks/test_provider_and_framework.py @@ -0,0 +1,546 @@ +"""Framework + OpenAI instrumentor on ONE manager: one model call, one owner. + +Each test drives the real framework against the real OpenAI SDK (loopback provider). When a +framework adapter captures a model call's media it runs the downstream SDK call under the owner +mark (``media_capture.OWNER_KEY``), so ``traceai_openai`` uploads nothing for that call: each +part is uploaded exactly once, under the framework span. DIFFERENT calls still each own their +occurrence, even with identical bytes (byte deduplication is server-side). With attachments off +for the framework nothing is marked and the provider captures as before. + +The mark is set only for a call in which the adapter observes at least one input media part. +Media the adapter never reads (an Agno ``filepath``, a LlamaIndex ``path``) is inlined by the +SDK integration as a data URI, so ``traceai_openai`` must still capture it on the provider span. +""" + +import asyncio +import json + +import pytest + +pytest.importorskip("openai") +pytest.importorskip("traceai_openai") + +from conftest import PDF, assert_uris_resolve, b64, flush, png, refs # noqa: E402 +from fi_instrumentation import TraceConfig # noqa: E402 +from traceai_openai import OpenAIInstrumentor # noqa: E402 + +IMAGE = png(seed=31) +URI = "data:image/png;base64," + b64(IMAGE) +PATH_IMAGE = png(seed=32) # only ever read from disk by the framework's SDK integration +OPENAI_SCOPE = "traceai_openai" + + +@pytest.fixture +def openai_on(spans): + tracer_provider, exporter = spans + state = {} + + def install(manager, config=None): + OpenAIInstrumentor().instrument(tracer_provider=tracer_provider, attachments=manager, + config=config or TraceConfig()) + state["on"] = True + + yield install + if state.get("on"): + OpenAIInstrumentor().uninstrument() + + +def sent_image(provider, count=1): + """The provider actually received the image ``count`` times (the same bytes, not a copy).""" + seen = sum(1 for path, body, raw in provider.requests + if path.endswith("/chat/completions") and b64(IMAGE) in json.dumps(body)) + return seen >= count + + +def owners(exporter): + return [s for s in exporter.get_finished_spans() if refs(s)] + + +def is_provider_span(span): + return getattr(span.instrumentation_scope, "name", "") == OPENAI_SCOPE + + +def assert_one_owner(f01, manager, provider, exporter): + """Exactly one upload, parented by and referenced on the framework span; the provider span holds none.""" + flush(manager) + assert sent_image(provider), "the framework did not send the image to the provider" + assert f01.payloads == [IMAGE], len(f01.payloads) + (owner,) = owners(exporter) + assert not is_provider_span(owner), owner.name + assert any(is_provider_span(s) for s in exporter.get_finished_spans()), "no provider span: not a 2-layer call" + (ref,) = refs(owner) + (upload,) = f01.uploads + assert upload["occurrence_id"] == ref["occurrence_id"] + assert upload["association"]["parent"]["span_id"] == format(owner.context.span_id, "016x") + assert_uris_resolve(exporter) + return owner + + +def direct_openai_call(provider): + import openai + + client = openai.OpenAI(api_key="sk-synthetic", base_url=provider.url + "/v1", max_retries=0) + client.chat.completions.create(model="m", messages=[{"role": "user", "content": [ + {"type": "image_url", "image_url": {"url": URI}}]}]) + + +async def _drain(stream): + async for _ in stream: + pass + + +# ------------------------------------------------------------------ framework drivers +class LangChain: + name = "langchain" + modes = ("sync", "stream", "async", "astream") + + def __init__(self, provider, tracer_provider): + pytest.importorskip("langchain_openai") + self.provider, self.tracer_provider = provider, tracer_provider + + def instrument(self, manager, config=None): + from traceai_langchain import LangChainInstrumentor + + kwargs = {"attachments": manager} if manager is not None else {} + LangChainInstrumentor().instrument(tracer_provider=self.tracer_provider, config=config or TraceConfig(), + **kwargs) + + def uninstrument(self): + from traceai_langchain import LangChainInstrumentor + + LangChainInstrumentor().uninstrument() + + def _model(self): + from langchain_openai import ChatOpenAI + + return ChatOpenAI(model="gpt-4o-mini", api_key="sk-synthetic", base_url=self.provider.url + "/v1", + max_retries=0) + + def _messages(self): + from langchain_core.messages import HumanMessage + + return [HumanMessage(content=[{"type": "text", "text": "what"}, + {"type": "image_url", "image_url": {"url": URI}}])] + + def call(self, mode="sync"): + model, messages = self._model(), self._messages() + if mode == "sync": + model.invoke(messages) + elif mode == "stream": + for _ in model.stream(messages): + pass + elif mode == "async": + asyncio.run(model.ainvoke(messages)) + else: + asyncio.run(_drain(model.astream(messages))) + + def open_stream(self): + stream = self._model().stream(self._messages()) + next(stream) # the provider request has been made; the stream is suspended mid-call + return stream + + +class LlamaIndex: + name = "llamaindex" + modes = ("sync", "stream", "async", "astream") + + def __init__(self, provider, tracer_provider): + pytest.importorskip("llama_index.llms.openai") + self.provider, self.tracer_provider = provider, tracer_provider + + def instrument(self, manager, config=None): + from traceai_llamaindex import LlamaIndexInstrumentor + + kwargs = {"attachments": manager} if manager is not None else {} + LlamaIndexInstrumentor().instrument(tracer_provider=self.tracer_provider, config=config or TraceConfig(), + **kwargs) + + def uninstrument(self): + from traceai_llamaindex import LlamaIndexInstrumentor + + LlamaIndexInstrumentor().uninstrument() + + def _llm(self): + from llama_index.llms.openai import OpenAI + + return OpenAI(model="gpt-4o-mini", api_key="sk-synthetic", api_base=self.provider.url + "/v1", + max_retries=0) + + def _messages(self): + from llama_index.core.base.llms.types import ChatMessage, ImageBlock, TextBlock + + return [ChatMessage(role="user", blocks=[TextBlock(text="what"), + ImageBlock(image=IMAGE, image_mimetype="image/png")])] + + def call(self, mode="sync"): + llm, messages = self._llm(), self._messages() + if mode == "sync": + llm.chat(messages) + elif mode == "stream": + for _ in llm.stream_chat(messages): + pass + elif mode == "async": + asyncio.run(llm.achat(messages)) + else: + async def run(): + await _drain(await llm.astream_chat(messages)) + asyncio.run(run()) + + def open_stream(self): + stream = self._llm().stream_chat(self._messages()) + next(stream) + return stream + + def path_call(self, path, kind, with_content=False): + """One chat call with a ``path`` block (never read by the adapter), optionally after a content block.""" + from llama_index.core.base.llms.types import ChatMessage, DocumentBlock, ImageBlock, TextBlock + + blocks = [TextBlock(text="what")] + if with_content: + blocks.append(ImageBlock(image=IMAGE, image_mimetype="image/png")) + blocks.append(ImageBlock(path=path, image_mimetype="image/png") if kind == "image" + else DocumentBlock(path=path, document_mimetype="application/pdf")) + self._llm().chat([ChatMessage(role="user", blocks=blocks)]) + + +class Agno: + name = "agno" + modes = ("sync", "stream") + + def __init__(self, provider, tracer_provider): + pytest.importorskip("agno") + pytest.importorskip("openinference.instrumentation.agno") + self.provider, self.tracer_provider = provider, tracer_provider + + def instrument(self, manager, config=None): + from traceai_agno import configure_agno_tracing + + assert config is None, "Agno takes an OpenInference TraceConfig; not exercised here" + kwargs = {"attachments": manager} if manager is not None else {} + configure_agno_tracing(tracer_provider=self.tracer_provider, **kwargs) + + def uninstrument(self): + from traceai_agno import AgnoInstrumentorWrapper + + AgnoInstrumentorWrapper().uninstrument() + + def _agent(self): + from agno.agent import Agent + from agno.models.openai import OpenAIChat + + model = OpenAIChat(id="gpt-4o-mini", api_key="sk-synthetic", base_url=self.provider.url + "/v1", + max_retries=0) + try: + return Agent(model=model, telemetry=False) + except TypeError: + return Agent(model=model) + + def call(self, mode="sync"): + from agno.media import Image + + images = [Image(content=IMAGE, format="png")] + if mode == "sync": + self._agent().run("what is this", images=images) + else: + for _ in self._agent().run("what is this", images=images, stream=True): + pass + + def path_call(self, path, kind, with_content=False): + """One model call with a ``filepath`` source (never read by the adapter), optionally after content.""" + from agno.media import File, Image + + if kind == "image": + images = ([Image(content=IMAGE, format="png")] if with_content else []) + [Image(filepath=str(path))] + self._agent().run("what is this", images=images) + else: + images = [Image(content=IMAGE, format="png")] if with_content else None + self._agent().run("what is this", images=images, files=[File(filepath=str(path))]) + + +class PydanticAI: + name = "pydantic_ai" + modes = ("sync", "stream") + + def __init__(self, provider, tracer_provider): + pytest.importorskip("pydantic_ai") + self.provider, self.tracer_provider = provider, tracer_provider + + def instrument(self, manager, config=None): + from traceai_pydantic_ai import PydanticAIInstrumentor + + kwargs = {"attachments": manager} if manager is not None else {} + # skip_dep_check: CI installs pydantic-ai-slim, which the instrumentor's "pydantic-ai" requirement + # does not name. + PydanticAIInstrumentor().instrument(tracer_provider=self.tracer_provider, config=config or TraceConfig(), + skip_dep_check=True, **kwargs) + + def uninstrument(self): + from traceai_pydantic_ai import PydanticAIInstrumentor + + PydanticAIInstrumentor().uninstrument() + + def _agent(self): + from pydantic_ai import Agent + + try: + from pydantic_ai.models.openai import OpenAIChatModel as Model + except ImportError: + from pydantic_ai.models.openai import OpenAIModel as Model + from pydantic_ai.providers.openai import OpenAIProvider + + return Agent(Model("gpt-4o-mini", provider=OpenAIProvider(base_url=self.provider.url + "/v1", + api_key="sk-synthetic"))) + + def _prompt(self): + from pydantic_ai import BinaryContent + + return ["what is this", BinaryContent(data=IMAGE, media_type="image/png")] + + def call(self, mode="sync", body=None): + if mode == "sync": + self._agent().run_sync(self._prompt()) + return + + async def run(): + async with self._agent().run_stream(self._prompt()): + if body is not None: + body() # the caller's own code inside the stream's ``async with`` + asyncio.run(run()) + + +FRAMEWORKS = {cls.name: cls for cls in (LangChain, LlamaIndex, Agno, PydanticAI)} +CALLS = [(name, mode) for name, cls in FRAMEWORKS.items() for mode in cls.modes] + + +@pytest.fixture +def framework(request, provider, spans): + tracer_provider, _ = spans + driver = FRAMEWORKS[request.param](provider, tracer_provider) + state = {} + original = driver.instrument + + def instrument(*args, **kwargs): + original(*args, **kwargs) + state["on"] = True + + driver.instrument = instrument + yield driver + if state.get("on"): + driver.uninstrument() + + +# ------------------------------------------------------------------ one call, one owner +@pytest.mark.parametrize("framework, mode", CALLS, indirect=["framework"]) +def test_framework_and_provider_one_call_uploads_each_part_once(f01, provider, spans, openai_on, framework, mode): + """Was 2 (framework span + provider span); the provider instrumentor now skips the owned call.""" + _, exporter = spans + manager = f01.manager() + framework.instrument(manager) + openai_on(manager) + framework.call(mode) + assert_one_owner(f01, manager, provider, exporter) + + +@pytest.mark.parametrize("framework", list(FRAMEWORKS), indirect=True) +def test_framework_alone_still_uploads_inside_its_own_mark(f01, provider, spans, framework): + """Control: the adapter's own capture must not be suppressed by the mark it sets.""" + _, exporter = spans + manager = f01.manager() + framework.instrument(manager) + framework.call() + flush(manager) + assert sent_image(provider) + assert f01.payloads == [IMAGE] + (owner,) = owners(exporter) + assert not is_provider_span(owner) + + +@pytest.mark.parametrize("framework", list(FRAMEWORKS), indirect=True) +def test_framework_attachments_off_marks_nothing(f01, provider, spans, openai_on, framework): + """Attachments off for the framework: no mark, so the provider captures exactly as before.""" + _, exporter = spans + manager = f01.manager() + framework.instrument(None) + openai_on(manager) + framework.call() + flush(manager) + assert sent_image(provider) + assert f01.payloads == [IMAGE] + (owner,) = owners(exporter) + assert is_provider_span(owner) + + +# ------------------------------------------------------------------ different calls +@pytest.mark.parametrize("framework", list(FRAMEWORKS), indirect=True) +def test_same_bytes_in_two_framework_calls_are_two_occurrences(f01, provider, spans, openai_on, framework): + _, exporter = spans + manager = f01.manager() + framework.instrument(manager) + openai_on(manager) + framework.call() + framework.call() + flush(manager) + assert sent_image(provider, 2) + assert f01.payloads == [IMAGE, IMAGE] + found = owners(exporter) + assert len(found) == 2 and not any(is_provider_span(s) for s in found) + assert len({r["occurrence_id"] for s in found for r in refs(s)}) == 2 + assert_uris_resolve(exporter) + + +@pytest.mark.parametrize("framework", list(FRAMEWORKS), indirect=True) +def test_framework_call_then_direct_provider_call_each_own_one(f01, provider, spans, openai_on, framework): + """The mark ends with the framework call: a later direct SDK call with the same bytes uploads its own.""" + _, exporter = spans + manager = f01.manager() + framework.instrument(manager) + openai_on(manager) + framework.call() + direct_openai_call(provider) + flush(manager) + assert f01.payloads == [IMAGE, IMAGE] + found = owners(exporter) + assert sorted(is_provider_span(s) for s in found) == [False, True] + assert_uris_resolve(exporter) + + +@pytest.mark.parametrize("framework", ["langchain", "llamaindex"], indirect=True) +def test_an_abandoned_framework_stream_leaves_no_mark(f01, provider, spans, openai_on, framework): + """A stream suspended mid-call (or dropped) never leaves the mark on the caller's context.""" + _, exporter = spans + manager = f01.manager() + framework.instrument(manager) + openai_on(manager) + stream = framework.open_stream() + direct_openai_call(provider) # while the framework stream is suspended + stream.close() + direct_openai_call(provider) # after it was abandoned + flush(manager) + assert f01.payloads == [IMAGE, IMAGE, IMAGE] + found = owners(exporter) + assert sorted(is_provider_span(s) for s in found) == [False, True, True] + + +@pytest.mark.parametrize("framework", ["pydantic_ai"], indirect=True) +def test_the_callers_stream_body_is_not_marked(f01, provider, spans, openai_on, framework): + """Pydantic AI marks entering and exiting the stream, never the caller's ``async with`` body.""" + _, exporter = spans + manager = f01.manager() + framework.instrument(manager) + openai_on(manager) + framework.call("stream", body=lambda: direct_openai_call(provider)) + flush(manager) + assert f01.payloads == [IMAGE, IMAGE] + found = owners(exporter) + assert sorted(is_provider_span(s) for s in found) == [False, True] + + +# ------------------------------------------------------------------ media only the SDK reads +PATH_SOURCES = [("agno", "image"), ("agno", "file"), ("llamaindex", "image"), ("llamaindex", "document")] + + +def sent(provider, data): + return any(path.endswith("/chat/completions") and b64(data) in json.dumps(body) + for path, body, raw in provider.requests) + + +def _path(tmp_path, kind): + data = PATH_IMAGE if kind == "image" else PDF + path = tmp_path / ("source.png" if kind == "image" else "source.pdf") + path.write_bytes(data) + return path, data + + +@pytest.mark.parametrize("framework, kind", PATH_SOURCES, indirect=["framework"]) +def test_a_source_only_the_sdk_reads_is_still_captured_by_the_provider(f01, provider, spans, openai_on, + framework, kind, tmp_path): + """The adapter never opens a path, the SDK integration inlines it: no mark, ONE upload, on the provider span. + + With the mark set on every call (16311a37..1080f58d) this was 0: nobody captured it. + """ + _, exporter = spans + manager = f01.manager() + framework.instrument(manager) + openai_on(manager) + path, data = _path(tmp_path, kind) + framework.path_call(path, kind) + flush(manager) + assert sent(provider, data), "the SDK integration did not inline the path source" + assert f01.payloads == [data], len(f01.payloads) + (owner,) = owners(exporter) + assert is_provider_span(owner), owner.name + (ref,) = refs(owner) + (upload,) = f01.uploads + assert upload["occurrence_id"] == ref["occurrence_id"] + assert upload["association"]["parent"]["span_id"] == format(owner.context.span_id, "016x") + assert_uris_resolve(exporter) + + +@pytest.mark.parametrize("framework, kind", PATH_SOURCES, indirect=["framework"]) +def test_the_framework_alone_never_reads_a_path_source(f01, provider, spans, framework, kind, tmp_path): + """Negative control for the row above: without traceai_openai the path source is uploaded by nobody.""" + _, exporter = spans + manager = f01.manager() + framework.instrument(manager) + path, data = _path(tmp_path, kind) + framework.path_call(path, kind) + flush(manager) + assert sent(provider, data) + assert f01.payloads == [] and owners(exporter) == [] + + +@pytest.mark.parametrize("framework", ["agno", "llamaindex"], indirect=True) +def test_residual_a_call_mixing_an_observed_part_and_a_path_part_is_marked(f01, provider, spans, openai_on, + framework, tmp_path): + """Disclosed residual: the adapter observes the content part, so the WHOLE call is marked and the + provider skips the path part too. One upload (the content part), on the framework span.""" + _, exporter = spans + manager = f01.manager() + framework.instrument(manager) + openai_on(manager) + path, data = _path(tmp_path, "image") + framework.path_call(path, "image", with_content=True) + flush(manager) + assert sent(provider, IMAGE) and sent(provider, data) + assert f01.payloads == [IMAGE], len(f01.payloads) + (owner,) = owners(exporter) + assert not is_provider_span(owner), owner.name + + +# ------------------------------------------------------------------ the mark carries the framework's deny +@pytest.mark.parametrize("framework", ["langchain", "llamaindex", "pydantic_ai"], indirect=True) +def test_framework_hide_inputs_is_not_bypassed_through_the_provider(f01, provider, spans, openai_on, framework): + """The framework hides its input: it uploads nothing, and the provider beneath it does not either.""" + _, exporter = spans + manager = f01.manager() + framework.instrument(manager, config=TraceConfig(hide_inputs=True)) + openai_on(manager) + framework.call() + flush(manager) + assert sent_image(provider) + assert f01.payloads == [] + assert owners(exporter) == [] + + +def test_provider_alone_two_calls_same_image_each_own_their_occurrence(f01, provider, spans, openai_on): + """Provider only (no framework): two independent calls with the same image on one manager. + + Each call is its own trace. Each span uploads, references its own occurrence, and its + telemetry URI points at that occurrence (never at the other trace's). + """ + _, exporter = spans + manager = f01.manager() + openai_on(manager) + for _ in range(2): + direct_openai_call(provider) + flush(manager) + assert f01.payloads == [IMAGE, IMAGE] + found = owners(exporter) + assert len(found) == 2 and len({s.context.trace_id for s in found}) == 2 + assert len({r["occurrence_id"] for s in found for r in refs(s)}) == 2 + # A span may project its URI into several attributes; each must resolve to its OWN occurrence. + assert assert_uris_resolve(exporter) >= 2 + for span in found: + (own,) = [r["occurrence_id"] for r in refs(span)] + assert any("fi-attachment:" + own in value for value in (span.attributes or {}).values() + if isinstance(value, str)), span.name diff --git a/python/tests/attachments_frameworks/test_pydantic_ai_attachments.py b/python/tests/attachments_frameworks/test_pydantic_ai_attachments.py new file mode 100644 index 000000000..d8927f608 --- /dev/null +++ b/python/tests/attachments_frameworks/test_pydantic_ai_attachments.py @@ -0,0 +1,208 @@ +"""Real Pydantic AI agent runs (FunctionModel) -> traceai_pydantic_ai -> loopback F01.""" + +import http.server +import threading + +import pytest + +pytest.importorskip("pydantic_ai") +pytest.importorskip("traceai_pydantic_ai") + +from conftest import PDF, assert_uris_resolve, b64, flush, png, refs, wav # noqa: E402 +from fi_instrumentation import TraceConfig # noqa: E402 +from fi_instrumentation.attachments import AttachmentConfig, create_attachment_manager # noqa: E402 +from pydantic_ai import Agent, BinaryContent, ImageUrl # noqa: E402 +from pydantic_ai.messages import ModelResponse, TextPart, ToolCallPart # noqa: E402 +from pydantic_ai.models.function import AgentInfo, FunctionModel # noqa: E402 +from traceai_pydantic_ai import PydanticAIInstrumentor # noqa: E402 + +try: + from pydantic_ai.messages import FilePart +except ImportError: # older SDKs have no generated-file parts + FilePart = None + +IMAGE = png(seed=61) +GENERATED = png(seed=62) +SCREEN = png(seed=63) + + +def echo(messages, info: AgentInfo): + return ModelResponse(parts=[TextPart("ok")]) + + +@pytest.fixture +def instrument(spans): + tracer_provider, exporter = spans + state = {} + + def install(manager, config=None): + # skip_dep_check: pydantic-ai-slim does not satisfy the instrumentor's "pydantic-ai" requirement. + PydanticAIInstrumentor().instrument(tracer_provider=tracer_provider, attachments=manager, + config=config or TraceConfig(), skip_dep_check=True) + state["on"] = True + return exporter + + yield install + if state.get("on"): + PydanticAIInstrumentor().uninstrument() + + +def request_spans(exporter): + return [s for s in exporter.get_finished_spans() if "model" in s.name.lower() and "request" in s.name.lower()] + + +def test_binary_content_prompt(f01, instrument): + manager = f01.manager() + exporter = instrument(manager) + Agent(FunctionModel(echo)).run_sync(["what is this", BinaryContent(data=IMAGE, media_type="image/png"), + BinaryContent(data=PDF, media_type="application/pdf"), + BinaryContent(data=wav(), media_type="audio/wav")]) + flush(manager) + assert sorted(f01.payloads) == sorted([IMAGE, PDF, wav()]) + purposes = {body["mime_type"]: body["purpose"] for body in f01.uploads} + assert purposes == {"image/png": "model_input", "application/pdf": "document", "audio/wav": "model_input"} + owners = [s for s in request_spans(exporter) if refs(s)] + assert len(owners) == 1 and len(refs(owners[0])) == 3 + + +def _screenshot_agent(): + calls = {"n": 0} + + def model(messages, info: AgentInfo): + calls["n"] += 1 + if calls["n"] == 1: + return ModelResponse(parts=[ToolCallPart(tool_name="screenshot", args={}, tool_call_id="call_1")]) + return ModelResponse(parts=[TextPart("done")]) + + agent = Agent(FunctionModel(model)) + + @agent.tool_plain + def screenshot() -> BinaryContent: + return BinaryContent(data=SCREEN, media_type="image/png") + + try: + agent.run_sync("take a screenshot") + except Exception as exc: # noqa: BLE001 + # pydantic-ai 0.1.0's FunctionModel cannot JSON-serialize a binary tool return when it estimates + # usage for the second request; the tool span has already finished by then. + if "serializ" not in str(exc).lower(): + raise + + +def tool_spans(exporter): + return [s for s in exporter.get_finished_spans() if s.name.startswith("pydantic_ai.tool.call")] + + +def test_tool_result_binary_is_owned_by_each_observing_span(f01, instrument): + manager = f01.manager() + exporter = instrument(manager) + _screenshot_agent() + flush(manager) + (tool,) = [s for s in tool_spans(exporter) if refs(s)] + (ref,) = refs(tool) + # Span refs carry purpose and indices only; tool_call_id travels in the upload request. + assert ref["purpose"] == "tool_result" + by_occurrence = {b["occurrence_id"]: b for b in f01.uploads} + body = by_occurrence[ref["occurrence_id"]] + assert body["association"]["parent"]["span_id"] == format(tool.context.span_id, "016x") + assert body["purpose"] == "tool_result" and body["direction"] == "input" and body.get("tool_call_id") == "call_1" + # Newer SDKs reach the second model request, which re-sends the tool return as model input: that span + # owns ANOTHER occurrence of the same bytes (never the tool span's). + owners = [s for s in exporter.get_finished_spans() if refs(s)] + assert len(f01.payloads) == len(owners) and set(f01.payloads) == {SCREEN} + for span in owners: + for r in refs(span): + assert by_occurrence[r["occurrence_id"]]["association"]["parent"]["span_id"] == \ + format(span.context.span_id, "016x") + assert_uris_resolve(exporter) + + +@pytest.mark.parametrize("flag", ["hide_outputs", "hide_output_text", "hide_output_messages"]) +def test_tool_span_output_hide_flags_deny_its_tool_result_media(f01, instrument, flag): + """The tool span redacts its result under the OUTPUT hide flags; the same media must not be captured there.""" + manager = f01.manager() + exporter = instrument(manager, config=TraceConfig(**{flag: True})) + _screenshot_agent() + flush(manager) + tools = tool_spans(exporter) + assert tools, [s.name for s in exporter.get_finished_spans()] + tool_ids = {format(s.context.span_id, "016x") for s in tools} + for span in tools: + assert refs(span) == [], (flag, span.name) + result = (span.attributes or {}).get("pydantic_ai.tool.result") + assert result is None or result == "__REDACTED__", (flag, result) + assert not [b for b in f01.uploads if b["association"]["parent"]["span_id"] in tool_ids], flag + # Only a later model request (input-governed, so not hidden by an output flag) may own a copy. + owners = {s.name for s in exporter.get_finished_spans() if refs(s)} + assert all(span in {s.name for s in request_spans(exporter)} for span in owners), owners + assert_uris_resolve(exporter) + + +@pytest.mark.skipif(FilePart is None, reason="FilePart needs a newer pydantic-ai") +def test_generated_file_output_and_error_path(f01, instrument): + manager = f01.manager() + instrument(manager) + + def generate(messages, info): + return ModelResponse(parts=[TextPart("here"), FilePart(content=BinaryContent(data=GENERATED, + media_type="image/png"))]) + + Agent(FunctionModel(generate)).run_sync("draw") + + def failing(messages, info): + raise RuntimeError("provider error body: secret") + + with pytest.raises(Exception): + Agent(FunctionModel(failing)).run_sync(["fail", BinaryContent(data=IMAGE, media_type="image/png")]) + flush(manager) + outputs = [body for body in f01.uploads if body["direction"] == "output"] + assert len(outputs) == 1 and outputs[0]["purpose"] == "model_output" + assert GENERATED in f01.payloads + + +def test_urls_are_never_fetched(f01, instrument): + hits = [] + + class Handler(http.server.BaseHTTPRequestHandler): + def do_GET(self): # noqa: N802 + hits.append(self.path) + self.send_response(200) + self.end_headers() + + def log_message(self, *args): + pass + + server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), Handler) + threading.Thread(target=server.serve_forever, daemon=True).start() + try: + manager = f01.manager() + instrument(manager) + Agent(FunctionModel(echo)).run_sync(["look", ImageUrl(url="http://127.0.0.1:%d/private.png" + % server.server_port), + ImageUrl(url="http://169.254.169.254/latest/meta-data")]) + flush(manager) + assert hits == [] and f01.after_bootstrap() == [] + finally: + server.shutdown() + server.server_close() + + +@pytest.mark.parametrize("config", [TraceConfig(hide_inputs=True), TraceConfig(hide_input_images=True), + TraceConfig(pii_redaction=True), TraceConfig(hide_input_text=True)]) +def test_local_restrictions_deny(f01, instrument, config): + manager = f01.manager() + instrument(manager, config=config) + Agent(FunctionModel(echo)).run_sync(["x", BinaryContent(data=IMAGE, media_type="image/png")]) + flush(manager) + assert f01.after_bootstrap() == [] + + +def test_mismatched_declared_type_and_disabled_manager(f01, instrument): + instrument(create_attachment_manager(config=AttachmentConfig(enabled=False))) + Agent(FunctionModel(echo)).run_sync(["x", BinaryContent(data=IMAGE, media_type="image/png")]) + PydanticAIInstrumentor().uninstrument() + manager = f01.manager() + instrument(manager) + Agent(FunctionModel(echo)).run_sync(["x", BinaryContent(data=PDF, media_type="image/png")]) + flush(manager) + assert f01.payloads == []