diff --git a/.github/workflows/attachments-python-providers-tests.yml b/.github/workflows/attachments-python-providers-tests.yml new file mode 100644 index 000000000..a2ad1ade1 --- /dev/null +++ b/.github/workflows/attachments-python-providers-tests.yml @@ -0,0 +1,72 @@ +name: Python provider attachments + +# Real OpenAI, Anthropic and Groq SDKs (declared floor and latest) through the +# Python instrumentors to a TLS loopback F01 attachment server. Outbound network +# is disabled while the tests run. + +on: + pull_request: + paths: + - 'python/fi_instrumentation/attachments/**' + - 'python/frameworks/openai/**' + - 'python/frameworks/anthropic/**' + - 'python/frameworks/groq/**' + - 'python/traceai/**' + - 'python/tests/attachments_providers/**' + - '.github/workflows/attachments-python-providers-tests.yml' + push: + branches: [dev] + paths: + - 'python/fi_instrumentation/attachments/**' + - 'python/frameworks/openai/**' + - 'python/frameworks/anthropic/**' + - 'python/frameworks/groq/**' + - 'python/traceai/**' + - 'python/tests/attachments_providers/**' + - '.github/workflows/attachments-python-providers-tests.yml' + workflow_dispatch: + +permissions: + contents: read + +jobs: + providers: + name: Provider attachments (${{ matrix.name }}) + runs-on: ubuntu-24.04 + timeout-minutes: 20 + strategy: + fail-fast: false + matrix: + include: + # Declared floors (pyproject.toml of each integration). + - name: floor + python: '3.10' + sdks: 'openai==1.69.0 anthropic==0.41.0 groq==0.9.0 httpx<0.28' + # Latest releases. + - name: latest + python: '3.12' + sdks: 'openai==3.49.0 anthropic==2.8.0 groq==1.7.0' + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: false + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: ${{ matrix.python }} + - name: Install source integrations and the selected SDKs + env: + SDKS: ${{ matrix.sdks }} + run: | + python -m venv .venv + # shellcheck disable=SC2086 + .venv/bin/pip install -e python -e python/frameworks/openai -e python/frameworks/anthropic \ + -e python/frameworks/groq opentelemetry-sdk 'pytest>=7,<9' $SDKS + .venv/bin/python -c 'import openai, anthropic, groq; print("openai", openai.__version__, "anthropic", anthropic.__version__, "groq", groq.__version__)' + - name: Run with outbound network disabled + run: | + sudo unshare --net -- bash -e -c ' + ip link set lo up + cd "$2/python" + PYTHONPATH=. "$1" -m pytest tests/attachments_providers -q -p no:cacheprovider -o addopts="" \ + -o faulthandler_timeout=120 + ' _ "$PWD/.venv/bin/python" "$PWD" diff --git a/python/fi_instrumentation/attachments/media_capture.py b/python/fi_instrumentation/attachments/media_capture.py new file mode 100644 index 000000000..a2fd637c4 --- /dev/null +++ b/python/fi_instrumentation/attachments/media_capture.py @@ -0,0 +1,457 @@ +"""Shared provider-media helpers, built only on the public manager API. + +Provider adapters (``traceai_openai``, ``traceai_anthropic``, ``traceai_groq``) +use these to capture inline base64/data-URI parts and speech audio. Rules: + +- A source is never read or decoded before ``manager.preflight`` admitted it. + An admitted ticket that is not used is released at once. +- Nothing here fetches a URL, opens a path, reads an open file object, or pulls + from a caller's stream. Consumed audio is only the chunks the application + itself already read (``ConsumedAudio``). +- A local ``TraceConfig`` with PII redaction or a hide flag turns capture off + for that direction (deny wins). The manager's own policy still applies. +- One registry per manager remembers admitted (span, position) pairs, so a + second observer of the same span and position gets the FIRST occurrence back + and uploads nothing. + +Every public function is non-throwing and returns ``None``/empty on denial. +""" + +import base64 +import binascii +import dataclasses +import re +import threading +import weakref +from collections import OrderedDict +from typing import Any, Callable, Iterable, Mapping, NamedTuple, Optional, Tuple + +from .policy import CapturePolicy +from .stream import StreamOptions +from .types import Bytes, CorrelationHandle + +KiB = 1024 +MiB = 1024 * KiB +MAX_INLINE_BYTES = 256 * KiB +MAX_PARTS = 256 +MAX_SPANS = 1024 +# Consumed-stream reservations, largest first. A server whose max_upload_bytes is +# below 4 MiB denies the larger ones with too_large (after every privacy step), +# so stepping down is safe and needs no private limit accessor. +STREAM_RESERVATIONS = (4 * MiB, 2 * MiB, 1 * MiB, 512 * KiB, 256 * KiB) +IMAGE_MIMES = ("image/png", "image/jpeg", "image/webp") +FILE_MIMES = ("application/pdf", "application/octet-stream") +TOOL_ID = re.compile(r"[a-zA-Z0-9_-]{1,128}") +_DATA_URI = re.compile(r"data:([a-z]+/[a-z0-9.+-]+);base64") +_BASE64 = re.compile(r"[A-Za-z0-9+/]*={0,2}") +_INPUT_HIDE = ("hide_inputs", "hide_input_messages", "hide_input_text") +_OUTPUT_HIDE = ("hide_outputs", "hide_output_messages", "hide_output_text") +# Speech response_format values -> MIME. aac (raw ADTS), pcm (headerless) and +# mulaw have no F01 MIME, so they are never captured. +SPEECH_FORMATS = {"mp3": "audio/mpeg", "opus": "audio/ogg", "ogg": "audio/ogg", "flac": "audio/flac", + "wav": "audio/wav"} + + +class Occurrence(NamedTuple): + occurrence_id: str + mime_type: str + + +EMPTY: Mapping[str, Occurrence] = {} + + +def active(manager) -> bool: + """A manager that can admit anything. The disabled null object never can.""" + if manager is None: + return False + from ._disabled import DisabledAttachmentManager + + return not isinstance(manager, DisabledAttachmentManager) and callable(getattr(manager, "preflight", None)) + + +def get(value: Any, key: str) -> Any: + """Read a field of a request dict or an SDK response model; never raises.""" + try: + if isinstance(value, Mapping): + return value.get(key) + return getattr(value, key, None) + except Exception: + return None + + +def items(value: Any) -> list: + return list(value[:MAX_PARTS]) if isinstance(value, (list, tuple)) else [] + + +def _restricted(config, name: str) -> bool: + if config is None: + return False + try: + return bool(getattr(config, name, False)) + except Exception: + # An unreadable restriction is a restriction, never permission. + return True + + +def local_capture_policy(trace_config=None, *, direction: str, image: bool = False, error_detail: bool = False, + base: Optional[CapturePolicy] = None) -> CapturePolicy: + """Fold the instrumentor's local TraceConfig into a deny-only CapturePolicy. + + Any hide flag for the direction, or ``pii_redaction``, sets + ``capture_content=False``; the core cannot relax it, not even with + ``allow_unredacted_media_with_text_pii``. Input media keeps model-input + ancestry, so a tool screenshot is still treated as model input. + """ + base = base if isinstance(base, CapturePolicy) else CapturePolicy() + names = _INPUT_HIDE if direction == "input" else _OUTPUT_HIDE + hidden = any(_restricted(trace_config, name) for name in names) or _restricted(trace_config, "pii_redaction") + if image and direction == "input" and _restricted(trace_config, "hide_input_images"): + hidden = True + if error_detail and _restricted(trace_config, "hide_error_details"): + hidden = True + changes = {} + if hidden: + changes["capture_content"] = False + if direction == "input": + changes["model_input"] = True + if error_detail: + changes["error_detail"] = True + return dataclasses.replace(base, **changes) + + +def span_key(span) -> Optional[str]: + """The emitted identity of a sampled, recording span, else None.""" + try: + correlation = CorrelationHandle.from_span(span) + except Exception: + return None + return "%s/%s" % (correlation.trace_id, correlation.span_id) if correlation.sampled else None + + +class _Registry: + """Admitted positions per manager and span. Holds IDs only, never sources.""" + + def __init__(self) -> None: + self._lock = threading.Lock() + self._managers: "weakref.WeakKeyDictionary" = weakref.WeakKeyDictionary() + + def get(self, manager, key: str, position: str) -> Optional[Occurrence]: + with self._lock: + spans = self._managers.get(manager) + positions = spans.get(key) if spans is not None else None + return positions.get(position) if positions is not None else None + + def put(self, manager, key: str, position: str, occurrence: Occurrence) -> None: + with self._lock: + spans = self._managers.get(manager) + if spans is None: + spans = OrderedDict() + self._managers[manager] = spans + positions = spans.get(key) + if positions is None: + if len(spans) >= MAX_SPANS: + spans.popitem(last=False) + positions = spans[key] = {} + if len(positions) < MAX_PARTS: + positions[position] = occurrence + + +REGISTRY = _Registry() + + +def encoded_length(encoded: Any, max_bytes: int = MAX_INLINE_BYTES) -> int: + """Exact decoded length from the string's shape only (no decode); 0 when out of bounds.""" + if not isinstance(encoded, str): + return 0 + length = len(encoded) + if not length or length % 4 or length > 4 * -(-max_bytes // 3): + return 0 + return (length // 4) * 3 - (2 if encoded.endswith("==") else 1 if encoded.endswith("=") else 0) + + +def split_inline(value: Any, mime_types: Tuple[str, ...], raw: bool, + max_bytes: int = MAX_INLINE_BYTES) -> Optional[Tuple[str, str, int]]: + """Structural parse: (mime, encoded, exact decoded length) or None. Bounded before slicing.""" + if not isinstance(value, str): + return None + mime, start = mime_types[0], 0 + if not raw: + comma = value.find(",", 0, 128) + match = _DATA_URI.fullmatch(value, 0, comma) if comma > 5 else None + if match is None or match.group(1) not in mime_types: + return None + mime, start = match.group(1), comma + 1 + if not 0 < len(value) - start <= 4 * -(-max_bytes // 3): + return None + encoded = value[start:] + bound = encoded_length(encoded, max_bytes) + return (mime, encoded, bound) if bound else None + + +def strict_decode(encoded: str, max_bytes: int, cap: int = MAX_INLINE_BYTES) -> Optional[bytes]: + """Strict alphabet, canonical padding, and the ticket's budget as a hard bound.""" + if not encoded_length(encoded, cap) or not _BASE64.fullmatch(encoded): + return None + try: + data = base64.b64decode(encoded, validate=True) + except (binascii.Error, ValueError): + return None + if not data or len(data) > max_bytes or len(data) > cap or base64.b64encode(data).decode("ascii") != encoded: + return None + return data + + +def _fields(message_index=None, part_index=None, role=None, tool_call_id=None) -> dict: + fields = {"message_index": message_index, "part_index": part_index, "role": role} + if isinstance(tool_call_id, str) and TOOL_ID.fullmatch(tool_call_id): + fields["tool_call_id"] = tool_call_id + return fields + + +def attach_inline(manager, span, read: Callable[[], Any], mime_types: Tuple[str, ...], *, raw: bool, + direction: str, purpose: Optional[str], capture: CapturePolicy, + message_index: Optional[int] = None, part_index: Optional[int] = None, + role: Optional[str] = None, tool_call_id: Optional[str] = None, + max_bytes: int = MAX_INLINE_BYTES) -> Optional[Occurrence]: + """Preflight, then read once, decode strictly and attach. Never raises.""" + try: + state = {} + + def parsed(): + if "value" not in state: + state["value"] = split_inline(read(), mime_types, raw, max_bytes) + return state["value"] + + def preflight(mime): + options = dict(mime_type=mime, direction=direction, purpose=purpose, capture_policy=capture) + ticket = manager.preflight(span, max_bytes=max_bytes, **options) + # too_large (negotiated limit below the cap) and queue_full (budget nearly used) come after + # every privacy, correlation and sampling step, so sizing exactly and retrying once is safe. + if ticket.reason in ("too_large", "queue_full"): + found = parsed() + if found: + ticket = manager.preflight(span, max_bytes=found[2], **options) + return ticket + + ticket = preflight(mime_types[0]) + if not ticket.admitted: + return None + found = parsed() + if not found: + ticket.release() + return None + mime, encoded, _ = found + if mime != mime_types[0]: + # A data URI names a sibling MIME of the same modality: re-admit under the exact type. + ticket.release() + ticket = preflight(mime) + if not ticket.admitted: + return None + data = strict_decode(encoded, ticket.max_bytes, max_bytes) + if data is None: + ticket.release() + return None + handle = ticket.attach(Bytes(data), **_fields(message_index, part_index, role, tool_call_id)) + return Occurrence(handle.occurrence_id, mime) if handle.admission == "queued" else None + except Exception: + return None + + +class Observation: + """One adapter observation of one span: registry lookup plus admitted positions.""" + + def __init__(self, manager, span, namespace: str): + self.manager, self.span, self.namespace = manager, span, namespace + self.key = span_key(span) if active(manager) else None + self.captured: "OrderedDict[str, Occurrence]" = OrderedDict() + self._visited = 0 + + @property + def usable(self) -> bool: + return self.key is not None + + def visit(self, position: str, capture: Callable[[], Optional[Occurrence]]) -> None: + if self.key is None: + return + self._visited += 1 + if self._visited > MAX_PARTS: + return + registry_position = "%s/%s" % (self.namespace, position) + previous = REGISTRY.get(self.manager, self.key, registry_position) + if previous is not None: + self.captured[position] = previous + return + occurrence = capture() + if occurrence is not None: + REGISTRY.put(self.manager, self.key, registry_position, occurrence) + self.captured[position] = occurrence + + +# ------------------------------------------------------------------ audio +def sniff_audio(head: Any) -> Optional[str]: + """MIME from container magic bytes the application already holds; None when unknown.""" + if not isinstance(head, (bytes, bytearray, memoryview)): + return None + head = bytes(head[:12]) + if len(head) >= 12 and head[:4] == b"RIFF" and head[8:12] == b"WAVE": + return "audio/wav" + if head[:4] == b"OggS": + return "audio/ogg" + if head[:4] == b"fLaC": + return "audio/flac" + if head[:3] == b"ID3": + return "audio/mpeg" + # MPEG audio frame sync with a defined layer. Layer bits 00 is AAC ADTS, which is not captured. + if len(head) >= 2 and head[0] == 0xFF and head[1] & 0xE0 == 0xE0 and (head[1] >> 1) & 3: + return "audio/mpeg" + if head[:4] == b"\x1a\x45\xdf\xa3": + return "audio/webm" + return None + + +def audio_mime(head: Any, declared_format: Optional[str] = None) -> Optional[str]: + """The sniffed container, cross-checked against a declared format when one was sent.""" + sniffed = sniff_audio(head) + if sniffed is None: + return None + if declared_format is not None and SPEECH_FORMATS.get(declared_format) != sniffed: + return None + return sniffed + + +def attach_audio(manager, span, data: Any, *, direction: str, purpose: Optional[str], capture: CapturePolicy, + declared_format: Optional[str] = None, message_index: Optional[int] = None, + part_index: Optional[int] = None, role: Optional[str] = None) -> Optional[Occurrence]: + """Complete in-memory audio the SDK already holds (never a stream or file). Never raises.""" + try: + if not isinstance(data, (bytes, bytearray, memoryview)): + return None + length = data.nbytes if isinstance(data, memoryview) else len(data) + if length <= 0: + return None + options = dict(direction=direction, purpose=purpose, capture_policy=capture, max_bytes=length) + # Admit the modality before looking at the bytes, then re-admit under the exact container type. + ticket = manager.preflight(span, mime_type="audio/wav", **options) + if not ticket.admitted: + return None + mime = audio_mime(data[:12], declared_format) + if mime is None: + ticket.release() + return None + if mime != "audio/wav": + ticket.release() + ticket = manager.preflight(span, mime_type=mime, **options) + if not ticket.admitted: + return None + handle = ticket.attach(Bytes(bytes(data) if not isinstance(data, bytes) else data), + **_fields(message_index, part_index, role)) + return Occurrence(handle.occurrence_id, mime) if handle.admission == "queued" else None + except Exception: + return None + + +class ConsumedAudio: + """Forward audio chunks the APPLICATION consumed; never pulls, clones or tees. + + Admission happens on the first consumed bytes. Exceeding the reservation, + a tightened policy, ``abort()``, or ending without ``finish()`` uploads + nothing. Only ``finish()`` after a complete read uploads. + """ + + _HEAD = 12 + + def __init__(self, manager, span, *, direction: str, purpose: Optional[str], capture: CapturePolicy, + declared_format: Optional[str] = None, role: Optional[str] = None): + self._manager, self._span = manager, span + self._direction, self._purpose, self._capture = direction, purpose, capture + self._declared, self._role = declared_format, role + self._head = bytearray() + self._stream = None + self._done = False + self.reason: Optional[str] = None + self.occurrence: Optional[Occurrence] = None + self._mime: Optional[str] = None + + def _start(self) -> None: + mime = audio_mime(bytes(self._head), self._declared) + if mime is None: + self._stop("source_unavailable") + return + ticket = None + for reserve in STREAM_RESERVATIONS: + ticket = self._manager.preflight(self._span, mime_type=mime, direction=self._direction, + max_bytes=reserve, purpose=self._purpose, + capture_policy=self._capture) + if ticket.reason != "too_large": + break + if ticket is None or not ticket.admitted: + self._stop(getattr(ticket, "reason", "policy_denied")) + return + stream = ticket.start_stream(StreamOptions( + mime_type=mime, direction=self._direction, max_bytes=ticket.max_bytes, purpose=self._purpose, + capture_policy=self._capture, role=self._role)) + if not stream.open: + self._stop(stream.reason) + return + self._stream, self._mime = stream, mime + head, self._head = bytes(self._head), bytearray() + if not stream.try_write(head): + self._stop(stream.reason) + + def _stop(self, reason: Optional[str]) -> None: + self._done = True + self.reason = reason or "incomplete" + self._head = bytearray() + if self._stream is not None: + self._stream.abort() + self._stream = None + + def write(self, chunk: Any) -> None: + try: + if self._done or not isinstance(chunk, (bytes, bytearray, memoryview)) or not len(chunk): + return + if self._stream is None: + need = self._HEAD - len(self._head) + self._head += bytes(chunk[:need]) + chunk = chunk[need:] + if len(self._head) < self._HEAD: + return + self._start() + if self._stream is None or not len(chunk): + return + if not self._stream.try_write(chunk): + self._stop(self._stream.reason) + except Exception: + self._stop("source_unavailable") + + def finish(self) -> Optional[Occurrence]: + try: + if self._done or self._stream is None: + self._stop(self.reason or "incomplete") + return None + stream, self._stream = self._stream, None + self._done = True + handle = stream.finish() + self.reason = handle.admission + if handle.admission == "queued": + self.occurrence = Occurrence(handle.occurrence_id, self._mime) + return self.occurrence + except Exception: + self._stop("source_unavailable") + return None + + def abort(self) -> None: + if not self._done: + self._stop("incomplete") + + +def request_files(files: Any) -> Iterable[Tuple[int, Any]]: + """(index, content) of multipart file entries; content is whatever the SDK will send.""" + try: + entries = files.items() if isinstance(files, Mapping) else files + for index, entry in enumerate(list(entries or [])[:MAX_PARTS]): + value = entry[1] if isinstance(entry, tuple) and len(entry) == 2 else None + content = value[1] if isinstance(value, tuple) and len(value) >= 2 else value + yield index, content + except Exception: + return diff --git a/python/frameworks/anthropic/traceai_anthropic/__init__.py b/python/frameworks/anthropic/traceai_anthropic/__init__.py index bf81c28aa..664e94d84 100644 --- a/python/frameworks/anthropic/traceai_anthropic/__init__.py +++ b/python/frameworks/anthropic/traceai_anthropic/__init__.py @@ -24,6 +24,23 @@ _instruments = ("anthropic >= 0.30.0",) +def _attachment_manager(kwargs: Any) -> Any: + """The shared manager passed as ``attachments`` (or auto's ``attachment_manager``), if it can admit. + + A missing, disabled or pre-attachments core leaves tracing exactly as before. + """ + manager = kwargs.get("attachments") + if manager is None: + manager = kwargs.get("attachment_manager") + if manager is None: + return None + try: + from fi_instrumentation.attachments.media_capture import active + except ImportError: + return None + return manager if active(manager) else None + + class AnthropicInstrumentor(BaseInstrumentor): # type: ignore[misc] """An instrumentor for the Anthropic framework.""" @@ -76,6 +93,7 @@ def _instrument(self, **kwargs: Any) -> None: trace_api.get_tracer(__name__, __version__, tracer_provider), config=config, ) + attachments = _attachment_manager(kwargs) if Completions is not None: self._original_completions_create = Completions.create @@ -102,7 +120,10 @@ def _instrument(self, **kwargs: Any) -> None: module="anthropic.resources.messages", name="Messages.create", wrapper=_MessagesWrapper( - tracer=self._tracer, native_tracing=native_tracing + tracer=self._tracer, + native_tracing=native_tracing, + attachments=attachments, + config=config, ), ) @@ -111,7 +132,10 @@ def _instrument(self, **kwargs: Any) -> None: module="anthropic.resources.messages", name="AsyncMessages.create", wrapper=_AsyncMessagesWrapper( - tracer=self._tracer, native_tracing=native_tracing + tracer=self._tracer, + native_tracing=native_tracing, + attachments=attachments, + config=config, ), ) @@ -169,3 +193,19 @@ def _uninstrument(self, **kwargs: Any) -> None: if native_tracing is not None: native_tracing.uninstall() self._native_tracing = None + + +_ATTACHMENT_EXPORTS = ( + "capture_anthropic_attachments", + "project_anthropic_media", + "anthropic_attachment_capabilities", +) + + +def __getattr__(name: str) -> Any: + # Lazy: these need a fi-instrumentation-otel release with attachments. + if name in _ATTACHMENT_EXPORTS: + from traceai_anthropic import _attachments + + return getattr(_attachments, name) + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/python/frameworks/anthropic/traceai_anthropic/_attachments.py b/python/frameworks/anthropic/traceai_anthropic/_attachments.py new file mode 100644 index 000000000..bd71cf67e --- /dev/null +++ b/python/frameworks/anthropic/traceai_anthropic/_attachments.py @@ -0,0 +1,152 @@ +"""Anthropic Messages media capture through the shared, provider-owned attachment manager. + +Mirrors ``@traceai/anthropic`` (``captureAnthropicAttachments`` / +``projectAnthropicMedia``): typed base64 image and PDF document blocks, and one +level of ``tool_result`` content. A tool-result image is a ``computer_screenshot`` +only when the request declares a typed computer tool (``computer_YYYYMMDD``) AND +an assistant ``tool_use`` of that tool has the result's ``tool_use_id``; message +and block indices preserve causal order for the screenshot filmstrip. Nothing +here fetches URL sources or resolves file IDs. + +This module imports ``fi_instrumentation.attachments``; the instrumentor imports +it only when an attachment manager is passed. +""" + +import re +from types import MappingProxyType +from typing import Any, Mapping + +from fi_instrumentation.attachments.media_capture import ( + EMPTY, + IMAGE_MIMES, + MAX_INLINE_BYTES, + TOOL_ID, + Observation, + Occurrence, + attach_inline, + get, + items, + local_capture_policy, +) + +__all__ = ("capture_anthropic_attachments", "project_anthropic_media", "anthropic_attachment_capabilities") + +#: Declarative handoff; native SDK span ownership is not overridden by this adapter. +anthropic_attachment_capabilities = MappingProxyType({ + "schema_version": 1, + "operations": ("messages.create.input", "tool_result.input"), + "sources": ("inline_base64",), + # Fail closed: no capture is attempted for these. + "unsupported": ("remote_url", "provider_file", "generated_image", "beta_messages_hookup", "streamed_output"), + "max_inline_bytes": MAX_INLINE_BYTES, +}) + +_COMPUTER_TOOL = re.compile(r"computer_\d{8}") +_DOCUMENT_MIMES = ("application/pdf",) + + +def _position(message_index: int, position: str) -> str: + return "input/%d/%s" % (message_index, position) + + +def capture_anthropic_attachments(manager, span, payload: Any, *, trace_config=None, + capture=None) -> Mapping[str, Occurrence]: + """Capture typed media of one Messages request on the span the instrumentor emitted. + + ``payload`` holds the call's keyword arguments (``messages``, ``tools``). + Returns admitted positions (position key -> occurrence); empty when no + manager, disabled, unsampled or denied. Never raises or changes ``payload``. + """ + observation = Observation(manager, span, "anthropic") + if not observation.usable: + return EMPTY + try: + messages = items(get(payload, "messages")) + computer_names = set() + for tool in items(get(payload, "tools")): + kind, name = get(tool, "type"), get(tool, "name") + if isinstance(kind, str) and _COMPUTER_TOOL.fullmatch(kind) and isinstance(name, str): + computer_names.add(name) + computer_calls = set() + for message in messages: + if get(message, "role") != "assistant": + continue + for block in items(get(message, "content")): + block_id, name = get(block, "id"), get(block, "name") + if get(block, "type") == "tool_use" and isinstance(block_id, str) and name in computer_names: + computer_calls.add(block_id) + + def visit(block, message_index, index, position, role, tool_call_id=None, error_detail=False): + kind = get(block, "type") + if kind not in ("image", "document"): + return + source = get(block, "source") + if get(source, "type") != "base64": + return + mime = get(source, "media_type") + if mime not in (IMAGE_MIMES if kind == "image" else _DOCUMENT_MIMES): + return + if kind == "document": + purpose = "document" + elif tool_call_id is not None: + purpose = "computer_screenshot" if tool_call_id in computer_calls else "tool_result" + else: + purpose = "model_input" + policy = local_capture_policy(trace_config, direction="input", base=capture, image=kind == "image", + error_detail=error_detail) + observation.visit(_position(message_index, position), lambda: attach_inline( + manager, span, lambda: get(source, "data"), (mime,), raw=True, direction="input", + purpose=purpose, capture=policy, message_index=message_index, part_index=index, role=role, + tool_call_id=tool_call_id)) + + for message_index, message in enumerate(messages): + role = "assistant" if get(message, "role") == "assistant" else "user" + for index, block in enumerate(items(get(message, "content"))): + if get(block, "type") == "tool_result" and isinstance(get(block, "content"), (list, tuple)): + tool_use_id = get(block, "tool_use_id") + # Invalid tool IDs fail closed instead of losing causal identity. + if not isinstance(tool_use_id, str) or not TOOL_ID.fullmatch(tool_use_id): + continue + for child_index, child in enumerate(items(get(block, "content"))): + visit(child, message_index, child_index, "%d/%d" % (index, child_index), "tool", + tool_use_id, get(block, "is_error") is True) + else: + visit(block, message_index, index, str(index), role) + except Exception: + pass # No source-bearing exceptions or changes to the application's call. + return MappingProxyType(dict(observation.captured)) if observation.captured else EMPTY + + +def project_anthropic_media(payload: Any, captured: Mapping[str, Occurrence]) -> Any: + """Telemetry-only copy: ONLY admitted blocks get a ``fi-attachment:`` URL source. + + With nothing admitted the SAME object is returned; the call's arguments are + never mutated and only plain mappings are copied. + """ + if not captured or not isinstance(payload, Mapping) or not isinstance(payload.get("messages"), list): + return payload + try: + def replace(value, position): + occurrence = captured.get(position) + if occurrence is None or not isinstance(value, Mapping): + return value + return {**value, "source": {"type": "url", "url": "fi-attachment:" + occurrence.occurrence_id}} + + def message(value, message_index): + if not isinstance(value, Mapping) or not isinstance(value.get("content"), list): + return value + content = [] + for index, block in enumerate(value["content"]): + if isinstance(block, Mapping) and block.get("type") == "tool_result" \ + and isinstance(block.get("content"), list): + block = {**block, "content": [ + replace(child, _position(message_index, "%d/%d" % (index, child_index))) + for child_index, child in enumerate(block["content"])]} + content.append(block) + else: + content.append(replace(block, _position(message_index, str(index)))) + return {**value, "content": content} + + return {**payload, "messages": [message(value, index) for index, value in enumerate(payload["messages"])]} + except Exception: + return payload diff --git a/python/frameworks/anthropic/traceai_anthropic/_wrappers.py b/python/frameworks/anthropic/traceai_anthropic/_wrappers.py index 9815f9042..8ca085471 100644 --- a/python/frameworks/anthropic/traceai_anthropic/_wrappers.py +++ b/python/frameworks/anthropic/traceai_anthropic/_wrappers.py @@ -66,12 +66,52 @@ def __init__( tracer: trace_api.Tracer, *args: Any, native_tracing: Optional[_NativeTracing] = None, + attachments: Any = None, + config: Any = None, **kwargs: Any, ) -> None: super().__init__(*args, **kwargs) self._tracer = tracer self._native_tracing = native_tracing + # Provider-owned shared attachment manager, or None (no capture, no imports). + self._attachments = attachments + self._config = config + + @contextmanager + def _start_messages_media_span(self, span_name: str, arguments: Mapping[str, Any]) -> Iterator[_WithSpan]: + """Messages span with attachment capture: content attributes are set after capture, + from a telemetry-only copy whose admitted media blocks are fi-attachment URIs.""" + with self._start_as_current_span( + span_name=span_name, + attributes=dict( + chain( + get_attributes_from_context(), + _get_llm_model_name_from_input(arguments), + _get_llm_provider(), + _get_llm_system(), + _get_llm_span_kind(), + ) + ), + ) as span: + telemetry = arguments + try: + from traceai_anthropic._attachments import ( + capture_anthropic_attachments, + project_anthropic_media, + ) + + captured = capture_anthropic_attachments( + self._attachments, span._span, arguments, trace_config=self._config + ) + telemetry = project_anthropic_media(arguments, captured) + except Exception: + logger.debug("Attachment capture unavailable", exc_info=True) + try: + span.set_attributes(_content_attributes(telemetry)) + except Exception: + logger.warning("Failed to set input attributes") + yield span def _without_native_span(self, wrapped: Callable[..., Any], instance: Any) -> Callable[..., Any]: """The original method for this call, with the SDK's duplicate native span turned off.""" @@ -279,6 +319,15 @@ def __call__( return wrapped(*args, **kwargs) wrapped = self._without_native_span(wrapped, instance) + if self._attachments is not None: + with self._start_messages_media_span("Messages", kwargs) as span: + try: + response = wrapped(*args, **kwargs) + except BaseException as exception: + _finish_request_error(span, exception) + raise + return _observe_message_response(response, span) + arguments = kwargs llm_input_messages = dict(arguments).pop("messages", None) llm_prompt = dict(arguments).pop("system", None) @@ -337,6 +386,15 @@ async def __call__( return await wrapped(*args, **kwargs) wrapped = self._without_native_span_async(wrapped, instance) + if self._attachments is not None: + with self._start_messages_media_span("AsyncMessages", kwargs) as span: + try: + response = await wrapped(*args, **kwargs) + except BaseException as exception: + _finish_request_error(span, exception) + raise + return _observe_message_response(response, span, is_async=True) + arguments = kwargs llm_prompt = dict(arguments).pop("system", None) llm_input_messages = dict(arguments).pop("messages", None) @@ -467,6 +525,27 @@ def __call__( return response +def _content_attributes(arguments: Mapping[str, Any]) -> Dict[str, Any]: + """The Messages content attributes, in the same order the non-attachment path sets them.""" + llm_input_messages = dict(arguments).pop("messages", None) + llm_prompt = dict(arguments).pop("system", None) + invocation_parameters = _get_invocation_parameters(arguments) + extracted_data = _extract_image_data(llm_input_messages) + return dict( + chain( + _get_llm_prompts(llm_prompt), + _get_llm_input_messages(llm_input_messages), + _get_llm_invocation_parameters(invocation_parameters), + _get_llm_tools(invocation_parameters), + _get_inputs(extracted_data["filtered_messages"]), + _get_image_inputs(extracted_data["input_images"]), + _get_eval_input(extracted_data["eval_input"]), + _get_raw_input(arguments), + _get_query(extracted_data["query"]), + ) + ) + + def _get_inputs(arguments: Mapping[str, Any]) -> Iterator[Tuple[str, Any]]: yield INPUT_VALUE, safe_json_dumps(arguments) yield INPUT_MIME_TYPE, JSON @@ -519,7 +598,12 @@ def process_string_content(content): def process_image_item(item): source = item.get("source", {}) - if source.get("type") == "base64": + url = source.get("url") + if source.get("type") == "url" and isinstance(url, str) and url.startswith("fi-attachment:"): + # A captured image (telemetry-only projection) keeps its eval placeholder. + input_images.append(url) + eval_input.append("{{" + f"{SpanAttributes.INPUT_IMAGES}.{len(input_images) - 1}" + "}}") + elif source.get("type") == "base64": data = source.get("data") if data: input_images.append(data) diff --git a/python/frameworks/groq/traceai_groq/__init__.py b/python/frameworks/groq/traceai_groq/__init__.py index 28f61563d..6544393ca 100644 --- a/python/frameworks/groq/traceai_groq/__init__.py +++ b/python/frameworks/groq/traceai_groq/__init__.py @@ -23,6 +23,20 @@ _instruments = ("groq >= 0.9.0",) +def _attachment_manager(kwargs: Any) -> Any: + """The shared manager passed as ``attachments`` (or auto's ``attachment_manager``), if it can admit.""" + manager = kwargs.get("attachments") + if manager is None: + manager = kwargs.get("attachment_manager") + if manager is None: + return None + try: + from fi_instrumentation.attachments.media_capture import active + except ImportError: + return None + return manager if active(manager) else None + + class GroqInstrumentor(BaseInstrumentor): # type: ignore[misc] """An instrumentor for the Groq framework.""" @@ -30,6 +44,7 @@ class GroqInstrumentor(BaseInstrumentor): # type: ignore[misc] "_original_completions_create", "_original_async_completions_create", "_tracer", + "_original_audio", ) def instrumentation_dependencies(self) -> Collection[str]: @@ -61,6 +76,25 @@ def _instrument(self, **kwargs: Any) -> None: wrapper=_AsyncCompletionsWrapper(tracer=self._tracer), ) + # Audio spans exist only with an enabled attachment manager: plain tracing is unchanged. + self._original_audio = [] + attachments = _attachment_manager(kwargs) + if attachments is not None: + from traceai_groq._audio import AUDIO_METHODS, wrapper_for + + for module_name, class_name, operation, is_async in AUDIO_METHODS: + try: + owner = getattr(import_module(module_name), class_name) + original = owner.create + except (ImportError, AttributeError): + continue # Older SDKs have no such method (groq 0.9.0 has no speech). + wrap_function_wrapper( + module=module_name, + name=f"{class_name}.create", + wrapper=wrapper_for(self._tracer, attachments, config, operation, is_async), + ) + self._original_audio.append((owner, original)) + if Protect is not None: self._original_protect = Protect.protect wrap_function_wrapper( @@ -72,6 +106,9 @@ def _instrument(self, **kwargs: Any) -> None: self._original_protect = None def _uninstrument(self, **kwargs: Any) -> None: + for owner, original in getattr(self, "_original_audio", None) or (): + owner.create = original + self._original_audio = [] groq_module = import_module("groq.resources.chat.completions") if self._original_completions_create is not None: groq_module.Completions.create = self._original_completions_create diff --git a/python/frameworks/groq/traceai_groq/_audio.py b/python/frameworks/groq/traceai_groq/_audio.py new file mode 100644 index 000000000..00c791ecc --- /dev/null +++ b/python/frameworks/groq/traceai_groq/_audio.py @@ -0,0 +1,165 @@ +"""Groq speech and transcription/translation spans with attachment capture (opt-in). + +Installed only when an enabled attachment manager is passed to ``instrument()``, +so plain tracing is unchanged. Capture goes through the same shared helper as +``traceai_openai`` (``fi_instrumentation.attachments.media_capture``): + +- speech output: only bytes the SDK already buffered (a non-streamed response); + a streamed body the application has not read is never read here; +- transcription/translation input: ``bytes`` or ``(name, bytes, ...)`` file + values only. Paths and open file objects are the application's files and + streams and are never opened or read. +""" + +import logging +from typing import Any, Callable, Mapping, Tuple + +from fi_instrumentation.attachments.media_capture import ( + Observation, + attach_audio, + get, + local_capture_policy, + request_files, +) +from fi_instrumentation.fi_types import FiSpanKindValues, SpanAttributes +from opentelemetry import context as context_api +from opentelemetry import trace as trace_api + +logger = logging.getLogger(__name__) +logger.addHandler(logging.NullHandler()) + +__all__ = ("capture_groq_speech", "capture_groq_audio_input", "groq_attachment_capabilities") + +groq_attachment_capabilities = { + "schema_version": 1, + "operations": ("audio.speech.create.output", "audio.transcriptions.create.input", + "audio.translations.create.input"), + "sources": ("sdk_buffered_bytes", "inline_bytes"), + "unsupported": ("streamed_speech", "file_path", "open_file_object", "chat_media"), +} + + +def _buffered(response: Any) -> Any: + """Body bytes the SDK already read; an unread streamed body raises instead of reading.""" + http_response = getattr(response, "http_response", None) + if http_response is None: + http_response = getattr(response, "response", None) + try: + return http_response.content + except Exception: + return None + + +def capture_groq_speech(manager, span, response: Any, *, request: Any = None, trace_config=None, capture=None): + """Capture a non-streamed Groq speech response on its span. Never raises.""" + observation = Observation(manager, span, "groq") + if observation.usable: + observation.visit("speech/output/0/0", lambda: attach_audio( + manager, span, _buffered(response), direction="output", purpose="model_output", role="assistant", + capture=local_capture_policy(trace_config, direction="output", base=capture), + declared_format=get(request, "response_format"))) + return observation.captured.get("speech/output/0/0") + + +def capture_groq_audio_input(manager, span, file: Any, *, operation: str = "transcriptions", trace_config=None, + capture=None): + """Capture an in-memory transcription/translation input file. Never raises.""" + observation = Observation(manager, span, "groq") + if observation.usable: + policy = local_capture_policy(trace_config, direction="input", base=capture) + for index, content in request_files([("file", file)]): + if isinstance(content, (bytes, bytearray)): + observation.visit("%s/input/%d/0" % (operation, index), lambda c=content, i=index: attach_audio( + manager, span, c, direction="input", purpose="model_input", role="user", capture=policy, + message_index=i, part_index=0)) + return dict(observation.captured) + + +class _AudioWrapper: + def __init__(self, tracer: trace_api.Tracer, manager: Any, config: Any, operation: str) -> None: + self._tracer, self._manager, self._config, self._operation = tracer, manager, config, operation + + def _start(self, kwargs: Mapping[str, Any]) -> Any: + attributes = { + SpanAttributes.GEN_AI_SPAN_KIND: FiSpanKindValues.LLM.value, + SpanAttributes.GEN_AI_PROVIDER_NAME: "groq", + } + if isinstance(model := kwargs.get("model"), str) and model: + attributes[SpanAttributes.GEN_AI_REQUEST_MODEL] = model + return self._tracer.start_span("Groq Audio " + self._operation.capitalize(), attributes=attributes) + + def _before(self, span: Any, kwargs: Mapping[str, Any]) -> None: + if self._operation in ("transcriptions", "translations"): + capture_groq_audio_input(self._manager, span, kwargs.get("file"), operation=self._operation, + trace_config=self._config) + + def _after(self, span: Any, kwargs: Mapping[str, Any], response: Any) -> None: + if self._operation == "speech": + capture_groq_speech(self._manager, span, response, request=kwargs, trace_config=self._config) + + @staticmethod + def _fail(span: Any, error: BaseException) -> None: + try: + span.set_status(trace_api.Status(trace_api.StatusCode.ERROR, type(error).__name__)) + span.end() + except Exception: + logger.debug("Failed to end span", exc_info=True) + + @staticmethod + def _ok(span: Any) -> None: + try: + span.set_status(trace_api.Status(trace_api.StatusCode.OK)) + span.end() + except Exception: + logger.debug("Failed to end span", exc_info=True) + + +class _SyncAudioWrapper(_AudioWrapper): + def __call__(self, wrapped: Callable[..., Any], instance: Any, args: Tuple[Any, ...], + kwargs: Mapping[str, Any]) -> Any: + if context_api.get_value(context_api._SUPPRESS_INSTRUMENTATION_KEY): + return wrapped(*args, **kwargs) + span = self._start(kwargs) + with trace_api.use_span(span, end_on_exit=False, record_exception=False, set_status_on_exception=False): + self._before(span, kwargs) + try: + response = wrapped(*args, **kwargs) + except BaseException as error: + self._fail(span, error) + raise + self._after(span, kwargs, response) + self._ok(span) + return response + + +class _AsyncAudioWrapper(_AudioWrapper): + async def __call__(self, wrapped: Callable[..., Any], instance: Any, args: Tuple[Any, ...], + kwargs: Mapping[str, Any]) -> Any: + if context_api.get_value(context_api._SUPPRESS_INSTRUMENTATION_KEY): + return await wrapped(*args, **kwargs) + span = self._start(kwargs) + with trace_api.use_span(span, end_on_exit=False, record_exception=False, set_status_on_exception=False): + self._before(span, kwargs) + try: + response = await wrapped(*args, **kwargs) + except BaseException as error: + self._fail(span, error) + raise + self._after(span, kwargs, response) + self._ok(span) + return response + + +# (module, class, operation) for each audio method; missing modules (older SDKs) are skipped. +AUDIO_METHODS = ( + ("groq.resources.audio.speech", "Speech", "speech", False), + ("groq.resources.audio.speech", "AsyncSpeech", "speech", True), + ("groq.resources.audio.transcriptions", "Transcriptions", "transcriptions", False), + ("groq.resources.audio.transcriptions", "AsyncTranscriptions", "transcriptions", True), + ("groq.resources.audio.translations", "Translations", "translations", False), + ("groq.resources.audio.translations", "AsyncTranslations", "translations", True), +) + + +def wrapper_for(tracer: Any, manager: Any, config: Any, operation: str, is_async: bool) -> Any: + return (_AsyncAudioWrapper if is_async else _SyncAudioWrapper)(tracer, manager, config, operation) diff --git a/python/frameworks/openai/traceai_openai/__init__.py b/python/frameworks/openai/traceai_openai/__init__.py index 73110e68d..55d01cc34 100644 --- a/python/frameworks/openai/traceai_openai/__init__.py +++ b/python/frameworks/openai/traceai_openai/__init__.py @@ -27,6 +27,23 @@ _MODULE = "openai" +def _attachment_manager(kwargs: Any) -> Any: + """The shared manager passed as ``attachments`` (or auto's ``attachment_manager``), if it can admit. + + A missing, disabled or pre-attachments core leaves tracing exactly as before. + """ + manager = kwargs.get("attachments") + if manager is None: + manager = kwargs.get("attachment_manager") + if manager is None: + return None + try: + from fi_instrumentation.attachments.media_capture import active + except ImportError: + return None + return manager if active(manager) else None + + class OpenAIInstrumentor(BaseInstrumentor): # type: ignore """ An instrumentor for openai @@ -57,18 +74,23 @@ def _instrument(self, **kwargs: Any) -> None: trace_api.get_tracer(__name__, __version__, tracer_provider), config=config, ) + # The provider-owned shared manager (register(...).attachment_manager). Auto passes + # it as attachment_manager; nothing is captured unless an enabled manager is supplied. + attachments = _attachment_manager(kwargs) openai = import_module(_MODULE) self._original_request = openai.OpenAI.request self._original_async_request = openai.AsyncOpenAI.request wrap_function_wrapper( module=_MODULE, name="OpenAI.request", - wrapper=_Request(tracer=tracer, openai=openai, config=config), + wrapper=_Request(tracer=tracer, openai=openai, config=config, attachments=attachments), ) wrap_function_wrapper( module=_MODULE, name="AsyncOpenAI.request", - wrapper=_AsyncRequest(tracer=tracer, openai=openai, config=config), + wrapper=_AsyncRequest( + tracer=tracer, openai=openai, config=config, attachments=attachments + ), ) self._original_process_response = openai.OpenAI._process_response self._original_async_process_response = openai.AsyncOpenAI._process_response @@ -94,3 +116,23 @@ def _uninstrument(self, **kwargs: Any) -> None: openai.AsyncOpenAI.request = self._original_async_request openai.OpenAI._process_response = self._original_process_response openai.AsyncOpenAI._process_response = self._original_async_process_response + + +_ATTACHMENT_EXPORTS = ( + "capture_openai_attachments", + "project_openai_media", + "capture_openai_speech", + "observe_openai_speech", + "capture_openai_audio_input", + "openai_attachment_capabilities", +) + + +def __getattr__(name: str) -> Any: + # Lazy: these need a fi-instrumentation-otel release with attachments, which + # plain tracing does not. + if name in _ATTACHMENT_EXPORTS: + from traceai_openai import _attachments + + return getattr(_attachments, name) + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/python/frameworks/openai/traceai_openai/_attachments.py b/python/frameworks/openai/traceai_openai/_attachments.py new file mode 100644 index 000000000..c89175f4f --- /dev/null +++ b/python/frameworks/openai/traceai_openai/_attachments.py @@ -0,0 +1,325 @@ +"""OpenAI media capture through the shared, provider-owned attachment manager. + +Mirrors ``@traceai/openai`` (``captureOpenAIAttachments`` / ``projectOpenAIMedia``): +typed inline parts only, preflight before any read or decode, one shared +registry per manager so a second observation of the same span and position +returns the first occurrence and uploads nothing. Speech output is captured +from bytes the SDK already buffered, or from chunks the application consumed. +Nothing here fetches URLs, resolves provider file IDs, or reads open files. + +This module imports ``fi_instrumentation.attachments``; the instrumentor imports +it only when an attachment manager is passed. +""" + +from types import MappingProxyType +from typing import Any, Mapping, Optional + +from fi_instrumentation.attachments.media_capture import ( + EMPTY, + FILE_MIMES, + IMAGE_MIMES, + MAX_INLINE_BYTES, + ConsumedAudio, + Observation, + Occurrence, + active, + attach_audio, + attach_inline, + get, + items, + local_capture_policy, + request_files, +) + +__all__ = ( + "capture_openai_attachments", + "project_openai_media", + "capture_openai_speech", + "observe_openai_speech", + "capture_openai_audio_input", + "openai_attachment_capabilities", +) + +#: Declarative handoff for auto/wire owners and reconciliation. +openai_attachment_capabilities = MappingProxyType({ + "schema_version": 1, + "operations": ( + "chat.completions.create.input", "responses.create.input", "responses.create.output", + "responses.stream.output", "images.generate.output", "audio.speech.create.output", + "audio.transcriptions.create.input", "audio.translations.create.input", + ), + "sources": ("inline_base64", "sdk_buffered_bytes", "consumed_reader"), + # Fail closed: nothing is attempted for these. Streamed Responses output is captured only from the + # terminal response.completed event the application consumed; partial (preview) images never are. + "unsupported": ( + "remote_url", "provider_file", "partial_image", "images.generate.stream", "images.edit", + "images.create_variation", "open_file_object", "speech.stream_format.sse", + ), + "max_inline_bytes": MAX_INLINE_BYTES, +}) + +_ROLES = ("user", "assistant", "tool", "system") + + +def _key(operation: str, direction: str, message_index: int, part_index: int) -> str: + return "%s/%s/%d/%d" % (operation, direction, message_index, part_index) + + +def _image_mime(fmt: Any) -> str: + # The Images API and image_generation_call default to PNG; a typed format wins. + return "image/jpeg" if fmt == "jpeg" else "image/webp" if fmt == "webp" else "image/png" + + +def capture_openai_attachments(manager, span, payload: Any, *, operation: str, direction: str, + trace_config=None, capture=None, + image_format: Optional[str] = None) -> Mapping[str, Occurrence]: + """Capture typed inline media of one OpenAI request or response. + + ``operation`` is ``"chat"``, ``"responses"`` or ``"images"``; ``direction`` + ``"input"`` or ``"output"``. ``span`` is the span the instrumentor emitted + for this call. Returns the admitted positions (position key -> occurrence) + so the caller can project them out of span telemetry; empty when no manager, + disabled, unsampled or denied. Never raises and never changes ``payload``. + """ + observation = Observation(manager, span, "openai") + if not observation.usable: + return EMPTY + try: + def attach(read, mimes, message_index, part_index, purpose, *, raw=False, role="user", tool_call_id=None): + policy = local_capture_policy(trace_config, direction=direction, base=capture, + image=mimes[0].startswith("image/")) + observation.visit(_key(operation, direction, message_index, part_index), lambda: attach_inline( + manager, span, read, mimes, raw=raw, direction=direction, purpose=purpose, capture=policy, + message_index=message_index, part_index=part_index, role=role, tool_call_id=tool_call_id)) + + def parts(content, message_index, role): + for part_index, part in enumerate(items(content)): + kind = get(part, "type") + if kind == "image_url": + attach(lambda p=part: get(get(p, "image_url"), "url"), IMAGE_MIMES, message_index, part_index, + "model_input", role=role) + elif kind == "input_image": + attach(lambda p=part: get(p, "image_url"), IMAGE_MIMES, message_index, part_index, + "model_input", role=role) + elif kind == "input_file": + attach(lambda p=part: get(p, "file_data"), FILE_MIMES, message_index, part_index, + "document", role=role) + elif kind == "file": + attach(lambda p=part: get(get(p, "file"), "file_data"), FILE_MIMES, message_index, part_index, + "document", role=role) + + if direction == "output" and get(payload, "error"): + return EMPTY + if direction == "input": + messages = get(payload, "messages" if operation == "chat" else "input") \ + if operation in ("chat", "responses") else None + for message_index, message in enumerate(items(messages)): + kind = get(message, "type") + if kind == "computer_call_output": + output = get(message, "output") + # A screenshot is model-bound input; its call id keeps the causal link to the computer_call. + if get(output, "type") == "computer_screenshot": + attach(lambda o=output: get(o, "image_url"), IMAGE_MIMES, message_index, 0, + "computer_screenshot", role="tool", tool_call_id=get(message, "call_id")) + elif kind in (None, "message"): + role = get(message, "role") + parts(get(message, "content"), message_index, role if role in _ROLES else "unknown") + elif operation == "images": + mime = _image_mime(get(payload, "output_format") or image_format) + for index, item in enumerate(items(get(payload, "data"))): + attach(lambda i=item: get(i, "b64_json"), (mime,), index, 0, "model_output", raw=True, + role="assistant") + elif operation == "responses": + for index, item in enumerate(items(get(payload, "output"))): + if get(item, "type") == "image_generation_call" and get(item, "status") == "completed": + attach(lambda i=item: get(i, "result"), (_image_mime(get(item, "output_format")),), index, 0, + "model_output", raw=True, role="assistant") + except Exception: + pass # Instrumentation must not throw or log source-bearing errors. + return MappingProxyType(dict(observation.captured)) if observation.captured else EMPTY + + +def _uri(occurrence: Occurrence) -> str: + return "fi-attachment:" + occurrence.occurrence_id + + +def project_openai_media(payload: Any, captured: Mapping[str, Occurrence], operation: str, direction: str) -> Any: + """Telemetry-only copy: ONLY admitted positions become ``fi-attachment:``. + + With nothing admitted the SAME object is returned. The SDK request and + response are never mutated; only plain mappings are copied. + """ + if not captured or not isinstance(payload, Mapping): + return payload + try: + field = ("messages" if operation == "chat" else "input") if direction == "input" else "output" + values = payload.get(field) + if not isinstance(values, list): + return payload + + def at(message_index, part_index): + return captured.get(_key(operation, direction, message_index, part_index)) + + def message(value, message_index): + if not isinstance(value, Mapping): + return value + kind = value.get("type") + if kind == "computer_call_output": + occurrence = at(message_index, 0) + output = value.get("output") + if occurrence and isinstance(output, Mapping): + return {**value, "output": {**output, "image_url": _uri(occurrence)}} + return value + if kind == "image_generation_call": + occurrence = at(message_index, 0) + return {**value, "result": _uri(occurrence)} if occurrence else value + content = value.get("content") + if not isinstance(content, list): + return value + return {**value, "content": [part(item, message_index, index) for index, item in enumerate(content)]} + + def part(value, message_index, part_index): + occurrence = at(message_index, part_index) + if not occurrence or not isinstance(value, Mapping): + return value + kind = value.get("type") + if kind == "image_url" and isinstance(value.get("image_url"), Mapping): + return {**value, "image_url": {**value["image_url"], "url": _uri(occurrence)}} + if kind == "input_image": + return {**value, "image_url": _uri(occurrence)} + if kind == "input_file": + return {**value, "file_data": _uri(occurrence)} + if kind == "file" and isinstance(value.get("file"), Mapping): + return {**value, "file": {**value["file"], "file_data": _uri(occurrence)}} + return value + + return {**payload, field: [message(value, index) for index, value in enumerate(values)]} + except Exception: + return payload + + +def _speech_policy(trace_config, capture): + return local_capture_policy(trace_config, direction="output", base=capture) + + +def capture_openai_speech(manager, span, data: Any, *, request: Any = None, trace_config=None, + capture=None) -> Optional[Occurrence]: + """Speech audio the SDK already buffered in memory (never a stream or file).""" + observation = Observation(manager, span, "openai") + if not observation.usable: + return None + observation.visit(_key("speech", "output", 0, 0), lambda: attach_audio( + manager, span, data, direction="output", purpose="model_output", role="assistant", + capture=_speech_policy(trace_config, capture), declared_format=get(request, "response_format"))) + return observation.captured.get(_key("speech", "output", 0, 0)) + + +def observe_openai_speech(manager, span, *, request: Any = None, trace_config=None, + capture=None) -> Optional[ConsumedAudio]: + """Observer for speech chunks the application consumes; None when nothing can be captured.""" + if not active(manager) or Observation(manager, span, "openai").key is None: + return None + return ConsumedAudio(manager, span, direction="output", purpose="model_output", role="assistant", + capture=_speech_policy(trace_config, capture), + declared_format=get(request, "response_format")) + + +def capture_openai_audio_input(manager, span, files: Any, *, operation: str = "transcriptions", + trace_config=None, capture=None) -> Mapping[str, Occurrence]: + """Transcription/translation input from multipart entries the SDK already holds as bytes. + + The SDK reads a path into bytes before sending, so those bytes are captured. + Open file objects (including BytesIO) are the application's stream and are + never read: they report nothing. + """ + observation = Observation(manager, span, "openai") + if not observation.usable: + return EMPTY + policy = local_capture_policy(trace_config, direction="input", base=capture) + for index, content in request_files(files): + if isinstance(content, (bytes, bytearray)): + observation.visit(_key(operation, "input", index, 0), lambda c=content, i=index: attach_audio( + manager, span, c, direction="input", purpose="model_input", role="user", capture=policy, + message_index=i, part_index=0)) + return MappingProxyType(dict(observation.captured)) if observation.captured else EMPTY + + +# ----------------------------------------------------------- instrumentor glue +_ROUTES = (("/chat/completions", "chat"), ("/responses", "responses"), ("/images/generations", "images"), + ("/audio/speech", "speech"), ("/audio/transcriptions", "transcriptions"), + ("/audio/translations", "translations")) + + +def _route(options: Any) -> Optional[str]: + url = getattr(options, "url", None) + if not isinstance(url, str): + return None + path = url.split("?", 1)[0].rstrip("/") + for suffix, route in _ROUTES: + if path.endswith(suffix): + return route + return None + + +class RequestMedia: + """Per-call capture state the instrumentor threads through one request.""" + + __slots__ = ("manager", "route", "options", "config", "request", "span", "output_captured") + + def __init__(self, manager, route: str, options: Any, config) -> None: + self.manager, self.route, self.options, self.config = manager, route, options, config + self.request: Any = None + self.span = None + self.output_captured: Mapping[str, Occurrence] = EMPTY + + def capture_input(self, span, request_parameters: Mapping) -> Mapping: + """Capture request media on the emitted span; return the telemetry-only projected parameters.""" + self.span, self.request = span, request_parameters + if self.route in ("chat", "responses"): + captured = capture_openai_attachments(self.manager, span, request_parameters, operation=self.route, + direction="input", trace_config=self.config) + return project_openai_media(request_parameters, captured, self.route, "input") + if self.route in ("transcriptions", "translations"): + capture_openai_audio_input(self.manager, span, getattr(self.options, "files", None), + operation=self.route, trace_config=self.config) + return request_parameters + + def capture_output(self, response: Any) -> Mapping[str, Occurrence]: + if self.span is None: + return EMPTY + if self.route == "responses": + self.output_captured = capture_openai_attachments(self.manager, self.span, response, + operation="responses", direction="output", + trace_config=self.config) + elif self.route == "images": + capture_openai_attachments(self.manager, self.span, response, operation="images", direction="output", + trace_config=self.config, image_format=get(self.request, "output_format")) + elif self.route == "speech": + data = None + try: + # Only a body the SDK already read; an unread streamed body raises instead of reading. + data = response.content + except Exception: + data = None + capture_openai_speech(self.manager, self.span, data, request=self.request, trace_config=self.config) + return self.output_captured + + def project_output(self, output: Any) -> Any: + return project_openai_media(output, self.output_captured, "responses", "output") + + def speech_bytes(self, data: Any) -> None: + if self.route == "speech" and self.span is not None: + capture_openai_speech(self.manager, self.span, data, request=self.request, trace_config=self.config) + + def speech_observer(self) -> Optional[ConsumedAudio]: + if self.route != "speech" or self.span is None: + return None + return observe_openai_speech(self.manager, self.span, request=self.request, trace_config=self.config) + + +def request_media(manager, options: Any, config) -> Optional[RequestMedia]: + """None unless an active manager is bound and this request is a media route.""" + if not active(manager): + return None + route = _route(options) + return RequestMedia(manager, route, options, config) if route else None diff --git a/python/frameworks/openai/traceai_openai/_request.py b/python/frameworks/openai/traceai_openai/_request.py index 111cdc6e3..81a0869ca 100644 --- a/python/frameworks/openai/traceai_openai/_request.py +++ b/python/frameworks/openai/traceai_openai/_request.py @@ -73,6 +73,7 @@ def _start_as_current_span( context_attributes: Iterable[Tuple[str, AttributeValue]], extra_attributes: Iterable[Tuple[str, AttributeValue]], secrets: Tuple[str, ...] = (), + prepare: Optional[Callable[[Any], Tuple[Iterable[Tuple[str, AttributeValue]], Any]]] = None, ) -> Iterator[_WithSpan]: # Keep primary attributes so finalization can restore them after optional # details: OTel's bounded attribute map evicts the oldest entries. @@ -83,6 +84,17 @@ def _start_as_current_span( except Exception: logger.exception("Failed to start span") span = INVALID_SPAN + if prepare is not None: + # Attachment capture needs the emitted span, and admitted media must be + # projected out of content attributes, so those are set after capture. + try: + deferred, extra_attributes = prepare(span) + deferred = {key: redact(value, secrets) for key, value in deferred} + if deferred: + span.set_attributes(deferred) + initial_attributes.update(deferred) + except Exception: + logger.exception("Failed to set request attributes") with trace_api.use_span( span, end_on_exit=False, @@ -109,6 +121,7 @@ class _WithOpenAI(ABC): "_response_attributes_extractor", "_response_accumulator_factories", "_config", + "_attachments", ) def __init__( @@ -116,11 +129,14 @@ def __init__( openai: ModuleType, *args: Any, config: Optional[TraceConfig] = None, + attachments: Any = None, **kwargs: Any, ) -> None: super().__init__(*args, **kwargs) self._openai = openai self._config = config if config is not None else TraceConfig() + # Provider-owned shared attachment manager, or None (no capture, no imports). + self._attachments = attachments self._stream_types = (openai.Stream, openai.AsyncStream) self._request_attributes_extractor = _RequestAttributesExtractor(openai=openai) self._response_attributes_extractor = _ResponseAttributesExtractor( @@ -151,6 +167,54 @@ def responses_accumulator(request_parameters: _RequestParameters) -> Any: openai.types.responses.response.Response: responses_accumulator, } + def _request_media(self, options: Any) -> Any: + """Per-call attachment state for media routes; None leaves tracing unchanged.""" + if self._attachments is None: + return None + try: + from traceai_openai._attachments import request_media + + return request_media(self._attachments, options, self._config) + except Exception: + logger.debug("Attachment capture unavailable", exc_info=True) + return None + + def _span_arguments( + self, instance: Any, cast_to: type, request_parameters: Mapping[str, Any], media: Any, + holder: dict, + ) -> dict: + if media is None: + return dict( + attributes=chain( + self._get_attributes_from_instance(instance, request_parameters), + self._get_attributes_from_request( + cast_to=cast_to, + request_parameters=request_parameters, + ), + ), + extra_attributes=self._get_extra_attributes_from_request( + cast_to=cast_to, + request_parameters=request_parameters, + ), + ) + + def prepare(span: Any) -> Tuple[Iterable[Tuple[str, AttributeValue]], Any]: + projected = media.capture_input(span, request_parameters) + holder["request_parameters"] = projected + return ( + self._get_attributes_from_request(cast_to=cast_to, request_parameters=projected), + self._get_extra_attributes_from_request(cast_to=cast_to, request_parameters=projected), + ) + + return dict( + attributes=chain( + self._get_attributes_from_instance(instance, request_parameters), + ((SpanAttributes.GEN_AI_SPAN_KIND, self._get_span_kind(cast_to=cast_to)),), + ), + extra_attributes=(), + prepare=prepare, + ) + def _get_span_kind(self, cast_to: type) -> str: return ( FiSpanKindValues.EMBEDDING.value @@ -257,6 +321,7 @@ def _finalize_response( request_parameters: Mapping[str, Any], instance: Any = None, request_options: Any = None, + media: Any = None, ) -> Any: """ Monkey-patch the response object to trace the stream, or finish tracing if the response is @@ -296,6 +361,10 @@ def _finalize_response( config=self._config, ) + if media is not None and isinstance(response_accumulator, _ResponsesAccumulator): + # Streamed Responses: only the terminal response.completed event the application consumed. + response_accumulator.observe_media(media.capture_output, media.project_output) + def on_parse(parsed: Any) -> Any: if self._is_streaming(parsed): stream_wrapper = _AsyncStream if self._is_async_stream(parsed) else _Stream @@ -336,6 +405,8 @@ def on_parse(parsed: Any) -> Any: except Exception: body = None status = _raw_response_status(body) + if media is not None: + telemetry_output = self._media_output(media, parsed, telemetry_output) add_io_to_span_attributes(with_span, None, telemetry_output, config=self._config) _finish_tracing( status=status, @@ -358,6 +429,8 @@ def on_parse(parsed: Any) -> Any: _AsyncRawResponse if iscoroutinefunction(response.parse) else _RawResponse ) raw = raw_wrapper(response, with_span, response_accumulator, on_parse) + if media is not None and media.route == "speech": + raw._self_media = media if isinstance(response, LegacyAPIResponse): # Preserve eager legacy non-stream telemetry. Stream parse caches # remain SDK-owned, and every later explicit parse type is traced. @@ -377,6 +450,20 @@ def on_parse(parsed: Any) -> Any: return raw return on_parse(response) + def _media_output(self, media: Any, parsed: Any, telemetry_output: Any) -> Any: + """Capture output media; telemetry carries fi-attachment URIs only for admitted positions.""" + try: + captured = media.capture_output(parsed) + if captured and media.route == "responses" and telemetry_output is not None: + body = response_output(telemetry_output, self._config) + output_text = getattr(telemetry_output, "output_text", None) + if output_text: + body = {"output_text": output_text, **body} + return media.project_output(body) + except Exception: + logger.debug("Failed to capture response attachments", exc_info=True) + return telemetry_output + def _raw_embeddings_json(self, response: CreateEmbeddingResponse) -> str: """The raw embeddings response; vectors are replaced by their dimension count when FI_HIDE_EMBEDDING_VECTORS / hide_embedding_vectors is set.""" @@ -414,34 +501,23 @@ def __call__( cast_to, request_parameters = _parse_request_args(args) secrets = request_credentials(instance, args[1], request_parameters) request_parameters = safe_request(request_parameters) - # Extract input data more robustly - input_data = ( - request_parameters.get("messages") - or request_parameters.get("prompt") - or request_parameters.get("input") - ) llm_tools = request_parameters.get("tools") span_name: str = cast_to.__name__.split(".")[-1] + media = self._request_media(args[1]) except Exception: logger.exception("Failed to parse request args") return wrapped(*args, **kwargs) + holder: dict = {} with self._start_as_current_span( span_name=span_name, secrets=secrets, - attributes=chain( - self._get_attributes_from_instance(instance, request_parameters), - self._get_attributes_from_request( - cast_to=cast_to, - request_parameters=request_parameters, - ), - ), context_attributes=get_attributes_from_context(), - extra_attributes=self._get_extra_attributes_from_request( - cast_to=cast_to, - request_parameters=request_parameters, - ), + **self._span_arguments(instance, cast_to, request_parameters, media, holder), ) as with_span: + # Telemetry-only parameters: admitted media is replaced by fi-attachment URIs. + request_parameters = holder.get("request_parameters", request_parameters) + input_data = _input_data(request_parameters) # Add input data to span attributes before the request with_span.set_attribute( SpanAttributes.GEN_AI_TOOL_DEFINITIONS, safe_json_dumps(llm_tools) @@ -460,6 +536,7 @@ def __call__( request_parameters=request_parameters, instance=instance, request_options=args[1], + media=media, ) except BaseException as e: with_span.add_secrets(request_credentials(instance, getattr(e, "request", None), {})) @@ -486,34 +563,23 @@ async def __call__( cast_to, request_parameters = _parse_request_args(args) secrets = request_credentials(instance, args[1], request_parameters) request_parameters = safe_request(request_parameters) - # Extract input data more robustly - input_data = ( - request_parameters.get("messages") - or request_parameters.get("prompt") - or request_parameters.get("input") - ) llm_tools = request_parameters.get("tools") span_name: str = cast_to.__name__.split(".")[-1] + media = self._request_media(args[1]) except Exception: logger.exception("Failed to parse request args") return await wrapped(*args, **kwargs) + holder: dict = {} with self._start_as_current_span( span_name=span_name, secrets=secrets, - attributes=chain( - self._get_attributes_from_instance(instance, request_parameters), - self._get_attributes_from_request( - cast_to=cast_to, - request_parameters=request_parameters, - ), - ), context_attributes=get_attributes_from_context(), - extra_attributes=self._get_extra_attributes_from_request( - cast_to=cast_to, - request_parameters=request_parameters, - ), + **self._span_arguments(instance, cast_to, request_parameters, media, holder), ) as with_span: + # Telemetry-only parameters: admitted media is replaced by fi-attachment URIs. + request_parameters = holder.get("request_parameters", request_parameters) + input_data = _input_data(request_parameters) # Add input data to span attributes before the request with_span.set_attribute( SpanAttributes.GEN_AI_TOOL_DEFINITIONS, safe_json_dumps(llm_tools) @@ -545,6 +611,7 @@ async def __call__( request_parameters=request_parameters, instance=instance, request_options=args[1], + media=media, ) except Exception: logger.exception( @@ -554,6 +621,14 @@ async def __call__( return response +def _input_data(request_parameters: Mapping[str, Any]) -> Any: + return ( + request_parameters.get("messages") + or request_parameters.get("prompt") + or request_parameters.get("input") + ) + + def _parse_request_args( args: Tuple[type, Any], *, include_extra_body: bool = False, ) -> Tuple[type, Mapping[str, Any]]: diff --git a/python/frameworks/openai/traceai_openai/_response_accumulator.py b/python/frameworks/openai/traceai_openai/_response_accumulator.py index 01a6df777..d12b728de 100644 --- a/python/frameworks/openai/traceai_openai/_response_accumulator.py +++ b/python/frameworks/openai/traceai_openai/_response_accumulator.py @@ -75,6 +75,8 @@ class _ResponsesAccumulator: "_request_parameters", "_response_attributes_extractor", "_chat_completion_type", + "_on_completed", + "_project_output", ) def __init__( @@ -91,6 +93,12 @@ def __init__( self._is_null = True self._cached_result: Optional[Dict[str, Any]] = None self._values: Optional["ResponseStreamEvent"] = None + self._on_completed: Optional[Callable[[Any], Any]] = None + self._project_output: Optional[Callable[[Any], Any]] = None + + def observe_media(self, on_completed: Callable[[Any], Any], project_output: Callable[[Any], Any]) -> None: + """Attachment hooks: capture from the consumed terminal event, project telemetry output.""" + self._on_completed, self._project_output = on_completed, project_output def process_chunk(self, chunk: Any) -> None: if getattr(chunk, "type", None) in { @@ -99,6 +107,11 @@ def process_chunk(self, chunk: Any) -> None: self._is_null = False self._cached_result = None self._values = chunk + if self._on_completed is not None and chunk.type == "response.completed": + try: + self._on_completed(getattr(chunk, "response", None)) + except Exception: + pass def _result(self) -> Any: if self._is_null: @@ -144,6 +157,8 @@ def get_attributes(self) -> Iterator[Tuple[str, AttributeValue]]: # Keep response-owned data, never echoed request fields, and lead # with the answer just as non-streaming Responses previews do. output = response_output(result, self._config) + if self._project_output is not None: + output = self._project_output(output) if output_content: output = {"output_text": output_content, **output} yield from _as_output_attributes(_io_value_and_type(output)) diff --git a/python/frameworks/openai/traceai_openai/_stream.py b/python/frameworks/openai/traceai_openai/_stream.py index 2c9073beb..94f9e5c91 100644 --- a/python/frameworks/openai/traceai_openai/_stream.py +++ b/python/frameworks/openai/traceai_openai/_stream.py @@ -136,6 +136,25 @@ def __init__( self._self_on_parse = on_parse self._self_parsed = {} self._self_body_status: Optional[trace_api.Status] = None + # Speech attachment capture (set by the request wrapper only for audio.speech with a manager). + self._self_media: Any = None + + def _speech_observer(self) -> Any: + # Only the FIRST consumer of the body is observed; a later read is not the whole body. + media, self._self_media = self._self_media, None + return media.speech_observer() if media is not None else None + + def _speech_bytes(self, method: str, result: Any) -> None: + if method == "read" and self._self_media is not None: + media, self._self_media = self._self_media, None + media.speech_bytes(result) + + def _stream_to_file_via_proxy(self) -> Any: + """The SDK's own stream_to_file, bound to this proxy so its iter_bytes is observed.""" + if self._self_media is None: + return None + method = getattr(type(self.__wrapped__), "stream_to_file", None) + return method if callable(method) else None def _body_buffer(self) -> Optional[_RawBodyBuffer]: if ( @@ -172,6 +191,7 @@ def _consume_body(self, method: str, *args: Any, **kwargs: Any) -> Any: except BaseException as exc: _finish_on_exit(self, exc) raise + self._speech_bytes(method, result) self._record_body_consumption() return result @@ -186,17 +206,24 @@ def json(self, *args: Any, **kwargs: Any) -> Any: def _consume_iterator(self, method: str, *args: Any, **kwargs: Any) -> Iterator[Any]: buffer = self._body_buffer() + audio = self._speech_observer() if method == "iter_bytes" else None try: for chunk in getattr(self.__wrapped__, method)(*args, **kwargs): if buffer is not None: buffer.add(chunk) if method == "iter_lines": buffer.add(b"\n") + if audio is not None: + audio.write(chunk) yield chunk except BaseException as exc: + if audio is not None: + audio.abort() _finish_on_exit(self, exc) raise else: + if audio is not None: + audio.finish() # The caller may stop iteration early; only exhaustion consumes it. self._record_body_consumption(buffer) @@ -210,6 +237,14 @@ def iter_lines(self, *args: Any, **kwargs: Any) -> Iterator[str]: return self._consume_iterator("iter_lines", *args, **kwargs) def stream_to_file(self, *args: Any, **kwargs: Any) -> None: + method = self._stream_to_file_via_proxy() + if method is not None: + try: + method(self, *args, **kwargs) + except BaseException as exc: + _finish_on_exit(self, exc) + raise + return self._consume_body("stream_to_file", *args, **kwargs) @property @@ -307,6 +342,7 @@ async def _consume_body(self, method: str, *args: Any, **kwargs: Any) -> Any: except BaseException as exc: _finish_on_exit(self, exc) raise + self._speech_bytes(method, result) self._record_body_consumption() return result @@ -321,17 +357,24 @@ async def json(self, *args: Any, **kwargs: Any) -> Any: async def _consume_iterator(self, method: str, *args: Any, **kwargs: Any) -> AsyncIterator[Any]: buffer = self._body_buffer() + audio = self._speech_observer() if method == "iter_bytes" else None try: async for chunk in getattr(self.__wrapped__, method)(*args, **kwargs): if buffer is not None: buffer.add(chunk) if method == "iter_lines": buffer.add(b"\n") + if audio is not None: + audio.write(chunk) yield chunk except BaseException as exc: + if audio is not None: + audio.abort() _finish_on_exit(self, exc) raise else: + if audio is not None: + audio.finish() self._record_body_consumption(buffer) async def iter_bytes(self, *args: Any, **kwargs: Any) -> AsyncIterator[bytes]: @@ -347,6 +390,14 @@ async def iter_lines(self, *args: Any, **kwargs: Any) -> AsyncIterator[str]: yield chunk async def stream_to_file(self, *args: Any, **kwargs: Any) -> None: + method = self._stream_to_file_via_proxy() + if method is not None: + try: + await method(self, *args, **kwargs) + except BaseException as exc: + _finish_on_exit(self, exc) + raise + return await self._consume_body("stream_to_file", *args, **kwargs) async def parse(self, *args: Any, **kwargs: Any) -> Any: diff --git a/python/tests/attachments_providers/conftest.py b/python/tests/attachments_providers/conftest.py new file mode 100644 index 000000000..5f295ddf1 --- /dev/null +++ b/python/tests/attachments_providers/conftest.py @@ -0,0 +1,366 @@ +"""Loopback harness for provider attachment tests: synthetic data only. + +- ``f01``: a TLS loopback F01 attachment server (sessions, uploads, content, + complete). It records every request and uploaded payload. +- ``provider``: a plain-HTTP loopback that answers like the OpenAI, Anthropic + and Groq APIs (JSON and SSE), so the REAL installed SDKs run end to end. +- ``spans``: an OpenTelemetry provider with an in-memory exporter. +""" + +import base64 +import io +import json +import os +import ssl +import struct +import subprocess +import sys +import threading +import uuid +import wave +import zlib +from datetime import datetime, timedelta, timezone +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).parent)) + +pytest.importorskip("fi_instrumentation.attachments.media_capture") + +from fi_instrumentation.attachments import AttachmentConfig, AttachmentManager # noqa: E402 +from fi_instrumentation.attachments.protocol import HTTPTransport # noqa: E402 +from opentelemetry.sdk.trace import TracerProvider # noqa: E402 +from opentelemetry.sdk.trace.export import SimpleSpanProcessor # noqa: E402 +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter # noqa: E402 + + +def png(width=2, height=2, seed=0) -> bytes: + """A small valid PNG; ``seed`` changes the pixels so payloads are distinguishable.""" + raw = b"".join(b"\x00" + bytes((seed + x + y) % 256 for x in range(width * 3)) for y in range(height)) + + def chunk(kind, data): + return struct.pack(">I", len(data)) + kind + data + struct.pack(">I", zlib.crc32(kind + data)) + + return (b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0)) + + chunk(b"IDAT", zlib.compress(raw)) + chunk(b"IEND", b"")) + + +def wav(frames=800, seed=0) -> bytes: + buffer = io.BytesIO() + with wave.open(buffer, "wb") as writer: + writer.setnchannels(1) + writer.setsampwidth(2) + writer.setframerate(8000) + writer.writeframes(bytes((seed + i) % 256 for i in range(frames * 2))) + return buffer.getvalue() + + +PDF = b"%PDF-1.4\n1 0 obj<<>>endobj\ntrailer<<>>\n%%EOF\n" + + +def b64(data: bytes) -> str: + return base64.b64encode(data).decode() + + +def expiry(seconds=600): + return (datetime.now(timezone.utc) + timedelta(seconds=seconds)).isoformat() + + +@pytest.fixture(autouse=True) +def clean_env(monkeypatch): + for key in list(os.environ): + if key.startswith(("FI_ATTACHMENTS_", "FI_HIDE_")) or key in ("FI_PII_REDACTION",): + monkeypatch.delenv(key, raising=False) + monkeypatch.setenv("FI_BASE_URL", "http://127.0.0.1:9") + monkeypatch.setenv("FI_GRPC_URL", "http://127.0.0.1:9") + for key in ("HTTP_PROXY", "HTTPS_PROXY", "ALL_PROXY", "http_proxy", "https_proxy", "all_proxy"): + monkeypatch.delenv(key, raising=False) + monkeypatch.setenv("NO_PROXY", "127.0.0.1,localhost") + + +class F01: + def __init__(self, server, thread, cert, state): + self.server, self.thread, self.cert, self.state = server, thread, cert, state + + @property + def endpoint(self): + return "https://localhost:%d" % self.server.server_port + + def transport(self): + return HTTPTransport(self.endpoint, {"X-API-Key": "SYNTHETIC_KEY", "X-Secret-Key": "SYNTHETIC_SECRET"}, + ssl_context=ssl.create_default_context(cafile=str(self.cert))) + + def manager(self, **config): + manager = AttachmentManager(config=AttachmentConfig(enabled=True, **config), project_name="synthetic", + transport=self.transport()) + assert manager.ready(5), manager.status() + return manager + + @property + def uploads(self): + """Initialize bodies (POST .../uploads) in arrival order.""" + return [body for path, body in self.state["requests"] if path.endswith("/uploads")] + + @property + def payloads(self): + return list(self.state["payloads"]) + + def after_bootstrap(self): + return [path for path, _ in self.state["requests"] if not path.endswith("attachment-sessions")] + + +@pytest.fixture +def f01(tmp_path): + key, cert = tmp_path / "synthetic.key", tmp_path / "synthetic.crt" + subprocess.run(["openssl", "req", "-x509", "-newkey", "rsa:2048", "-nodes", "-keyout", str(key), "-out", + str(cert), "-days", "1", "-subj", "/CN=localhost", "-addext", "subjectAltName=DNS:localhost"], + check=True, capture_output=True) + lock = threading.Lock() + state = {"requests": [], "payloads": [], "uploads": {}, "policy": {}} + scope = {k: str(uuid.uuid4()) for k in ("organization_id", "workspace_id", "project_id")} + + class Handler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def log_message(self, *args): + pass + + def respond(self, status, body): + data = json.dumps(body).encode() + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(data))) + self.end_headers() + self.wfile.write(data) + + def do_POST(self): + body = json.loads(self.rfile.read(int(self.headers.get("Content-Length") or 0)) or b"{}") + with lock: + state["requests"].append((self.path, body)) + if self.path.endswith("attachment-sessions"): + policy = {"version": "v1", "enabled": True, "allowed_sources": ["Bytes"], + "allowed_modalities": ["audio", "image", "document", "file"], + "require_media_redaction": False, "require_metadata_removal": False, + "allow_unredacted_media_with_text_pii": False, "record_audio": False, + "record_video": False, **state["policy"]} + self.respond(200, {"schema_version": 1, "scope": scope, + "capture_session_id": body["capture_session_id"], "expires_at": expiry(), + "transfer_modes": ["authenticated_stream"], "policy": policy, + "limits": {"max_upload_bytes": 8388608, "segment_bytes": 8388608, + "segment_duration_ms": 30000, "recording_bytes": 1073741824, + "recording_duration_ms": 7200000, "active_transfers": 16, + "active_recordings": 4, "reserved_bytes": 1073741824, + "validation_jobs": 1024, "initializations_per_minute": 120, + "links_per_minute": 1000, "upload_bytes_per_second": 8388608, + "playback_bytes_per_second": 1048576, "transfer_timeout_seconds": 30, + "grant_ttl_seconds": 60, "revocation_check_seconds": 5, + **state.get("limits", {})}}) + elif self.path.endswith("/uploads"): + upload_id = str(uuid.uuid4()) + with lock: + state["uploads"][upload_id] = body + self.respond(201, {"upload_id": upload_id, "attachment_id": body["attachment_id"], + "occurrence_id": body["occurrence_id"], "state": "pending_upload", + "reserved_bytes": body["declared_bytes"], "expires_at": expiry(60), + "transfer_mode": "authenticated_stream", + "write_capability": "SYNTHETIC_UPLOAD_ONLY"}) + elif self.path.endswith("/complete"): + upload_id = self.path.rstrip("/").split("/")[-2] + self.respond(202, {"upload_id": upload_id, "state": "validating", "reason": "binding_pending"}) + elif self.path.endswith("/abort"): + upload_id = self.path.rstrip("/").split("/")[-2] + self.respond(200, {"upload_id": upload_id, "state": "failed", "reason": "incomplete"}) + else: + self.respond(404, {}) + + def do_PUT(self): + payload = self.rfile.read(int(self.headers.get("Content-Length") or 0)) + upload_id = self.path.rstrip("/").split("/")[-2] + with lock: + state["requests"].append((self.path, None)) + state["payloads"].append(payload) + self.respond(200, {"upload_id": upload_id, "state": "pending_upload", "reason": "binding_pending"}) + + server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + context.load_cert_chain(certfile=str(cert), keyfile=str(key)) + server.socket = context.wrap_socket(server.socket, server_side=True) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + yield F01(server, thread, cert, state) + server.shutdown() + server.server_close() + thread.join(2) + + +# ------------------------------------------------------------------ provider +def _sse(events): + return "".join("event: %s\ndata: %s\n\n" % (event.get("type", "message"), json.dumps(event)) + for event in events).encode() + + +class ProviderState: + def __init__(self): + self.requests = [] + self.speech = wav(seed=7) + self.image = png(seed=9) + self.speech_chunk = 256 + + +def _chat(body): + if body.get("stream"): + chunks = [{"id": "c1", "object": "chat.completion.chunk", "created": 1, "model": "m", + "choices": [{"index": 0, "delta": {"role": "assistant", "content": "ok"}, "finish_reason": None}]}, + {"id": "c1", "object": "chat.completion.chunk", "created": 1, "model": "m", + "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}]}] + return "text/event-stream", "".join("data: %s\n\n" % json.dumps(c) for c in chunks).encode() + b"data: [DONE]\n\n" + return "application/json", json.dumps({ + "id": "c1", "object": "chat.completion", "created": 1, "model": "m", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}}).encode() + + +def _response(state, body, status="completed"): + output = [{"type": "message", "id": "m1", "status": "completed", "role": "assistant", + "content": [{"type": "output_text", "text": "ok", "annotations": []}]}] + if body.get("tools") and any(t.get("type") == "image_generation" for t in body["tools"]): + output.append({"type": "image_generation_call", "id": "ig1", "status": "completed", + "result": b64(state.image)}) + return {"id": "r1", "object": "response", "created_at": 1, "model": "m", "status": status, "output": output, + "parallel_tool_calls": True, "tool_choice": "auto", "tools": [], "error": None, + "incomplete_details": None, "instructions": None, "metadata": {}, "temperature": 1.0, "top_p": 1.0, + "usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2, + "input_tokens_details": {"cached_tokens": 0}, + "output_tokens_details": {"reasoning_tokens": 0}}} + + +def _responses(state, body): + if body.get("stream"): + final = _response(state, body) + created = dict(final, status="in_progress", output=[]) + events = [{"type": "response.created", "sequence_number": 0, "response": created}] + if any(item["type"] == "image_generation_call" for item in final["output"]): + events.append({"type": "response.image_generation_call.partial_image", "sequence_number": 1, + "item_id": "ig1", "output_index": 1, "partial_image_index": 0, + "partial_image_b64": b64(png(seed=3))}) + events.append({"type": "response.completed", "sequence_number": 2, "response": final}) + return "text/event-stream", _sse(events) + return "application/json", json.dumps(_response(state, body)).encode() + + +def _anthropic(body): + message = {"id": "msg_1", "type": "message", "role": "assistant", "model": body.get("model", "m"), + "content": [{"type": "text", "text": "ok"}], "stop_reason": "end_turn", "stop_sequence": None, + "usage": {"input_tokens": 1, "output_tokens": 1}} + if body.get("stream"): + events = [{"type": "message_start", "message": dict(message, content=[])}, + {"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}}, + {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "ok"}}, + {"type": "content_block_stop", "index": 0}, + {"type": "message_delta", "delta": {"stop_reason": "end_turn", "stop_sequence": None}, + "usage": {"output_tokens": 1}}, + {"type": "message_stop"}] + return "text/event-stream", _sse(events) + return "application/json", json.dumps(message).encode() + + +@pytest.fixture +def provider(): + state = ProviderState() + + class Handler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def log_message(self, *args): + pass + + def send(self, content_type, data, chunked=False): + self.send_response(200) + self.send_header("Content-Type", content_type) + if chunked: + self.send_header("Transfer-Encoding", "chunked") + self.end_headers() + for offset in range(0, len(data), state.speech_chunk): + piece = data[offset:offset + state.speech_chunk] + self.wfile.write(b"%x\r\n%s\r\n" % (len(piece), piece)) + self.wfile.write(b"0\r\n\r\n") + else: + self.send_header("Content-Length", str(len(data))) + self.end_headers() + self.wfile.write(data) + + def body(self): + if self.headers.get("Transfer-Encoding", "").lower() == "chunked": + parts = [] + while True: + size = int(self.rfile.readline().split(b";")[0].strip() or b"0", 16) + if not size: + self.rfile.readline() + return b"".join(parts) + parts.append(self.rfile.read(size)) + self.rfile.readline() + return self.rfile.read(int(self.headers.get("Content-Length") or 0)) + + def do_POST(self): + raw = self.body() + content_type = self.headers.get("Content-Type", "") + body = json.loads(raw) if content_type.startswith("application/json") and raw else {} + state.requests.append((self.path, body, raw)) + path = self.path.split("?")[0] + if path.endswith("/chat/completions"): + self.send(*_chat(body)) + elif path.endswith("/responses"): + self.send(*_responses(state, body)) + elif path.endswith("/images/generations"): + self.send("application/json", json.dumps({"created": 1, "data": [ + {"b64_json": b64(state.image)}, {"url": "https://images.invalid/x.png"}]}).encode()) + elif path.endswith("/audio/speech"): + self.send("audio/wav", state.speech, chunked=True) + elif path.endswith(("/audio/transcriptions", "/audio/translations")): + self.send("application/json", json.dumps({"text": "hello"}).encode()) + elif path.endswith("/messages"): + self.send(*_anthropic(body)) + else: + self.send_response(404) + self.send_header("Content-Length", "0") + self.end_headers() + + server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + state.url = "http://127.0.0.1:%d" % server.server_port + yield state + server.shutdown() + server.server_close() + thread.join(2) + + +@pytest.fixture +def spans(): + exporter = InMemorySpanExporter() + tracer_provider = TracerProvider() + tracer_provider.add_span_processor(SimpleSpanProcessor(exporter)) + yield tracer_provider, exporter + tracer_provider.shutdown() + + +def refs(span): + raw = (span.attributes or {}).get("fi.attachment.refs") + return json.loads(raw) if raw else [] + + +def wait_uploads(f01, count, timeout=10.0): + import time + + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if len(f01.payloads) >= count: + return + time.sleep(0.02) + + +def flush(manager): + result = manager.flush(10) + assert result.unfinished == 0, result diff --git a/python/tests/attachments_providers/test_anthropic_attachments.py b/python/tests/attachments_providers/test_anthropic_attachments.py new file mode 100644 index 000000000..ada8bdf6d --- /dev/null +++ b/python/tests/attachments_providers/test_anthropic_attachments.py @@ -0,0 +1,190 @@ +"""Real Anthropic SDK -> traceai_anthropic -> loopback F01: vision, PDF and computer-use screenshots.""" + +import asyncio +import json + +import pytest + +anthropic = pytest.importorskip("anthropic") +pytest.importorskip("traceai_anthropic") + +from conftest import PDF, b64, flush, png, refs # noqa: E402 +from fi_instrumentation import TraceConfig # noqa: E402 +from fi_instrumentation.attachments import AttachmentConfig, create_attachment_manager # noqa: E402 +from traceai_anthropic import AnthropicInstrumentor # noqa: E402 + +IMAGE = png(seed=11) +SHOT_1 = png(seed=12) +SHOT_2 = png(seed=13) +TOOL_IMAGE = png(seed=14) + + +@pytest.fixture +def instrument(spans): + tracer_provider, exporter = spans + state = {} + + def install(manager, config=None): + AnthropicInstrumentor().instrument(tracer_provider=tracer_provider, attachments=manager, + config=config or TraceConfig()) + state["on"] = True + return exporter + + yield install + if state.get("on"): + AnthropicInstrumentor().uninstrument() + + +def image(data, media_type="image/png"): + return {"type": "image", "source": {"type": "base64", "media_type": media_type, "data": b64(data)}} + + +def computer_use_request(stream=False): + """A computer-use loop: two screenshots returned for the computer tool, one ordinary tool image.""" + return dict( + model="claude-test", max_tokens=16, stream=stream, + tools=[{"type": "computer_20250124", "name": "computer", "display_width_px": 1024, + "display_height_px": 768}, + {"name": "chart", "description": "draws", "input_schema": {"type": "object"}}], + messages=[ + {"role": "user", "content": [ + {"type": "text", "text": "open the page"}, image(IMAGE), + {"type": "document", "source": {"type": "base64", "media_type": "application/pdf", "data": b64(PDF)}}, + {"type": "image", "source": {"type": "url", "url": "https://example.invalid/x.png"}}, + ]}, + {"role": "assistant", "content": [ + {"type": "tool_use", "id": "toolu_1", "name": "computer", "input": {"action": "screenshot"}}]}, + {"role": "user", "content": [ + {"type": "tool_result", "tool_use_id": "toolu_1", "content": [image(SHOT_1)]}]}, + {"role": "assistant", "content": [ + {"type": "tool_use", "id": "toolu_2", "name": "computer", "input": {"action": "left_click"}}, + {"type": "tool_use", "id": "toolu_3", "name": "chart", "input": {}}]}, + {"role": "user", "content": [ + {"type": "tool_result", "tool_use_id": "toolu_2", "content": [image(SHOT_2)]}, + {"type": "tool_result", "tool_use_id": "toolu_3", "content": [ + {"type": "text", "text": "chart"}, image(TOOL_IMAGE)]}]}, + ]) + + +def run(provider, request, async_): + if async_: + async def go(): + result = await anthropic.AsyncAnthropic(api_key="sk-synthetic", base_url=provider.url, + max_retries=0).messages.create(**request) + if request.get("stream"): + async for _ in result: + pass + asyncio.run(go()) + else: + result = anthropic.Anthropic(api_key="sk-synthetic", base_url=provider.url, + max_retries=0).messages.create(**request) + if request.get("stream"): + for _ in result: + pass + + +@pytest.mark.parametrize("async_", [False, True]) +@pytest.mark.parametrize("stream", [False, True]) +def test_vision_document_and_computer_screenshots(f01, provider, instrument, async_, stream): + manager = f01.manager() + exporter = instrument(manager) + run(provider, computer_use_request(stream), async_) + flush(manager) + assert sorted(f01.payloads) == sorted([IMAGE, PDF, SHOT_1, SHOT_2, TOOL_IMAGE]) + span = [s for s in exporter.get_finished_spans() if s.name in ("Messages", "AsyncMessages")][-1] + span_refs = refs(span) + by_purpose = {} + for ref in span_refs: + by_purpose.setdefault(ref["purpose"], []).append(ref) + # Causal order for the filmstrip: screenshots keep their message/block positions. + shots = sorted(by_purpose["computer_screenshot"], key=lambda r: r["message_index"]) + assert [(r["message_index"], r["part_index"], r["role"]) for r in shots] == [(2, 0, "tool"), (4, 0, "tool")] + assert [(r["message_index"], r["part_index"]) for r in by_purpose["tool_result"]] == [(4, 1)] + assert [(r["message_index"], r["part_index"]) for r in by_purpose["model_input"]] == [(0, 1)] + assert by_purpose["document"][0]["mime_type"] == "application/pdf" + tool_ids = sorted(u.get("tool_call_id") for u in f01.uploads if u.get("tool_call_id")) + assert tool_ids == ["toolu_1", "toolu_2", "toolu_3"] + exported = json.dumps(dict(span.attributes)) + for data in (IMAGE, PDF, SHOT_1, SHOT_2, TOOL_IMAGE): + assert b64(data) not in exported + # The existing attribute schema keeps one content value per message (a later tool_result in the same + # message overwrites an earlier one), so check the URIs on positions it always exports. + exported_positions = {(0, 1), (2, 0)} + for ref in span_refs: + if (ref["message_index"], ref["part_index"]) in exported_positions: + assert "fi-attachment:" + ref["occurrence_id"] in exported + # The application's request reached the provider unchanged. + sent = provider.requests[-1][1] + assert sent["messages"][2]["content"][0]["content"][0]["source"]["data"] == b64(SHOT_1) + manager.shutdown(5) + + +def test_screenshot_needs_typed_computer_tool(f01, provider, instrument): + manager = f01.manager() + instrument(manager) + request = computer_use_request() + request["tools"] = request["tools"][1:] # no computer tool declared + run(provider, request, False) + flush(manager) + purposes = sorted(u["purpose"] for u in f01.uploads) + assert "computer_screenshot" not in purposes and purposes.count("tool_result") == 3 + manager.shutdown(5) + + +def test_invalid_tool_use_id_fails_closed(f01, provider, instrument): + manager = f01.manager() + instrument(manager) + run(provider, dict(model="m", max_tokens=8, messages=[{"role": "user", "content": [ + {"type": "tool_result", "tool_use_id": "bad id!", "content": [image(SHOT_1)]}]}]), False) + flush(manager) + assert f01.payloads == [] + manager.shutdown(5) + + +def test_error_tool_result_respects_hide_error_details(f01, provider, instrument): + manager = f01.manager() + instrument(manager, config=TraceConfig(hide_error_details=True)) + run(provider, dict(model="m", max_tokens=8, messages=[{"role": "user", "content": [ + {"type": "tool_result", "tool_use_id": "toolu_9", "is_error": True, "content": [image(SHOT_1)]}, + image(IMAGE)]}]), False) + flush(manager) + assert f01.payloads == [IMAGE] + manager.shutdown(5) + + +@pytest.mark.parametrize("config", [ + TraceConfig(hide_input_images=True), TraceConfig(hide_inputs=True), TraceConfig(pii_redaction=True), + TraceConfig(hide_input_messages=True), +]) +def test_local_restrictions_deny_screenshots(f01, provider, instrument, config): + manager = f01.manager(allow_unredacted_media_with_text_pii=True) + instrument(manager, config=config) + run(provider, computer_use_request(), False) + flush(manager) + for shot in (SHOT_1, SHOT_2, TOOL_IMAGE, IMAGE): + assert shot not in f01.payloads # no model-bound screenshot escapes as tool output + manager.shutdown(5) + + +def test_disabled_manager_keeps_tracing_unchanged(provider, instrument): + manager = create_attachment_manager(config=AttachmentConfig(enabled=False)) + exporter = instrument(manager) + run(provider, computer_use_request(), False) + span = [s for s in exporter.get_finished_spans() if s.name == "Messages"][-1] + assert "fi.attachment.refs" not in span.attributes + assert "fi-attachment:" not in json.dumps(dict(span.attributes)) + + +def test_second_observation_returns_first_occurrence(f01, spans): + from traceai_anthropic import capture_anthropic_attachments, project_anthropic_media + + tracer_provider, _ = spans + manager = f01.manager() + request = computer_use_request() + with tracer_provider.get_tracer("t").start_as_current_span("llm") as span: + first = capture_anthropic_attachments(manager, span, request) + second = capture_anthropic_attachments(manager, span, request) + flush(manager) + assert first == second and len(first) == 5 and len(f01.payloads) == 5 + assert project_anthropic_media(request, {}) is request + manager.shutdown(5) diff --git a/python/tests/attachments_providers/test_auto_init.py b/python/tests/attachments_providers/test_auto_init.py new file mode 100644 index 000000000..a9ba1d425 --- /dev/null +++ b/python/tests/attachments_providers/test_auto_init.py @@ -0,0 +1,68 @@ +"""The real auto path: traceai.init(attachment_config=enabled) -> OpenAI/Anthropic capture -> loopback F01.""" + +import pytest + +traceai = pytest.importorskip("traceai") +openai = pytest.importorskip("openai") +pytest.importorskip("traceai_openai") + +from conftest import b64, flush, png # noqa: E402 +from fi_instrumentation.attachments import AttachmentConfig # noqa: E402 +from opentelemetry.sdk.trace.export import SimpleSpanProcessor # noqa: E402 +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter # noqa: E402 + + +@pytest.fixture +def auto_state(monkeypatch): + monkeypatch.setattr(traceai, "_tracer_provider", None) + monkeypatch.setattr(traceai, "_init_options", None) + monkeypatch.setattr(traceai, "_instrumented", set()) + monkeypatch.setattr(traceai, "_adapter_status", {}) + yield + for module, name in (("traceai_openai", "OpenAIInstrumentor"), ("traceai_anthropic", "AnthropicInstrumentor")): + try: + instrumentor = getattr(__import__(module), name)() + if instrumentor.is_instrumented_by_opentelemetry: + instrumentor.uninstrument() + except ImportError: + pass + + +def test_auto_init_with_enabled_attachments_captures_chat_image(f01, provider, auto_state, monkeypatch): + monkeypatch.setenv("SSL_CERT_FILE", str(f01.cert)) + monkeypatch.setenv("FI_API_KEY", "SYNTHETIC_KEY") + monkeypatch.setenv("FI_SECRET_KEY", "SYNTHETIC_SECRET") + tracer_provider = traceai.init(project_name="synthetic", project_type="observe", auto_instrument=["openai"], + verbose=False, set_global_tracer_provider=False, + attachment_config=AttachmentConfig(enabled=True, endpoint=f01.endpoint)) + exporter = InMemorySpanExporter() + tracer_provider.add_span_processor(SimpleSpanProcessor(exporter)) + manager = tracer_provider.attachment_manager + try: + assert manager.ready(5), manager.status() + image = png(seed=31) + openai.OpenAI(api_key="sk-synthetic", base_url=provider.url + "/v1", max_retries=0).chat.completions.create( + model="m", messages=[{"role": "user", "content": [ + {"type": "image_url", "image_url": {"url": "data:image/png;base64," + b64(image)}}]}]) + flush(manager) + assert f01.payloads == [image] + assert traceai.attachment_status()["runtime"] == "ready" + span = [s for s in exporter.get_finished_spans() if "fi.attachment.refs" in s.attributes] + assert span, [s.name for s in exporter.get_finished_spans()] + finally: + manager.shutdown(5) + tracer_provider.shutdown() + + +def test_auto_init_with_attachments_off_is_unchanged(provider, auto_state): + tracer_provider = traceai.init(project_name="synthetic", auto_instrument=["openai"], verbose=False, + set_global_tracer_provider=False) + try: + assert tracer_provider.attachment_manager.status().state == "disabled" + from traceai_openai import OpenAIInstrumentor + + wrapper = openai.OpenAI.request._self_wrapper + assert wrapper._attachments is None + assert OpenAIInstrumentor().is_instrumented_by_opentelemetry + finally: + tracer_provider.shutdown() diff --git a/python/tests/attachments_providers/test_exit_probe.py b/python/tests/attachments_providers/test_exit_probe.py new file mode 100644 index 000000000..e5ce8a1f7 --- /dev/null +++ b/python/tests/attachments_providers/test_exit_probe.py @@ -0,0 +1,67 @@ +"""Lifecycle: an instrumented process exits promptly with attachments enabled, disabled, and after fork.""" + +import os +import subprocess +import sys +import textwrap +import time + +import pytest + +pytest.importorskip("openai") +pytest.importorskip("traceai_openai") + +PROBE = textwrap.dedent(""" + import base64, os, ssl, sys, threading + sys.path.insert(0, os.environ["PROBE_DIR"]) + from conftest import png + import openai + from fi_instrumentation.attachments import AttachmentConfig, AttachmentManager, create_attachment_manager + from fi_instrumentation.attachments.protocol import HTTPTransport + from opentelemetry.sdk.trace import TracerProvider + from traceai_openai import OpenAIInstrumentor + + enabled = os.environ["PROBE_ENABLED"] == "1" + before = threading.active_count() + if enabled: + transport = HTTPTransport(os.environ["PROBE_F01"], {"X-API-Key": "SYNTHETIC_KEY"}, + ssl_context=ssl.create_default_context(cafile=os.environ["PROBE_CERT"])) + manager = AttachmentManager(config=AttachmentConfig(enabled=True), project_name="synthetic", + transport=transport) + assert manager.ready(5) + else: + manager = create_attachment_manager(config=AttachmentConfig(enabled=False)) + OpenAIInstrumentor().instrument(tracer_provider=TracerProvider(), attachments=manager) + client = openai.OpenAI(api_key="sk-synthetic", base_url=os.environ["PROBE_PROVIDER"] + "/v1", max_retries=0) + image = "data:image/png;base64," + base64.b64encode(png(seed=41)).decode() + client.chat.completions.create(model="m", messages=[{"role": "user", "content": [ + {"type": "image_url", "image_url": {"url": image}}]}]) + if not enabled: + assert threading.active_count() == before, "disabled attachments started a thread" + if os.environ.get("PROBE_FORK") == "1" and hasattr(os, "fork"): + pid = os.fork() + if pid == 0: + client.chat.completions.create(model="m", messages=[{"role": "user", "content": [ + {"type": "image_url", "image_url": {"url": image}}]}]) + os._exit(0) + os.waitpid(pid, 0) + print("PROBE_DONE", flush=True) + # No explicit flush or shutdown: interpreter exit must not hang. +""") + + +@pytest.mark.parametrize("enabled,fork", [(True, False), (False, False), (True, True)]) +def test_process_exits_promptly(f01, provider, tmp_path, enabled, fork): + script = tmp_path / "probe.py" + script.write_text(PROBE) + env = dict(os.environ, PROBE_DIR=os.path.dirname(__file__), PROBE_ENABLED="1" if enabled else "0", + PROBE_F01=f01.endpoint, PROBE_CERT=str(f01.cert), PROBE_PROVIDER=provider.url, + PROBE_FORK="1" if fork else "0") + started = time.monotonic() + result = subprocess.run([sys.executable, str(script)], env=env, capture_output=True, text=True, timeout=60) + elapsed = time.monotonic() - started + assert result.returncode == 0, result.stderr[-2000:] + assert "PROBE_DONE" in result.stdout + assert elapsed < 30, elapsed + if not enabled: + assert f01.state["requests"] == [] diff --git a/python/tests/attachments_providers/test_groq_audio_attachments.py b/python/tests/attachments_providers/test_groq_audio_attachments.py new file mode 100644 index 000000000..c30f06c57 --- /dev/null +++ b/python/tests/attachments_providers/test_groq_audio_attachments.py @@ -0,0 +1,105 @@ +"""Real Groq SDK -> traceai_groq audio spans -> loopback F01 (same helper as traceai_openai).""" + +import asyncio + +import pytest + +groq = pytest.importorskip("groq") +pytest.importorskip("traceai_groq") + +from conftest import flush, refs, wav # noqa: E402 +from fi_instrumentation import TraceConfig # noqa: E402 +from traceai_groq import GroqInstrumentor # noqa: E402 + +try: + from groq.resources.audio.speech import Speech # noqa: F401 + + HAS_SPEECH = True +except ImportError: # groq < 0.1x has no speech endpoint + HAS_SPEECH = False + + +@pytest.fixture +def instrument(spans): + tracer_provider, exporter = spans + state = {} + + def install(manager, config=None): + GroqInstrumentor().instrument(tracer_provider=tracer_provider, attachments=manager, + config=config or TraceConfig()) + state["on"] = True + return exporter + + yield install + if state.get("on"): + GroqInstrumentor().uninstrument() + + +def client(provider, async_=False): + cls = groq.AsyncGroq if async_ else groq.Groq + return cls(api_key="gsk-synthetic", base_url=provider.url, max_retries=0) + + +@pytest.mark.skipif(not HAS_SPEECH, reason="this groq SDK has no audio.speech") +@pytest.mark.parametrize("async_", [False, True]) +def test_speech_buffered_output(f01, provider, instrument, async_): + manager = f01.manager() + exporter = instrument(manager) + request = dict(model="playai-tts", voice="Fritz-PlayAI", input="hi", response_format="wav") + if async_: + response = asyncio.run(client(provider, True).audio.speech.create(**request)) + else: + response = client(provider).audio.speech.create(**request) + assert response.http_response.content == provider.speech # unchanged for the application + flush(manager) + assert f01.payloads == [provider.speech] + span = [s for s in exporter.get_finished_spans() if s.name == "Groq Audio Speech"][-1] + assert [r["purpose"] for r in refs(span)] == ["model_output"] + manager.shutdown(5) + + +@pytest.mark.parametrize("operation", ["transcriptions", "translations"]) +@pytest.mark.parametrize("async_", [False, True]) +def test_transcription_bytes_only(f01, provider, instrument, tmp_path, operation, async_): + manager = f01.manager() + instrument(manager) + audio = wav(seed=21) + path = tmp_path / "a.wav" + path.write_bytes(wav(seed=22)) + + if async_: + async def go(): + resource = getattr(client(provider, True).audio, operation) + await resource.create(model="whisper-large-v3", file=("a.wav", audio)) + await resource.create(model="whisper-large-v3", file=path) + asyncio.run(go()) + else: + resource = getattr(client(provider).audio, operation) + resource.create(model="whisper-large-v3", file=("a.wav", audio)) + resource.create(model="whisper-large-v3", file=path) # a path is never opened by the adapter + flush(manager) + assert f01.payloads == [audio] + manager.shutdown(5) + + +def test_privacy_controls_deny(f01, provider, instrument): + manager = f01.manager() + instrument(manager, config=TraceConfig(pii_redaction=True)) + client(provider).audio.transcriptions.create(model="whisper-large-v3", file=("a.wav", wav())) + if HAS_SPEECH: + client(provider).audio.speech.create(model="playai-tts", voice="v", input="hi") + flush(manager) + assert f01.payloads == [] + manager.shutdown(5) + + +def test_no_manager_leaves_audio_unwrapped(spans): + from groq.resources.audio.transcriptions import Transcriptions + + tracer_provider, _ = spans + original = Transcriptions.create + GroqInstrumentor().instrument(tracer_provider=tracer_provider) + try: + assert Transcriptions.create is original + finally: + GroqInstrumentor().uninstrument() diff --git a/python/tests/attachments_providers/test_media_capture.py b/python/tests/attachments_providers/test_media_capture.py new file mode 100644 index 000000000..f381b3b2f --- /dev/null +++ b/python/tests/attachments_providers/test_media_capture.py @@ -0,0 +1,114 @@ +"""The shared helper: preflight-before-read, strict decode, deny-wins folding, consumed audio.""" + +import base64 + +import pytest +from conftest import b64, flush, png, wav +from fi_instrumentation import TraceConfig +from fi_instrumentation.attachments import CapturePolicy +from fi_instrumentation.attachments.media_capture import ( + STREAM_RESERVATIONS, + ConsumedAudio, + attach_inline, + audio_mime, + local_capture_policy, + sniff_audio, + split_inline, + strict_decode, +) + + +def test_sniff_and_declared_format_cross_check(): + assert sniff_audio(wav()) == "audio/wav" + assert sniff_audio(b"OggS" + b"\x00" * 8) == "audio/ogg" + assert sniff_audio(b"fLaC" + b"\x00" * 8) == "audio/flac" + assert sniff_audio(b"ID3\x04" + b"\x00" * 8) == "audio/mpeg" + assert sniff_audio(b"\xff\xfb\x90\x00" + b"\x00" * 8) == "audio/mpeg" + assert sniff_audio(b"\xff\xf1\x50\x80" + b"\x00" * 8) is None # AAC ADTS has no F01 MIME + assert sniff_audio(b"\x00" * 12) is None + assert audio_mime(wav(), "wav") == "audio/wav" + assert audio_mime(wav(), "mp3") is None + assert audio_mime(wav(), "pcm") is None + + +def test_split_and_strict_decode_are_bounded_and_canonical(): + data = png() + assert split_inline("data:image/png;base64," + b64(data), ("image/png",), False)[0] == "image/png" + assert split_inline("data:image/gif;base64," + b64(data), ("image/png",), False) is None + assert split_inline("data:image/png;base64,abc", ("image/png",), False) is None # not a multiple of 4 + assert split_inline("x" * (400 * 1024), ("image/png",), True) is None + assert strict_decode(b64(data), len(data)) == data + assert strict_decode(b64(data), len(data) - 1) is None # the ticket budget is a hard bound + assert strict_decode("@@@@", 10) is None + non_canonical = base64.b64encode(b"ab").decode()[:-2] + "x=" # bad padding bits + assert strict_decode(non_canonical, 10) is None + + +def test_local_restrictions_fold_deny_only(): + assert local_capture_policy(None, direction="input").capture_content is True + assert local_capture_policy(None, direction="input").model_input is True + for name in ("hide_inputs", "hide_input_messages", "hide_input_text", "pii_redaction"): + assert local_capture_policy(TraceConfig(**{name: True}), direction="input").capture_content is False + assert local_capture_policy(TraceConfig(hide_input_images=True), direction="input", image=True) \ + .capture_content is False + assert local_capture_policy(TraceConfig(hide_input_images=True), direction="input").capture_content is True + assert local_capture_policy(TraceConfig(hide_outputs=True), direction="output").capture_content is False + assert local_capture_policy(TraceConfig(hide_outputs=True), direction="input").capture_content is True + # A caller base that already denies stays denied. + assert local_capture_policy(None, direction="input", base=CapturePolicy(capture_content=False)) \ + .capture_content is False + + +def test_denied_source_is_never_read(f01, spans): + tracer_provider, _ = spans + manager = f01.manager() + reads = [] + + def read(): + reads.append(1) + return "data:image/png;base64," + b64(png()) + + with tracer_provider.get_tracer("t").start_as_current_span("s") as span: + denied = attach_inline(manager, span, read, ("image/png",), raw=False, direction="input", + purpose="model_input", capture=CapturePolicy(capture_content=False)) + assert denied is None and reads == [] + admitted = attach_inline(manager, span, read, ("image/png",), raw=False, direction="input", + purpose="model_input", capture=CapturePolicy()) + assert admitted is not None and reads == [1] + flush(manager) + assert f01.payloads == [png()] + manager.shutdown(5) + + +def test_consumed_audio_steps_reservation_below_server_cap(f01, spans): + f01.state["limits"] = {"max_upload_bytes": 1024 * 1024} + tracer_provider, _ = spans + manager = f01.manager() + audio = wav(frames=4000) + with tracer_provider.get_tracer("t").start_as_current_span("s") as span: + observer = ConsumedAudio(manager, span, direction="output", purpose="model_output", + capture=CapturePolicy()) + for offset in range(0, len(audio), 5): + observer.write(audio[offset:offset + 5]) + assert observer.finish() is not None + flush(manager) + assert f01.payloads == [audio] + assert STREAM_RESERVATIONS[0] > 1024 * 1024 + manager.shutdown(5) + + +def test_consumed_audio_abort_and_unknown_container_upload_nothing(f01, spans): + tracer_provider, _ = spans + manager = f01.manager() + with tracer_provider.get_tracer("t").start_as_current_span("s") as span: + aborted = ConsumedAudio(manager, span, direction="output", purpose="model_output", capture=CapturePolicy()) + aborted.write(wav()) + aborted.abort() + assert aborted.finish() is None + unknown = ConsumedAudio(manager, span, direction="output", purpose="model_output", capture=CapturePolicy()) + unknown.write(b"\x00" * 64) + assert unknown.finish() is None and unknown.reason == "source_unavailable" + flush(manager) + assert f01.payloads == [] + assert manager.owned_bytes == 0 + manager.shutdown(5) diff --git a/python/tests/attachments_providers/test_openai_attachments.py b/python/tests/attachments_providers/test_openai_attachments.py new file mode 100644 index 000000000..1b546e83f --- /dev/null +++ b/python/tests/attachments_providers/test_openai_attachments.py @@ -0,0 +1,383 @@ +"""Real OpenAI SDK -> traceai_openai -> loopback F01: images, files, screenshots and speech.""" + +import asyncio +import json +import threading + +import pytest + +openai = pytest.importorskip("openai") +pytest.importorskip("traceai_openai") + +from conftest import PDF, b64, flush, png, refs, wait_uploads, wav # noqa: E402 +from fi_instrumentation import TraceConfig # noqa: E402 +from fi_instrumentation.attachments import AttachmentConfig, create_attachment_manager # noqa: E402 +from opentelemetry.sdk.trace import TracerProvider # noqa: E402 +from opentelemetry.sdk.trace.export import SimpleSpanProcessor # noqa: E402 +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter # noqa: E402 +from opentelemetry.sdk.trace.sampling import ALWAYS_OFF # noqa: E402 +from traceai_openai import OpenAIInstrumentor # noqa: E402 + +SDK = tuple(int(x) for x in openai.__version__.split(".")[:2]) +IMAGE = png(seed=1) +SCREENSHOT = png(seed=2) + + +@pytest.fixture +def instrument(spans): + tracer_provider, exporter = spans + state = {} + + def install(manager, config=None, **kwargs): + OpenAIInstrumentor().instrument(tracer_provider=tracer_provider, attachments=manager, + config=config or TraceConfig(), **kwargs) + state["on"] = True + return exporter + + yield install + if state.get("on"): + OpenAIInstrumentor().uninstrument() + + +def client(provider, async_=False): + cls = openai.AsyncOpenAI if async_ else openai.OpenAI + return cls(api_key="sk-synthetic", base_url=provider.url + "/v1", max_retries=0) + + +def chat_request(stream=False): + return dict(model="gpt-test", stream=stream, messages=[ + {"role": "system", "content": "be brief"}, + {"role": "user", "content": [ + {"type": "text", "text": "what is this"}, + {"type": "image_url", "image_url": {"url": "data:image/png;base64," + b64(IMAGE)}}, + {"type": "image_url", "image_url": {"url": "https://example.invalid/remote.png"}}, + {"type": "file", "file": {"filename": "a.pdf", "file_data": "data:application/pdf;base64," + b64(PDF)}}, + ]}, + ]) + + +def run_chat(provider, async_, stream): + if async_: + async def go(): + result = await client(provider, True).chat.completions.create(**chat_request(stream)) + if stream: + async for _ in result: + pass + asyncio.run(go()) + else: + result = client(provider).chat.completions.create(**chat_request(stream)) + if stream: + for _ in result: + pass + + +def llm_span(exporter, name=None): + found = [s for s in exporter.get_finished_spans() if name is None or s.name == name] + assert found, [s.name for s in exporter.get_finished_spans()] + return found[-1] + + +@pytest.mark.parametrize("async_", [False, True]) +@pytest.mark.parametrize("stream", [False, True]) +def test_chat_image_and_file_captured_and_projected(f01, provider, instrument, async_, stream): + manager = f01.manager() + exporter = instrument(manager) + run_chat(provider, async_, stream) + flush(manager) + assert sorted(f01.payloads) == sorted([IMAGE, PDF]) + uploads = {u["mime_type"]: u for u in f01.uploads} + assert set(uploads) == {"image/png", "application/pdf"} + assert uploads["image/png"]["purpose"] == "model_input" and uploads["image/png"]["direction"] == "input" + assert uploads["application/pdf"]["purpose"] == "document" + span = llm_span(exporter) + span_refs = refs(span) + assert {(r["mime_type"], r["message_index"], r["part_index"], r["role"]) for r in span_refs} == { + ("image/png", 1, 1, "user"), ("application/pdf", 1, 3, "user")} + exported = json.dumps(dict(span.attributes)) + assert b64(IMAGE) not in exported and b64(PDF) not in exported + for ref in span_refs: + assert "fi-attachment:" + ref["occurrence_id"] in exported + # The SDK request itself is unchanged: the provider still received the original media. + sent = provider.requests[-1][1] + assert sent["messages"][1]["content"][1]["image_url"]["url"] == "data:image/png;base64," + b64(IMAGE) + assert sent["messages"][1]["content"][2]["image_url"]["url"] == "https://example.invalid/remote.png" + manager.shutdown(5) + + +def responses_input(): + return [ + {"role": "user", "content": [ + {"type": "input_text", "text": "look"}, + {"type": "input_image", "image_url": "data:image/png;base64," + b64(IMAGE)}, + {"type": "input_file", "filename": "a.pdf", "file_data": "data:application/pdf;base64," + b64(PDF)}, + ]}, + {"type": "computer_call", "id": "cu_1", "call_id": "call_cu1", "status": "completed", + "action": {"type": "screenshot"}, "pending_safety_checks": []}, + {"type": "computer_call_output", "call_id": "call_cu1", + "output": {"type": "computer_screenshot", "image_url": "data:image/png;base64," + b64(SCREENSHOT)}}, + ] + + +@pytest.mark.parametrize("async_", [False, True]) +@pytest.mark.parametrize("stream", [False, True]) +def test_responses_inputs_screenshot_and_generated_image(f01, provider, instrument, async_, stream): + manager = f01.manager() + exporter = instrument(manager) + request = dict(model="computer-use-test", input=responses_input(), stream=stream, + tools=[{"type": "image_generation"}]) + + if async_: + async def go(): + result = await client(provider, True).responses.create(**request) + if stream: + async for _ in result: + pass + asyncio.run(go()) + else: + result = client(provider).responses.create(**request) + if stream: + for _ in result: + pass + flush(manager) + # Input image, input PDF, the computer screenshot and the COMPLETED generated image; never the partial. + assert sorted(f01.payloads) == sorted([IMAGE, PDF, SCREENSHOT, provider.image]) + assert png(seed=3) not in f01.payloads + by_purpose = {u["purpose"]: u for u in f01.uploads} + assert set(by_purpose) == {"model_input", "document", "computer_screenshot", "model_output"} + assert by_purpose["computer_screenshot"]["tool_call_id"] == "call_cu1" + assert by_purpose["model_output"]["direction"] == "output" + span = llm_span(exporter) + screenshot = [r for r in refs(span) if r.get("purpose") == "computer_screenshot"] + assert screenshot and screenshot[0]["role"] == "tool" and screenshot[0]["message_index"] == 2 + exported = json.dumps(dict(span.attributes)) + for data in (IMAGE, PDF, SCREENSHOT): + assert b64(data) not in exported + manager.shutdown(5) + + +@pytest.mark.parametrize("async_", [False, True]) +def test_images_generate_b64_output(f01, provider, instrument, async_): + manager = f01.manager() + instrument(manager) + if async_: + asyncio.run(client(provider, True).images.generate(model="gpt-image-1", prompt="a dot")) + else: + client(provider).images.generate(model="gpt-image-1", prompt="a dot") + flush(manager) + assert f01.payloads == [provider.image] # the url-only item is never fetched + assert f01.uploads[0]["mime_type"] == "image/png" and f01.uploads[0]["purpose"] == "model_output" + manager.shutdown(5) + + +@pytest.mark.parametrize("async_", [False, True]) +def test_speech_buffered_output(f01, provider, instrument, async_): + manager = f01.manager() + instrument(manager) + if async_: + result = asyncio.run(client(provider, True).audio.speech.create( + model="tts-1", voice="alloy", input="hi", response_format="wav")) + else: + result = client(provider).audio.speech.create(model="tts-1", voice="alloy", input="hi", + response_format="wav") + assert result.content == provider.speech # the application's result is unchanged + flush(manager) + assert f01.payloads == [provider.speech] + assert f01.uploads[0]["mime_type"] == "audio/wav" and f01.uploads[0]["direction"] == "output" + manager.shutdown(5) + + +@pytest.mark.parametrize("mode", ["stream_to_file", "iter_bytes", "read"]) +def test_speech_streamed_consumed_output(f01, provider, instrument, tmp_path, mode): + manager = f01.manager() + instrument(manager) + path = tmp_path / "out.wav" + with client(provider).audio.speech.with_streaming_response.create( + model="tts-1", voice="alloy", input="hi") as response: + if mode == "stream_to_file": + response.stream_to_file(path) + assert path.read_bytes() == provider.speech + elif mode == "iter_bytes": + assert b"".join(response.iter_bytes(100)) == provider.speech + else: + assert response.read() == provider.speech + flush(manager) + assert f01.payloads == [provider.speech] + manager.shutdown(5) + + +def test_speech_streamed_async_stream_to_file(f01, provider, instrument, tmp_path): + manager = f01.manager() + instrument(manager) + path = tmp_path / "out.wav" + + async def go(): + async with client(provider, True).audio.speech.with_streaming_response.create( + model="tts-1", voice="alloy", input="hi") as response: + await response.stream_to_file(path) + asyncio.run(go()) + assert path.read_bytes() == provider.speech + flush(manager) + assert f01.payloads == [provider.speech] + manager.shutdown(5) + + +def test_speech_partial_read_uploads_nothing(f01, provider, instrument): + manager = f01.manager() + instrument(manager) + with client(provider).audio.speech.with_streaming_response.create( + model="tts-1", voice="alloy", input="hi") as response: + iterator = response.iter_bytes(64) + next(iterator) + iterator.close() # the application stopped early + flush(manager) + assert f01.payloads == [] and f01.uploads == [] + manager.shutdown(5) + + +def test_speech_declared_format_mismatch_uploads_nothing(f01, provider, instrument): + manager = f01.manager() + instrument(manager) + client(provider).audio.speech.create(model="tts-1", voice="alloy", input="hi", response_format="mp3") + flush(manager) + assert f01.payloads == [] # WAV bytes labelled mp3 are never mislabelled + manager.shutdown(5) + + +@pytest.mark.parametrize("operation", ["transcriptions", "translations"]) +def test_transcription_input_bytes_and_paths_only(f01, provider, instrument, tmp_path, operation): + manager = f01.manager() + instrument(manager) + audio = wav(seed=4) + resource = getattr(client(provider).audio, operation) + resource.create(model="whisper-1", file=("a.wav", audio, "audio/wav")) + path = tmp_path / "b.wav" + path.write_bytes(wav(seed=5)) + resource.create(model="whisper-1", file=path) # the SDK itself reads a path into bytes + with open(path, "rb") as handle: + resource.create(model="whisper-1", file=handle) # an open file is the app's stream: not read + flush(manager) + assert sorted(f01.payloads) == sorted([audio, wav(seed=5)]) # two workers: completion order varies + assert all(u["direction"] == "input" and u["purpose"] == "model_input" for u in f01.uploads) + manager.shutdown(5) + + +# ------------------------------------------------------------ privacy controls +@pytest.mark.parametrize("config", [ + TraceConfig(hide_input_images=True), TraceConfig(hide_inputs=True), TraceConfig(pii_redaction=True), + TraceConfig(hide_input_messages=True), TraceConfig(hide_input_text=True), +]) +def test_local_hide_and_pii_deny_input_media_including_screenshots(f01, provider, instrument, config): + # Even a manager that would relax text PII for media cannot relax a local restriction. + manager = f01.manager(allow_unredacted_media_with_text_pii=True) + instrument(manager, config=config) + client(provider).responses.create(model="m", input=responses_input()) + client(provider).chat.completions.create(**chat_request()) + flush(manager) + payloads = f01.payloads + assert SCREENSHOT not in payloads and IMAGE not in payloads + if not config.hide_input_images: + assert PDF not in payloads + manager.shutdown(5) + + +def test_hide_outputs_denies_generated_media_and_speech(f01, provider, instrument): + manager = f01.manager() + instrument(manager, config=TraceConfig(hide_outputs=True)) + client(provider).images.generate(model="gpt-image-1", prompt="x") + client(provider).audio.speech.create(model="tts-1", voice="alloy", input="hi") + flush(manager) + assert f01.payloads == [] + manager.shutdown(5) + + +def test_server_policy_audio_only_denies_images(f01, provider, instrument): + f01.state["policy"] = {"allowed_modalities": ["audio"]} + manager = f01.manager() + instrument(manager) + client(provider).chat.completions.create(**chat_request()) + flush(manager) + assert f01.payloads == [] and f01.after_bootstrap() == [] + manager.shutdown(5) + + +def test_unsampled_span_uploads_nothing(f01, provider): + manager = f01.manager() + exporter = InMemorySpanExporter() + tracer_provider = TracerProvider(sampler=ALWAYS_OFF) + tracer_provider.add_span_processor(SimpleSpanProcessor(exporter)) + OpenAIInstrumentor().instrument(tracer_provider=tracer_provider, attachments=manager) + try: + client(provider).chat.completions.create(**chat_request()) + finally: + OpenAIInstrumentor().uninstrument() + flush(manager) + assert f01.payloads == [] + manager.shutdown(5) + + +def test_disabled_manager_is_a_strict_no_op(provider, spans): + tracer_provider, exporter = spans + manager = create_attachment_manager(config=AttachmentConfig(enabled=False)) + before = {t.ident for t in threading.enumerate() if t.name.startswith("fi-attachment")} + OpenAIInstrumentor().instrument(tracer_provider=tracer_provider, attachments=manager) + try: + client(provider).chat.completions.create(**chat_request()) + wrapper = openai.OpenAI.request + assert getattr(wrapper, "_self_wrapper", None) is None or \ + getattr(wrapper._self_wrapper, "_attachments", None) is None + finally: + OpenAIInstrumentor().uninstrument() + # No attachment thread was started (earlier tests' managers may still be winding down theirs). + assert {t.ident for t in threading.enumerate() if t.name.startswith("fi-attachment")} <= before + span = llm_span(exporter) + assert "fi.attachment.refs" not in span.attributes + assert "fi-attachment:" not in json.dumps(dict(span.attributes)) + + +def test_env_disable_wins_over_enabled_config(f01, provider, instrument, monkeypatch): + monkeypatch.setenv("FI_ATTACHMENTS_ENABLED", "false") + manager = create_attachment_manager(config=AttachmentConfig(enabled=True), project_name="synthetic", + transport=f01.transport()) + instrument(manager) + client(provider).chat.completions.create(**chat_request()) + assert f01.state["requests"] == [] + + +def test_second_observation_returns_first_occurrence(f01, spans): + from traceai_openai import capture_openai_attachments + + tracer_provider, _ = spans + manager = f01.manager() + tracer = tracer_provider.get_tracer("t") + payload = chat_request() + with tracer.start_as_current_span("llm") as span: + first = capture_openai_attachments(manager, span, payload, operation="chat", direction="input") + second = capture_openai_attachments(manager, span, payload, operation="chat", direction="input") + flush(manager) + assert first == second and len(first) == 2 + assert len(f01.payloads) == 2 + manager.shutdown(5) + + +def test_projection_returns_same_object_without_capture(): + from traceai_openai import project_openai_media + + payload = chat_request() + assert project_openai_media(payload, {}, "chat", "input") is payload + + +def test_malformed_parts_fail_closed_without_upload(f01, provider, instrument): + manager = f01.manager() + instrument(manager) + bad = [ + {"type": "image_url", "image_url": {"url": "data:image/png;base64,@@@@"}}, + {"type": "image_url", "image_url": {"url": "data:image/gif;base64," + b64(IMAGE)}}, + {"type": "image_url", "image_url": {"url": "data:image/png;base64," + b64(IMAGE)[:-1]}}, + {"type": "input_file", "file_id": "file-abc"}, + {"type": "image_url", "image_url": {"url": "data:image/png;base64," + b64(b"\x00" * (300 * 1024))}}, + ] + client(provider).chat.completions.create(model="m", messages=[{"role": "user", "content": bad}]) + flush(manager) + assert f01.payloads == [] + assert manager.status().open_tickets == 0 and manager.owned_bytes == 0 + manager.shutdown(5)