From e1586827fc5421affe30c6f13bcd66dbc6a37c9a Mon Sep 17 00:00:00 2001 From: Nikhil Pareek <8347993+nik13@users.noreply.github.com> Date: Wed, 11 Nov 2026 19:04:56 -0800 Subject: [PATCH 01/15] feat(python): share the per-call capture-policy and owner context keys Byte-identical to the media_capture.py change on feat/attachments-python-auto (1b3072bb + 0b2eb4b5), so merging that branch later is a no-op here. Provider adapters read CAPTURE_POLICY_KEY / OWNER_KEY through local_capture_policy and Observation. Co-Authored-By: Claude Opus 5.5 --- .../attachments/media_capture.py | 53 ++++++++++++++++++- 1 file changed, 52 insertions(+), 1 deletion(-) diff --git a/python/fi_instrumentation/attachments/media_capture.py b/python/fi_instrumentation/attachments/media_capture.py index a2fd637c4..1673fd5f9 100644 --- a/python/fi_instrumentation/attachments/media_capture.py +++ b/python/fi_instrumentation/attachments/media_capture.py @@ -13,6 +13,11 @@ - One registry per manager remembers admitted (span, position) pairs, so a second observer of the same span and position gets the FIRST occurrence back and uploads nothing. +- A per-call ``CapturePolicy`` set in the OpenTelemetry context under + ``CAPTURE_POLICY_KEY`` (``traceai.with_attachment_capture_policy``) can only + add restrictions. A context marked under ``OWNER_KEY`` + (``traceai.with_attachment_ownership``) belongs to an adapter that captures + the media itself, so nothing is captured here for that call. Every public function is non-throwing and returns ``None``/empty on denial. """ @@ -26,6 +31,8 @@ from collections import OrderedDict from typing import Any, Callable, Iterable, Mapping, NamedTuple, Optional, Tuple +from opentelemetry import context as otel_context + from .policy import CapturePolicy from .stream import StreamOptions from .types import Bytes, CorrelationHandle @@ -50,6 +57,13 @@ # mulaw have no F01 MIME, so they are never captured. SPEECH_FORMATS = {"mp3": "audio/mpeg", "opus": "audio/ogg", "ogg": "audio/ogg", "flac": "audio/flac", "wav": "audio/wav"} +# Context keys shared BY NAME with the auto module (``traceai``) and any adapter, like the +# TypeScript ``Symbol.for`` keys: plain strings, so no package has to import another. +CAPTURE_POLICY_KEY = "traceai.auto.attachment_policy.v1" +OWNER_KEY = "traceai.auto.attachment_owner.v1" +# CapturePolicy facts that grant capture; every other fact only restricts. +_PERMISSIONS = frozenset(("capture_content", "record_inputs", "record_outputs", "requested")) +_DENY_ALL = CapturePolicy(capture_content=False, requested=False) class Occurrence(NamedTuple): @@ -93,6 +107,40 @@ def _restricted(config, name: str) -> bool: return True +def merge_capture_policies(*policies: Any) -> CapturePolicy: + """Deny-wins intersection: a grant needs every policy, a restriction needs any one. + + ``None`` adds nothing. Any other value that is not a ``CapturePolicy`` denies. + """ + present = [policy for policy in policies if policy is not None] + if any(not isinstance(policy, CapturePolicy) for policy in present): + return _DENY_ALL + if not present: + return CapturePolicy() + values = {} + for item in dataclasses.fields(CapturePolicy): + facts = [getattr(policy, item.name) for policy in present] + values[item.name] = all(facts) if item.name in _PERMISSIONS else any(facts) + return CapturePolicy(**values) + + +def context_capture_policy(context=None) -> Any: + """The per-call policy in the (current) OpenTelemetry context; None when unset.""" + try: + return otel_context.get_value(CAPTURE_POLICY_KEY, context) + except Exception: + # An unreadable restriction is a restriction, never permission. + return _DENY_ALL + + +def owned_elsewhere(context=None) -> bool: + """True when another adapter marked this call's media as its own.""" + try: + return otel_context.get_value(OWNER_KEY, context) is not None + except Exception: + return True + + def local_capture_policy(trace_config=None, *, direction: str, image: bool = False, error_detail: bool = False, base: Optional[CapturePolicy] = None) -> CapturePolicy: """Fold the instrumentor's local TraceConfig into a deny-only CapturePolicy. @@ -103,6 +151,7 @@ def local_capture_policy(trace_config=None, *, direction: str, image: bool = Fal ancestry, so a tool screenshot is still treated as model input. """ base = base if isinstance(base, CapturePolicy) else CapturePolicy() + base = merge_capture_policies(base, context_capture_policy()) names = _INPUT_HIDE if direction == "input" else _OUTPUT_HIDE hidden = any(_restricted(trace_config, name) for name in names) or _restricted(trace_config, "pii_redaction") if image and direction == "input" and _restricted(trace_config, "hide_input_images"): @@ -116,6 +165,8 @@ def local_capture_policy(trace_config=None, *, direction: str, image: bool = Fal changes["model_input"] = True if error_detail: changes["error_detail"] = True + if owned_elsewhere(): + changes["requested"] = False return dataclasses.replace(base, **changes) @@ -262,7 +313,7 @@ class Observation: def __init__(self, manager, span, namespace: str): self.manager, self.span, self.namespace = manager, span, namespace - self.key = span_key(span) if active(manager) else None + self.key = span_key(span) if active(manager) and not owned_elsewhere() else None self.captured: "OrderedDict[str, Occurrence]" = OrderedDict() self._visited = 0 From 70ca22536ae0a506f510e46eaed6567cd99bfc56 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek <8347993+nik13@users.noreply.github.com> Date: Wed, 11 Nov 2026 19:19:53 -0800 Subject: [PATCH 02/15] feat(python): typed provider media for Gemini, Vertex, Bedrock, LiteLLM and compatible chat New fi_instrumentation.attachments.provider_media (mirrors the TypeScript provider-media helper): preflight before any read, exact-type re-admission, magic-byte check after admission, deny-wins policy through media_capture, and a per-package capability table that is the Python support matrix. Bindings: google-genai (generate_content in/out, stream input), vertexai (GenerateContent in/out, Imagen predictions), bedrock (Converse incl. tool screenshots, InvokeModel Anthropic/Nova/Titan/Nova Canvas), litellm (completion input, image_generation output, owner claim around the downstream SDK call), groq/together/fireworks/xai/mistralai/openrouter/ portkey/ollama chat media. deepseek and cerebras export an empty descriptor (no qualified media surface). Co-Authored-By: Claude Opus 5.5 --- .../attachments/provider_media.py | 692 ++++++++++++++++++ .../bedrock/traceai_bedrock/__init__.py | 71 +- .../cerebras/traceai_cerebras/__init__.py | 9 + .../deepseek/traceai_deepseek/__init__.py | 9 + .../fireworks/traceai_fireworks/__init__.py | 29 +- .../fireworks/traceai_fireworks/_wrappers.py | 21 +- .../traceai_google_genai/__init__.py | 33 +- .../traceai_google_genai/_wrappers.py | 37 +- .../frameworks/groq/traceai_groq/__init__.py | 8 +- python/frameworks/groq/traceai_groq/_audio.py | 8 +- .../frameworks/groq/traceai_groq/_wrappers.py | 29 +- .../litellm/traceai_litellm/__init__.py | 133 +++- .../mistralai/traceai_mistralai/__init__.py | 36 +- .../traceai_mistralai/_chat_wrapper.py | 28 +- .../mistralai/traceai_mistralai/_native.py | 56 +- .../ollama/traceai_ollama/__init__.py | 36 +- .../ollama/traceai_ollama/_wrappers.py | 23 +- .../openrouter/traceai_openrouter/__init__.py | 45 +- .../portkey/traceai_portkey/__init__.py | 29 +- .../portkey/traceai_portkey/_wrappers.py | 28 +- .../together/traceai_together/__init__.py | 28 +- .../together/traceai_together/_wrappers.py | 29 +- .../vertexai/traceai_vertexai/__init__.py | 26 +- .../vertexai/traceai_vertexai/_wrapper.py | 51 +- python/frameworks/xai/traceai_xai/__init__.py | 29 +- .../frameworks/xai/traceai_xai/_wrappers.py | 29 +- 26 files changed, 1476 insertions(+), 76 deletions(-) create mode 100644 python/fi_instrumentation/attachments/provider_media.py diff --git a/python/fi_instrumentation/attachments/provider_media.py b/python/fi_instrumentation/attachments/provider_media.py new file mode 100644 index 000000000..a07208456 --- /dev/null +++ b/python/fi_instrumentation/attachments/provider_media.py @@ -0,0 +1,692 @@ +"""Typed provider media extraction shared by the Python provider instrumentors. + +Mirrors ``@traceai/provider-media`` (TypeScript): one helper, thin per-package +bindings, and a per-package capability table that is also the support matrix. + +Rules (same as ``media_capture``, which this builds on): + +- Typed shapes only: Gemini ``inline_data`` Parts, OpenAI-compatible + ``image_url`` data URIs and ``input_audio``, Bedrock Converse/Nova + ``image``/``document``/``video`` byte blocks, Anthropic-on-Bedrock base64 + blocks, Titan/Nova Canvas image fields, Ollama ``images`` and generated-image + ``b64_json``. No recursive walk of unknown bodies, no URL fetch, no provider + file or ``gs://``/S3 lookup, no filesystem access. +- ``manager.preflight`` runs BEFORE a source is read, sized beyond its shape, + decoded or sniffed. A preflight denied only for size is retried once at the + exact size; a source whose exact type differs from the gate is re-admitted + under that type. Unused tickets are released at once. +- After admission the leading bytes must match the declared type (a PNG + declared as ``audio/wav`` is not uploaded): a hide flag that applies to the + real modality can never be bypassed through a wrong label. +- Local ``TraceConfig`` hide/PII flags, the per-call context policy + (``media_capture.CAPTURE_POLICY_KEY``) and the owner mark + (``media_capture.OWNER_KEY``) apply through ``media_capture`` (deny wins). +- Every public function is non-throwing and returns an empty mapping on denial. + +Not in ``attachments.__all__``: an adapter-facing module, like ``media_capture``. +""" + +import re +from collections.abc import Sequence +from types import MappingProxyType +from typing import Any, Callable, Mapping, Optional, Tuple + +from opentelemetry import context as otel_context + +from . import media_capture as mc +from .types import Bytes + +KiB = 1024 +MiB = 1024 * KiB +#: Bound on synchronous base64 decoding, independent of the manager queue (as in TypeScript). +#: Raw bytes the SDK already holds are bounded by the manager alone (``too_large``/``queue_full``). +MAX_DECODED_BYTES = 1 * MiB +MAX_PARTS = 64 +DATA_URI_TYPES = ("image/png", "image/jpeg", "image/webp") +IMAGE_TYPES = DATA_URI_TYPES +DOCUMENT_TYPES = ("application/pdf",) +AUDIO_TYPES = ("audio/wav", "audio/mpeg", "audio/ogg", "audio/flac", "audio/webm") +VIDEO_TYPES = ("video/mp4", "video/webm") +# Gemini mime_type spellings that name an F01 type. Anything else (audio/L16 PCM, text/*, +# image/heic, ...) is not captured. +GEMINI_TYPES = { + **{mime: mime for mime in IMAGE_TYPES + DOCUMENT_TYPES + AUDIO_TYPES + VIDEO_TYPES}, + "audio/mp3": "audio/mpeg", "audio/x-wav": "audio/wav", "audio/wave": "audio/wav", + "audio/x-flac": "audio/flac", "image/jpg": "image/jpeg", +} +_COMPAT_AUDIO = {"wav": "audio/wav", "mp3": "audio/mpeg"} +_BEDROCK_IMAGE = {"png": "image/png", "jpeg": "image/jpeg", "jpg": "image/jpeg", "webp": "image/webp"} +_BEDROCK_VIDEO = {"mp4": "video/mp4", "webm": "video/webm"} +# Converse / Nova model families whose image and document blocks are qualified. Unknown model +# bodies are never scraped; a model-family name alone is not coverage. +_CONVERSE_MODELS = re.compile( + r"(?:arn:aws[\w-]*:bedrock:[\w-]+:\d*:(?:inference-profile|foundation-model)/)?" + r"(?:(?:us|eu|apac|global)\.)?(?:anthropic\.claude-(?:3|sonnet-4|opus-4|haiku-4)|amazon\.nova-(?:lite|pro|premier))", + re.IGNORECASE) +_NOVA_VIDEO_MODELS = re.compile( + r"(?:arn:aws[\w-]*:bedrock:[\w-]+:\d*:(?:inference-profile|foundation-model)/)?" + r"(?:(?:us|eu|apac|global)\.)?amazon\.nova-(?:lite|pro|premier)", re.IGNORECASE) +_ANTHROPIC_MODELS = re.compile( + r"(?:arn:aws[\w-]*:bedrock:[\w-]+:\d*:(?:inference-profile|foundation-model)/)?" + r"(?:(?:us|eu|apac|global)\.)?anthropic\.claude-", re.IGNORECASE) +_IMAGE_GENERATION_MODELS = re.compile( + r"(?:arn:aws[\w-]*:bedrock:[\w-]+:\d*:(?:inference-profile|foundation-model)/)?" + r"(?:(?:us|eu|apac|global)\.)?amazon\.(?:titan-image-generator-v[12]|nova-canvas-v1)", re.IGNORECASE) +_COMPUTER_TOOL = re.compile(r"computer_\d{8}") +# Vertex publisher image-generation models (Imagen). Other prediction endpoints are never read. +_IMAGEN_ENDPOINT = re.compile(r"/models/(?:imagen-|imagegeneration)") +# Titan Image Generator / Nova Canvas request fields that carry one base64 image each. +_IMAGE_PARAMS = ( + ("inPaintingParams", "image"), ("outPaintingParams", "image"), + ("colorGuidedGenerationParams", "referenceImage"), ("backgroundRemovalParams", "image"), + ("textToImageParams", "conditionImage"), +) + +EMPTY: Mapping[str, mc.Occurrence] = mc.EMPTY + + +# ------------------------------------------------------------------ manager and owner +def attachment_manager(manager: Any) -> Any: + """``manager`` when it can admit, else None. Disabled, missing or foreign objects are inert.""" + return manager if mc.active(manager) else None + + +def manager_from_kwargs(kwargs: Mapping[str, Any]) -> Any: + """The shared manager passed as ``attachments`` (or auto's ``attachment_manager``), if it can admit.""" + try: + manager = kwargs.get("attachments") + if manager is None: + manager = kwargs.get("attachment_manager") + return attachment_manager(manager) + except Exception: + return None + + +def claim(context=None): + """The context with this call's media marked as owned (``media_capture.OWNER_KEY``). + + An adapter that captures a call's media itself runs the downstream SDK call in this + context, so nested provider instrumentors (for example OpenAI under LiteLLM) skip it. + """ + return otel_context.set_value(mc.OWNER_KEY, True, context) + + +# ------------------------------------------------------------------ type checks +def sniff(head: Any) -> Optional[str]: + """The F01 type named by leading magic bytes the caller already holds; None when unknown.""" + if not isinstance(head, (bytes, bytearray, memoryview)): + return None + head = bytes(head[:16]) + if head[:8] == b"\x89PNG\r\n\x1a\n": + return "image/png" + if head[:3] == b"\xff\xd8\xff": + return "image/jpeg" + if len(head) >= 12 and head[:4] == b"RIFF" and head[8:12] == b"WEBP": + return "image/webp" + if head[:5] == b"%PDF-": + return "application/pdf" + if len(head) >= 12 and head[4:8] == b"ftyp": + return "video/mp4" + return mc.sniff_audio(head) + + +def consistent(mime: str, data: Any) -> bool: + """True when the leading bytes are the declared type (EBML may be audio or video WebM).""" + found = sniff(data) + if found is None: + return False + if found == "audio/webm": + return mime in ("audio/webm", "video/webm") + return found == mime + + +def _modality(mime: str) -> str: + return mime.split("/", 1)[0] + + +# ------------------------------------------------------------------ sources +class _Source: + """A source described from its shape: exact type, exact size, and a read that runs after admission.""" + + __slots__ = ("mime", "size", "take") + + def __init__(self, mime: str, size: int, take: Callable[[int], Optional[bytes]]): + self.mime, self.size, self.take = mime, size, take + + +def _bytes_source(data: Any, mime: Optional[str]) -> Optional[_Source]: + if not isinstance(data, (bytes, bytearray, memoryview)): + return None + size = data.nbytes if isinstance(data, memoryview) else len(data) + if size <= 0: + return None + if mime is None: + mime = sniff(data[:16]) + if mime is None: + return None + + def take(max_bytes: int) -> Optional[bytes]: + return bytes(data) if size <= max_bytes else None + + return _Source(mime, size, take) + + +def _base64_source(encoded: Any, mime: Optional[str]) -> Optional[_Source]: + size = mc.encoded_length(encoded, MAX_DECODED_BYTES) + if not size: + return None + decoded = {} + + def decode(max_bytes: int) -> Optional[bytes]: + if "data" not in decoded: + decoded["data"] = mc.strict_decode(encoded, max_bytes, MAX_DECODED_BYTES) + return decoded["data"] + + if mime is None: + # Untyped base64 (Ollama, Titan): the exact type comes from the decoded magic bytes. + # Only described after an exact-size image gate was admitted, so this is post-admission. + data = decode(MAX_DECODED_BYTES) + mime = sniff(data[:16]) if data else None + if mime is None: + return None + return _Source(mime, size, decode) + + +def _data_uri_source(value: Any) -> Optional[_Source]: + if not isinstance(value, str): + return None + for mime in DATA_URI_TYPES: + prefix = "data:%s;base64," % mime + if value.startswith(prefix): + return _base64_source(value[len(prefix):], mime) + return None + + +# ------------------------------------------------------------------ capture +class ProviderMedia: + """One adapter observation of one span. Typed extraction only; never raises. + + ``span`` is the real OpenTelemetry span the instrumentor emitted (unwrap any + ``_WithSpan`` helper first). ``namespace`` keeps positions of different + adapters on the same span apart. + """ + + def __init__(self, manager, span, namespace: str, *, trace_config=None): + self.manager, self.span, self.trace_config = attachment_manager(manager), span, trace_config + self.observation = mc.Observation(self.manager, span, namespace) if self.manager is not None else None + self._attempted = 0 + + @property + def usable(self) -> bool: + return self.observation is not None and self.observation.usable + + @property + def captured(self) -> Mapping[str, mc.Occurrence]: + if not self.usable or not self.observation.captured: + return EMPTY + return MappingProxyType(dict(self.observation.captured)) + + # -------------------------------------------------------------- admission + def _admit(self, position: str, gate_mime: str, gate_bytes: int, describe: Callable[[], Optional[_Source]], *, + direction: str, purpose: str, role: Optional[str] = None, message_index: Optional[int] = None, + part_index: Optional[int] = None, tool_call_id: Optional[str] = None, + error_detail: bool = False, exact: bool = True) -> None: + """Gate, describe, (re-)admit at the exact type and size, read, check the magic bytes, attach. + + ``exact`` means ``gate_bytes`` is already the source's exact size (bytes length, or the + decoded length from the base64 shape), so a size denial is final and nothing is described. + """ + if not self.usable or self._attempted >= MAX_PARTS or gate_bytes <= 0: + return + self._attempted += 1 + manager, span = self.manager, self.span + + def policy(mime): + return mc.local_capture_policy(self.trace_config, direction=direction, image=_modality(mime) == "image", + error_detail=error_detail) + + def capture() -> Optional[mc.Occurrence]: + taken = [] + attached = False + + def preflight(mime, size): + ticket = manager.preflight(span, mime_type=mime, direction=direction, max_bytes=size, + purpose=purpose, capture_policy=policy(mime)) + taken.append(ticket) + return ticket + + try: + ticket = preflight(gate_mime, gate_bytes) + # too_large / queue_full come after every privacy, correlation and sampling step, + # so describing the source to size it exactly is still safe. + if not ticket.admitted and (exact or ticket.reason not in ("too_large", "queue_full")): + return None + source = describe() + if source is None or source.size <= 0: + return None + if not ticket.admitted or source.mime != gate_mime or source.size > ticket.max_bytes: + ticket.release() + ticket = preflight(source.mime, source.size) + if not ticket.admitted: + return None + data = source.take(ticket.max_bytes) + if data is None or not consistent(source.mime, data): + return None + attached = True + handle = ticket.attach(Bytes(data), **mc._fields(message_index, part_index, role, tool_call_id)) + return mc.Occurrence(handle.occurrence_id, source.mime) if handle.admission == "queued" else None + except Exception: + return None + finally: + if not attached: + for ticket in taken: + try: + ticket.release() + except Exception: + pass + + try: + self.observation.visit(position, capture) + except Exception: + pass + + def _typed(self, position: str, data: Any, mime: str, **fields) -> None: + """Bytes or raw base64 of a declared type.""" + if isinstance(data, (bytes, bytearray, memoryview)): + size = data.nbytes if isinstance(data, memoryview) else len(data) + self._admit(position, mime, size, lambda: _bytes_source(data, mime), **fields) + elif isinstance(data, str): + size = mc.encoded_length(data, MAX_DECODED_BYTES) + if size: + self._admit(position, mime, size, lambda: _base64_source(data, mime), **fields) + + def _untyped_image(self, position: str, data: Any, **fields) -> None: + """Bytes or raw base64 with no declared type: one image gate, then the magic bytes decide.""" + if isinstance(data, (bytes, bytearray, memoryview)): + size = data.nbytes if isinstance(data, memoryview) else len(data) + self._admit(position, "image/png", size, lambda: _image_only(_bytes_source(data, None)), **fields) + elif isinstance(data, str): + size = mc.encoded_length(data, MAX_DECODED_BYTES) + if size: + self._admit(position, "image/png", size, lambda: _image_only(_base64_source(data, None)), **fields) + + def _data_uri(self, position: str, read: Callable[[], Any], **fields) -> None: + # Every candidate type is an image, so one image gate decides the part before the + # string is parsed. A remote or signed URL is never kept as fallback metadata. + self._admit(position, DATA_URI_TYPES[0], MAX_DECODED_BYTES, lambda: _data_uri_source(read()), exact=False, + **fields) + + # -------------------------------------------------------------- Gemini + def gemini_contents(self, contents: Any, direction: str = "input") -> "ProviderMedia": + """Gemini ``Content``/``Part`` values: ``inline_data`` only (``file_data`` URIs are never resolved).""" + if not self.usable: + return self + try: + values = _items(contents) if isinstance(contents, Sequence) and not isinstance(contents, str) else [contents] + for m, content in enumerate(values): + parts = mc.get(content, "parts") + if parts is None and _inline(content) is not None: + parts = [content] # A bare Part (or list of Parts) is one single-part message. + role = mc.get(content, "role") + role = "assistant" if role == "model" or direction == "output" else "user" + for p, part in enumerate(_items(parts)[:MAX_PARTS]): + inline = _inline(part) + if inline is None: + continue + mime = GEMINI_TYPES.get(_lower(mc.get(inline, "mime_type") or mc.get(inline, "mimeType"))) + if mime is None: + continue + self._typed("gemini/%s/%d/%d" % (direction, m, p), mc.get(inline, "data"), mime, + direction=direction, purpose=_purpose(direction, mime), role=role, + message_index=m, part_index=p) + except Exception: + pass + return self + + def gemini_response(self, response: Any) -> "ProviderMedia": + """Completed ``GenerateContentResponse``: candidate ``inline_data`` (generated images, audio, files).""" + if not self.usable: + return self + try: + candidates = _items(mc.get(response, "candidates"))[:MAX_PARTS] + self.gemini_contents([mc.get(candidate, "content") for candidate in candidates], "output") + except Exception: + pass + return self + + # -------------------------------------------------------------- OpenAI-compatible chat + def compatible_messages(self, messages: Any, dialect: str = "openai") -> "ProviderMedia": + """Chat ``image_url`` data URIs, plus ``input_audio`` (wav/mp3) for OpenAI-dialect routes. + + ``groq``, ``mistral`` and ``image`` dialects are image only; ``mistral`` also accepts + ``image_url`` as a plain string. + """ + if not self.usable: + return self + try: + for m, message in enumerate(_items(messages)[:MAX_PARTS]): + role = _role(mc.get(message, "role")) + for p, part in enumerate(_items(mc.get(message, "content"))[:MAX_PARTS]): + kind = mc.get(part, "type") + fields = dict(direction="input", purpose="model_input", role=role, message_index=m, part_index=p) + if kind == "image_url": + def read(part=part): + image = mc.get(part, "image_url") + if image is None and dialect == "mistral": + image = mc.get(part, "imageUrl") + return image if isinstance(image, str) else mc.get(image, "url") + + self._data_uri("chat/%d/%d" % (m, p), read, **fields) + elif kind == "input_audio" and dialect == "openai": + audio = mc.get(part, "input_audio") + mime = _COMPAT_AUDIO.get(mc.get(audio, "format")) + if mime is not None: + self._typed("chat/%d/%d" % (m, p), mc.get(audio, "data"), mime, **fields) + except Exception: + pass + return self + + def ollama_images(self, images: Any, *, message_index: Optional[int] = None, role: str = "user") -> "ProviderMedia": + """Ollama ``images``: bytes or base64 only. Path strings and ``Path`` objects are never opened.""" + if not self.usable: + return self + try: + for p, image in enumerate(_items(images)[:MAX_PARTS]): + value = mc.get(image, "value") if not isinstance(image, (str, bytes, bytearray)) else image + position = "ollama/%s/%d" % ("-" if message_index is None else message_index, p) + self._untyped_image(position, value, direction="input", purpose="model_input", role=role, + message_index=message_index, part_index=p) + except Exception: + pass + return self + + def ollama_messages(self, messages: Any) -> "ProviderMedia": + if not self.usable: + return self + try: + for m, message in enumerate(_items(messages)[:MAX_PARTS]): + self.ollama_images(mc.get(message, "images"), message_index=m, role=_role(mc.get(message, "role"))) + except Exception: + pass + return self + + def generated_images(self, data: Any, *, field: str = "b64_json") -> "ProviderMedia": + """Generated-image results that carry base64 (``b64_json``); URL results are never fetched.""" + if not self.usable: + return self + try: + for index, item in enumerate(_items(data)[:MAX_PARTS]): + self._untyped_image("generated/%d" % index, mc.get(item, field), direction="output", + purpose="model_output", role="assistant", part_index=index) + except Exception: + pass + return self + + # -------------------------------------------------------------- Bedrock + def bedrock_converse(self, model_id: Any, messages: Any, additional_fields: Any = None) -> "ProviderMedia": + """Converse (and Nova InvokeModel) ``image``/``document``/``video`` blocks of qualified model families.""" + if not self.usable or not isinstance(model_id, str) or not _CONVERSE_MODELS.match(model_id): + return self + try: + video = bool(_NOVA_VIDEO_MODELS.match(model_id)) + messages = _items(messages)[:MAX_PARTS] + computer = _computer_calls(messages, mc.get(additional_fields, "tools"), "toolUse", "toolUseId", + nested="toolUse") + for m, message in enumerate(messages): + role = _role(mc.get(message, "role")) + for p, block in enumerate(_items(mc.get(message, "content"))[:MAX_PARTS]): + self._bedrock_block("bedrock/%d/%d" % (m, p), block, video, role=role, message_index=m, + part_index=p) + result = mc.get(block, "toolResult") + if result is None: + continue + tool_id = mc.get(result, "toolUseId") + if not isinstance(tool_id, str) or not mc.TOOL_ID.fullmatch(tool_id): + continue # Invalid tool IDs fail closed instead of losing causal identity. + for q, child in enumerate(_items(mc.get(result, "content"))[:MAX_PARTS]): + self._bedrock_block("bedrock/%d/%d/%d" % (m, p, q), child, video, role="tool", + message_index=m, part_index=p, tool_call_id=tool_id, + screenshot=tool_id in computer, + error_detail=mc.get(result, "status") == "error") + except Exception: + pass + return self + + def _bedrock_block(self, position: str, block: Any, video: bool, *, tool_call_id: Optional[str] = None, + screenshot: bool = False, error_detail: bool = False, **fields) -> None: + for kind in ("image", "document", "video"): + media = mc.get(block, kind) + if media is None: + continue + fmt = _lower(mc.get(media, "format")) + if kind == "image": + mime = _BEDROCK_IMAGE.get(fmt) + elif kind == "document": + mime = "application/pdf" if fmt == "pdf" else None + else: + mime = _BEDROCK_VIDEO.get(fmt) if video else None + if mime is None: + return + # Only inline bytes (raw for Converse, base64 for Nova InvokeModel); S3 locations stay unsupported. + data = mc.get(mc.get(media, "source"), "bytes") + if tool_call_id is not None: + purpose = "computer_screenshot" if screenshot and kind == "image" else "tool_result" + else: + purpose = "document" if kind == "document" else "model_input" + self._typed(position, data, mime, direction="input", purpose=purpose, tool_call_id=tool_call_id, + error_detail=error_detail, **fields) + return + + def anthropic_messages(self, messages: Any, tools: Any = None) -> "ProviderMedia": + """Anthropic Messages bodies (Bedrock InvokeModel): base64 image/PDF blocks and one tool_result level.""" + if not self.usable: + return self + try: + messages = _items(messages)[:MAX_PARTS] + computer = _computer_calls(messages, tools, "tool_use", "id") + for m, message in enumerate(messages): + role = _role(mc.get(message, "role")) + for p, block in enumerate(_items(mc.get(message, "content"))[:MAX_PARTS]): + if mc.get(block, "type") == "tool_result": + tool_id = mc.get(block, "tool_use_id") + if not isinstance(tool_id, str) or not mc.TOOL_ID.fullmatch(tool_id): + continue + for q, child in enumerate(_items(mc.get(block, "content"))[:MAX_PARTS]): + self._anthropic_block("anthropic/%d/%d/%d" % (m, p, q), child, role="tool", + message_index=m, part_index=q, tool_call_id=tool_id, + screenshot=tool_id in computer, + error_detail=mc.get(block, "is_error") is True) + else: + self._anthropic_block("anthropic/%d/%d" % (m, p), block, role=role, message_index=m, + part_index=p) + except Exception: + pass + return self + + def _anthropic_block(self, position: str, block: Any, *, tool_call_id: Optional[str] = None, + screenshot: bool = False, error_detail: bool = False, **fields) -> None: + kind = mc.get(block, "type") + source = mc.get(block, "source") + if kind not in ("image", "document") or mc.get(source, "type") != "base64": + return + mime = mc.get(source, "media_type") + if mime not in (IMAGE_TYPES if kind == "image" else DOCUMENT_TYPES): + return + if kind == "document": + purpose = "document" + elif tool_call_id is not None: + purpose = "computer_screenshot" if screenshot else "tool_result" + else: + purpose = "model_input" + self._typed(position, mc.get(source, "data"), mime, direction="input", purpose=purpose, + tool_call_id=tool_call_id, error_detail=error_detail, **fields) + + def imagen_predictions(self, endpoint: Any, predictions: Any) -> "ProviderMedia": + """Vertex Imagen ``PredictResponse`` predictions: ``bytesBase64Encoded`` with a declared image type.""" + if not self.usable or not isinstance(endpoint, str) or not _IMAGEN_ENDPOINT.search(endpoint): + return self + try: + for index, prediction in enumerate(_items(predictions)): + mime = mc.get(prediction, "mimeType") + if mime in IMAGE_TYPES: + self._typed("imagen/output/%d" % index, mc.get(prediction, "bytesBase64Encoded"), mime, + direction="output", purpose="model_output", role="assistant", part_index=index) + except Exception: + pass + return self + + def bedrock_invoke_request(self, model_id: Any, body: Any) -> "ProviderMedia": + """InvokeModel request bodies of named families: Anthropic Messages, Nova, Titan/Nova Canvas images.""" + if not self.usable or not isinstance(model_id, str) or not isinstance(body, Mapping): + return self + try: + if _ANTHROPIC_MODELS.match(model_id) and "anthropic_version" in body: + self.anthropic_messages(body.get("messages"), body.get("tools")) + elif _CONVERSE_MODELS.match(model_id): + self.bedrock_converse(model_id, body.get("messages")) + elif _IMAGE_GENERATION_MODELS.match(model_id): + index = 0 + variation = body.get("imageVariationParams") + for image in _items(mc.get(variation, "images"))[:MAX_PARTS]: + self._untyped_image("titan/input/%d" % index, image, direction="input", purpose="model_input", + role="user", part_index=index) + index += 1 + for params, field in _IMAGE_PARAMS: + image = mc.get(body.get(params), field) + if image is not None: + self._untyped_image("titan/input/%s" % params, image, direction="input", + purpose="model_input", role="user", part_index=index) + index += 1 + except Exception: + pass + return self + + def bedrock_invoke_response(self, model_id: Any, body: Any) -> "ProviderMedia": + """InvokeModel response bodies the instrumentor already buffered: Titan/Nova Canvas ``images``.""" + if not self.usable or not isinstance(model_id, str) or not isinstance(body, Mapping): + return self + try: + if _IMAGE_GENERATION_MODELS.match(model_id): + for index, image in enumerate(_items(body.get("images"))[:MAX_PARTS]): + self._untyped_image("titan/output/%d" % index, image, direction="output", + purpose="model_output", role="assistant", part_index=index) + except Exception: + pass + return self + + +def _items(value: Any) -> list: + """A bounded list of a list, tuple or other sequence (proto repeated fields); never a str or bytes.""" + if isinstance(value, (str, bytes, bytearray, memoryview, Mapping)) or not isinstance(value, Sequence): + return [] + try: + return list(value[:MAX_PARTS]) + except Exception: + return [] + + +def _image_only(source: Optional[_Source]) -> Optional[_Source]: + return source if source is not None and source.mime in IMAGE_TYPES else None + + +def _inline(part: Any) -> Any: + inline = mc.get(part, "inline_data") + return inline if inline is not None else mc.get(part, "inlineData") + + +def _lower(value: Any) -> str: + return value.lower().split(";", 1)[0].strip() if isinstance(value, str) else "" + + +def _role(value: Any) -> str: + return value if value in ("user", "assistant", "tool", "system") else "unknown" + + +def _purpose(direction: str, mime: str) -> str: + if direction == "output": + return "model_output" + return "document" if mime in DOCUMENT_TYPES else "model_input" + + +def _computer_calls(messages, tools, use_type: str, id_field: str, nested: Optional[str] = None) -> frozenset: + """Tool-call IDs of assistant calls to a declared typed computer tool (``computer_YYYYMMDD``).""" + names = set() + for tool in _items(tools): + kind, name = mc.get(tool, "type"), mc.get(tool, "name") + if isinstance(kind, str) and _COMPUTER_TOOL.fullmatch(kind) and isinstance(name, str): + names.add(name) + if not names: + return frozenset() + calls = set() + for message in messages: + if mc.get(message, "role") != "assistant": + continue + for block in _items(mc.get(message, "content")): + use = mc.get(block, nested) if nested else (block if mc.get(block, "type") == use_type else None) + call_id = mc.get(use, id_field) + if use is not None and isinstance(call_id, str) and mc.get(use, "name") in names: + calls.add(call_id) + return frozenset(calls) + + +# ------------------------------------------------------------------ capability matrix +def _descriptor(operations: Tuple[str, ...], sources: Tuple[str, ...], unsupported: Tuple[str, ...], + note: Optional[str] = None) -> Mapping[str, Any]: + value = {"schema_version": 1, "operations": operations, "sources": sources, "unsupported": unsupported, + "max_inline_bytes": MAX_DECODED_BYTES} + if note: + value["note"] = note + return MappingProxyType(value) + + +_COMPAT_UNSUPPORTED = ("remote_url", "provider_file", "streamed_output", "output_media") + +#: Typed per-package capability descriptors; this IS the Python support matrix. Anything not listed +#: (remote URLs, provider file IDs, gs://, S3, PIL images, async/long-running operations, unknown model +#: bodies, native speech/OCR/image routes) is unsupported and never captured. A package with no +#: operations has no qualified media surface: it keeps tracing and captures nothing. +PROVIDER_MEDIA_CAPABILITIES: Mapping[str, Mapping[str, Any]] = MappingProxyType({ + "traceai_google_genai": _descriptor( + ("models.generate_content.input", "models.generate_content.output", "models.generate_content_stream.input"), + ("sdk_bytes", "inline_base64"), + ("file_data_uri", "gs_uri", "provider_file", "pil_image", "streamed_output", "models.generate_images", + "models.generate_videos", "live", "audio/L16")), + "traceai_vertexai": _descriptor( + ("GenerativeModel.generate_content.input", "GenerativeModel.generate_content.output", + "ImageGenerationModel.generate_images.output"), + ("sdk_bytes",), + ("file_data_uri", "gs_uri", "streamed_output", "ImageGenerationModel.edit_image", + "prediction_service_raw", "audio/L16")), + "traceai_bedrock": _descriptor( + ("converse.input", "invoke_model.anthropic.input", "invoke_model.nova.input", + "invoke_model.image_generation.input", "invoke_model.image_generation.output"), + ("sdk_bytes", "inline_base64"), + ("s3_location", "converse_stream", "invoke_model_with_response_stream", "unknown_model_body", + "video.anthropic", "converse.output")), + "traceai_litellm": _descriptor( + ("completion.input", "acompletion.input", "image_generation.output", "aimage_generation.output"), + ("data_uri", "inline_base64"), + ("remote_url", "provider_file", "streamed_output", "image_generation.url", "image_edit", + "speech", "transcription")), + "traceai_groq": _descriptor(("chat.completions.create.input",), ("data_uri",), _COMPAT_UNSUPPORTED), + "traceai_together": _descriptor(("chat.completions.create.input",), ("data_uri",), + _COMPAT_UNSUPPORTED + ("images.generate",)), + "traceai_fireworks": _descriptor(("chat.completions.create.input",), ("data_uri",), _COMPAT_UNSUPPORTED), + "traceai_xai": _descriptor(("chat.completions.create.input",), ("data_uri",), _COMPAT_UNSUPPORTED), + "traceai_mistralai": _descriptor(("chat.complete.input", "chat.stream.input"), ("data_uri",), + _COMPAT_UNSUPPORTED + ("ocr", "document_url", "input_audio")), + "traceai_openrouter": _descriptor(("chat.send.input",), ("data_uri", "inline_base64"), _COMPAT_UNSUPPORTED), + "traceai_ollama": _descriptor(("chat.input", "generate.input"), ("sdk_bytes", "inline_base64"), + ("image_path", "streamed_output", "output_media")), + "traceai_portkey": _descriptor(("chat.completions.create.input",), ("data_uri", "inline_base64"), + _COMPAT_UNSUPPORTED), + "traceai_deepseek": _descriptor((), (), ("chat.completions.create",), + "DeepSeek chat has no qualified media input or output; tracing only."), + "traceai_cerebras": _descriptor((), (), ("chat.completions.create",), + "Cerebras chat has no qualified media input or output; tracing only."), +}) + + +def capabilities(package: str) -> Mapping[str, Any]: + """The descriptor of one package; an empty-operations descriptor for unknown names.""" + return PROVIDER_MEDIA_CAPABILITIES.get(package) or _descriptor((), (), (), "not qualified") diff --git a/python/frameworks/bedrock/traceai_bedrock/__init__.py b/python/frameworks/bedrock/traceai_bedrock/__init__.py index ee49cf294..16a20e276 100644 --- a/python/frameworks/bedrock/traceai_bedrock/__init__.py +++ b/python/frameworks/bedrock/traceai_bedrock/__init__.py @@ -128,8 +128,21 @@ def reset(self) -> None: self._buffer.seek(0) +def _bedrock_media(attachments: Any, config: Any, span: Any) -> Any: + """A typed media capture for this span, or None (no manager, owned elsewhere, unsampled).""" + if attachments is None: + return None + try: + from fi_instrumentation.attachments.provider_media import ProviderMedia + + media = ProviderMedia(attachments, span, "bedrock", trace_config=config) + return media if media.usable else None + except Exception: + return None + + def _client_creation_wrapper( - tracer: Tracer, module_version: str + tracer: Tracer, module_version: str, attachments: Any = None, config: Any = None ) -> Callable[[ClientCreator], ClientCreator]: def _client_wrapper( wrapped: ClientCreator, @@ -167,14 +180,14 @@ def _client_wrapper( client = cast(InstrumentedClient, client) client._unwrapped_invoke_model = client.invoke_model - client.invoke_model = _model_invocation_wrapper(tracer)(client) + client.invoke_model = _model_invocation_wrapper(tracer, attachments, config)(client) client.invoke_model_with_response_stream = _InvokeModelWithResponseStream(tracer)( client.invoke_model_with_response_stream ) if module_version >= _MINIMUM_CONVERSE_BOTOCORE_VERSION: client._unwrapped_converse = client.converse - client.converse = _model_converse_wrapper(tracer)(client) + client.converse = _model_converse_wrapper(tracer, attachments, config)(client) # ConverseStream is an independent operation, not a call to converse. # Feature-detect it so older botocore service models remain supported. if hasattr(client, "converse_stream"): @@ -185,7 +198,9 @@ def _client_wrapper( return _client_wrapper # type: ignore -def _model_invocation_wrapper(tracer: Tracer) -> Callable[[InstrumentedClient], Callable[..., Any]]: +def _model_invocation_wrapper( + tracer: Tracer, attachments: Any = None, config: Any = None +) -> Callable[[InstrumentedClient], Callable[..., Any]]: def _invocation_wrapper(wrapped_client: InstrumentedClient) -> Callable[..., Any]: """Instruments a bedrock client's `invoke_model` or `converse` method.""" @@ -211,6 +226,11 @@ def instrumented_response(*args: Any, **kwargs: Any) -> Dict[str, Any]: model_id ) + # Typed media of named model families only (Anthropic, Nova, Titan/Nova Canvas). + media = _bedrock_media(attachments, config, span) + if media is not None: + media.bedrock_invoke_request(model_id, request_body) + # Set input attributes only when the request can be captured. if isinstance(request_body, dict): if is_claude_message_api: @@ -232,6 +252,9 @@ def instrumented_response(*args: Any, **kwargs: Any) -> Dict[str, Any]: ) response_body = json.loads(response.get("body").read()) response["body"].reset() + if media is not None: + # Generated images from the body this wrapper already buffered; no extra read. + media.bedrock_invoke_response(model_id, response_body) # Set response attributes based on model type if is_claude_message_api: @@ -249,7 +272,9 @@ def instrumented_response(*args: Any, **kwargs: Any) -> Dict[str, Any]: return _invocation_wrapper -def _model_converse_wrapper(tracer: Tracer) -> Callable[[InstrumentedClient], Callable[..., Any]]: +def _model_converse_wrapper( + tracer: Tracer, attachments: Any = None, config: Any = None +) -> Callable[[InstrumentedClient], Callable[..., Any]]: def _converse_wrapper(wrapped_client: InstrumentedClient) -> Callable[..., Any]: """Instruments a bedrock client's `converse` method.""" @@ -275,6 +300,14 @@ def instrumented_response(*args: Any, **kwargs: Any) -> Dict[str, Any]: span, SpanAttributes.GEN_AI_REQUEST_PARAMETERS, invocation_parameters ) + media = _bedrock_media(attachments, config, span) + if media is not None: + media.bedrock_converse( + kwargs.get("modelId"), + kwargs.get("messages"), + kwargs.get("additionalModelRequestFields"), + ) + aggregated_messages = [] if system_prompts := kwargs.get("system"): aggregated_messages.append( @@ -357,6 +390,20 @@ def instrumented_response(*args: Any, **kwargs: Any) -> Dict[str, Any]: return _converse_wrapper +def _attachment_manager(kwargs: Any) -> Any: + """The shared manager passed as ``attachments`` (or auto's ``attachment_manager``), if it can admit. + + A missing, disabled or pre-attachments core leaves tracing exactly as before. + """ + if kwargs.get("attachments") is None and kwargs.get("attachment_manager") is None: + return None + try: + from fi_instrumentation.attachments.provider_media import manager_from_kwargs + except ImportError: + return None + return manager_from_kwargs(kwargs) + + class BedrockInstrumentor(BaseInstrumentor): # type: ignore __slots__ = ( "_tracer", @@ -386,7 +433,10 @@ def _instrument(self, **kwargs: Any) -> None: module=_MODULE, name="ClientCreator.create_client", wrapper=_client_creation_wrapper( - tracer=self._tracer, module_version=botocore.__version__ + tracer=self._tracer, + module_version=botocore.__version__, + attachments=_attachment_manager(kwargs), + config=config, ), ) @@ -467,3 +517,12 @@ def _get_attributes_from_image( def is_iterable_of(lst: Iterable[object], tp: T) -> bool: return isinstance(lst, Iterable) and all(isinstance(x, tp) for x in lst) + + +def __getattr__(name: str) -> Any: + # Lazy: the descriptor needs a fi-instrumentation-otel release with attachments. + if name == "bedrock_attachment_capabilities": + from fi_instrumentation.attachments.provider_media import capabilities + + return capabilities("traceai_bedrock") + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/python/frameworks/cerebras/traceai_cerebras/__init__.py b/python/frameworks/cerebras/traceai_cerebras/__init__.py index 1062d0797..c7c7cce1c 100644 --- a/python/frameworks/cerebras/traceai_cerebras/__init__.py +++ b/python/frameworks/cerebras/traceai_cerebras/__init__.py @@ -117,3 +117,12 @@ def _uninstrument(self, **kwargs: Any) -> None: cerebras_module.AsyncCompletionsResource.create = self._original_async_completions_create except ImportError: pass + + +def __getattr__(name: str) -> Any: + # Lazy: the descriptor needs a fi-instrumentation-otel release with attachments. + if name == "cerebras_attachment_capabilities": + from fi_instrumentation.attachments.provider_media import capabilities + + return capabilities("traceai_cerebras") + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/python/frameworks/deepseek/traceai_deepseek/__init__.py b/python/frameworks/deepseek/traceai_deepseek/__init__.py index 569460d6d..36829275b 100644 --- a/python/frameworks/deepseek/traceai_deepseek/__init__.py +++ b/python/frameworks/deepseek/traceai_deepseek/__init__.py @@ -131,3 +131,12 @@ def _uninstrument(self, **kwargs: Any) -> None: completions.AsyncCompletions.create = self._original_async_chat_create except Exception as e: logger.warning(f"Failed to uninstrument: {e}") + + +def __getattr__(name: str) -> Any: + # Lazy: the descriptor needs a fi-instrumentation-otel release with attachments. + if name == "deepseek_attachment_capabilities": + from fi_instrumentation.attachments.provider_media import capabilities + + return capabilities("traceai_deepseek") + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/python/frameworks/fireworks/traceai_fireworks/__init__.py b/python/frameworks/fireworks/traceai_fireworks/__init__.py index 00259fba4..11f283eaf 100644 --- a/python/frameworks/fireworks/traceai_fireworks/__init__.py +++ b/python/frameworks/fireworks/traceai_fireworks/__init__.py @@ -59,6 +59,20 @@ ] +def _attachment_manager(kwargs: Any) -> Any: + """The shared manager passed as ``attachments`` (or auto's ``attachment_manager``), if it can admit. + + A missing, disabled or pre-attachments core leaves tracing exactly as before. + """ + if kwargs.get("attachments") is None and kwargs.get("attachment_manager") is None: + return None + try: + from fi_instrumentation.attachments.provider_media import manager_from_kwargs + except ImportError: + return None + return manager_from_kwargs(kwargs) + + class FireworksInstrumentor(BaseInstrumentor): """An instrumentor for Fireworks AI (OpenAI-compatible API).""" @@ -107,7 +121,7 @@ def _instrument(self, **kwargs: Any) -> None: wrap_function_wrapper( module="openai.resources.chat", name="Completions.create", - wrapper=_CompletionsWrapper(tracer=self._tracer, config=config), + wrapper=_CompletionsWrapper(tracer=self._tracer, config=config, attachments=_attachment_manager(kwargs)), ) # Wrap asynchronous completions @@ -115,7 +129,9 @@ def _instrument(self, **kwargs: Any) -> None: wrap_function_wrapper( module="openai.resources.chat", name="AsyncCompletions.create", - wrapper=_AsyncCompletionsWrapper(tracer=self._tracer, config=config), + wrapper=_AsyncCompletionsWrapper( + tracer=self._tracer, config=config, attachments=_attachment_manager(kwargs) + ), ) def _uninstrument(self, **kwargs: Any) -> None: @@ -127,3 +143,12 @@ def _uninstrument(self, **kwargs: Any) -> None: openai_module.AsyncCompletions.create = self._original_async_completions_create except ImportError: pass + + +def __getattr__(name: str) -> Any: + # Lazy: the descriptor needs a fi-instrumentation-otel release with attachments. + if name == "fireworks_attachment_capabilities": + from fi_instrumentation.attachments.provider_media import capabilities + + return capabilities("traceai_fireworks") + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/python/frameworks/fireworks/traceai_fireworks/_wrappers.py b/python/frameworks/fireworks/traceai_fireworks/_wrappers.py index 96ebbf8b1..f649c2c51 100644 --- a/python/frameworks/fireworks/traceai_fireworks/_wrappers.py +++ b/python/frameworks/fireworks/traceai_fireworks/_wrappers.py @@ -49,13 +49,30 @@ class _WithTracer(ABC): """Base class for wrappers that need a tracer.""" + _media_dialect = "image" + def __init__( self, tracer: trace_api.Tracer, *args: Any, - config: Optional[TraceConfig] = None, **kwargs: Any, + config: Optional[TraceConfig] = None, attachments: Any = None, **kwargs: Any, ) -> None: super().__init__(*args, **kwargs) self._tracer = tracer self._config = config if config is not None else TraceConfig() + # Provider-owned shared attachment manager, or None (no capture, no imports). + self._attachments = attachments + + def _capture_chat_media(self, span: Any, messages: Any) -> None: + """Typed chat media (data-URI images) through the shared attachment manager; never raises.""" + if self._attachments is None: + return + try: + from fi_instrumentation.attachments.provider_media import ProviderMedia + + ProviderMedia( + self._attachments, span, "fireworks", trace_config=self._config + ).compatible_messages(messages, self._media_dialect) + except Exception: + logger.debug("Attachment capture unavailable", exc_info=True) @contextmanager def _start_span( @@ -408,6 +425,7 @@ def __call__( input_attrs = _get_input_attributes(model, messages, config=self._config, **extra_kwargs) with self._start_span("Fireworks Chat Completions", input_attrs) as span: + self._capture_chat_media(span, messages) try: response = wrapped(*args, **kwargs) @@ -472,6 +490,7 @@ async def __call__( input_attrs = _get_input_attributes(model, messages, config=self._config, **extra_kwargs) with self._start_span("Fireworks Chat Completions", input_attrs) as span: + self._capture_chat_media(span, messages) try: response = await wrapped(*args, **kwargs) diff --git a/python/frameworks/google-genai/traceai_google_genai/__init__.py b/python/frameworks/google-genai/traceai_google_genai/__init__.py index 3b9305009..cbc62c6ec 100644 --- a/python/frameworks/google-genai/traceai_google_genai/__init__.py +++ b/python/frameworks/google-genai/traceai_google_genai/__init__.py @@ -13,6 +13,20 @@ logger.addHandler(logging.NullHandler()) +def _attachment_manager(kwargs: Any) -> Any: + """The shared manager passed as ``attachments`` (or auto's ``attachment_manager``), if it can admit. + + A missing, disabled or pre-attachments core leaves tracing exactly as before. + """ + if kwargs.get("attachments") is None and kwargs.get("attachment_manager") is None: + return None + try: + from fi_instrumentation.attachments.provider_media import manager_from_kwargs + except ImportError: + return None + return manager_from_kwargs(kwargs) + + class GoogleGenAIInstrumentor(BaseInstrumentor): # type: ignore """ An instrumentor for `google-genai` @@ -55,32 +69,34 @@ def _instrument(self, **kwargs: Any) -> None: _SyncGenerateContentStream, ) + media = dict(attachments=_attachment_manager(kwargs), config=config) + self._original_generate_content = Models.generate_content wrap_function_wrapper( module="google.genai.models", name="Models.generate_content", - wrapper=_SyncGenerateContent(tracer=self._tracer), + wrapper=_SyncGenerateContent(tracer=self._tracer, **media), ) self._original_async_generate_content = AsyncModels.generate_content wrap_function_wrapper( module="google.genai.models", name="AsyncModels.generate_content", - wrapper=_AsyncGenerateContentWrapper(tracer=self._tracer), + wrapper=_AsyncGenerateContentWrapper(tracer=self._tracer, **media), ) self._original_generate_content_stream = Models.generate_content_stream wrap_function_wrapper( module="google.genai.models", name="Models.generate_content_stream", - wrapper=_SyncGenerateContentStream(tracer=self._tracer), + wrapper=_SyncGenerateContentStream(tracer=self._tracer, **media), ) self._original_async_generate_content_stream = AsyncModels.generate_content_stream wrap_function_wrapper( module="google.genai.models", name="AsyncModels.generate_content_stream", - wrapper=_AsyncGenerateContentStream(tracer=self._tracer), + wrapper=_AsyncGenerateContentStream(tracer=self._tracer, **media), ) def _uninstrument(self, **kwargs: Any) -> None: @@ -99,3 +115,12 @@ def _uninstrument(self, **kwargs: Any) -> None: setattr( AsyncModels, "generate_content_stream", self._original_async_generate_content_stream ) + + +def __getattr__(name: str) -> Any: + # Lazy: the descriptor needs a fi-instrumentation-otel release with attachments. + if name == "google_genai_attachment_capabilities": + from fi_instrumentation.attachments.provider_media import capabilities + + return capabilities("traceai_google_genai") + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/python/frameworks/google-genai/traceai_google_genai/_wrappers.py b/python/frameworks/google-genai/traceai_google_genai/_wrappers.py index 39b0ecf18..e0e3cb8d7 100644 --- a/python/frameworks/google-genai/traceai_google_genai/_wrappers.py +++ b/python/frameworks/google-genai/traceai_google_genai/_wrappers.py @@ -54,9 +54,34 @@ class _WithTracer(ABC): Base class for wrappers that need a tracer. """ - def __init__(self, tracer: trace_api.Tracer, *args: Any, **kwargs: Any) -> None: + def __init__( + self, + tracer: trace_api.Tracer, + *args: Any, + attachments: Any = None, + config: Any = None, + **kwargs: Any, + ) -> None: super().__init__(*args, **kwargs) self._tracer = tracer + # Provider-owned shared attachment manager, or None (no capture, no imports). + self._attachments = attachments + self._config = config + + def _capture_input(self, span: _WithSpan, wrapped: Callable[..., Any], args: Any, kwargs: Any) -> Any: + """Typed inline_data Parts of the request; returns the capture for the response, or None.""" + if self._attachments is None: + return None + try: + from fi_instrumentation.attachments.provider_media import ProviderMedia + + bound = signature(wrapped).bind(*args, **kwargs).arguments + return ProviderMedia( + self._attachments, span._span, "google_genai", trace_config=self._config + ).gemini_contents(bound.get("contents")) + except Exception: + logger.debug("Attachment capture unavailable", exc_info=True) + return None @contextmanager def _start_as_current_span( @@ -148,6 +173,7 @@ def __call__( request_parameters ), ) as span: + media = self._capture_input(span, wrapped, args, kwargs) try: response = wrapped(*args, **kwargs) except Exception as exception: @@ -158,6 +184,9 @@ def __call__( ) span.finish_tracing(status=status) raise + if media is not None: + # Completed response only (generated images, audio, files); streams stay input-only. + media.gemini_response(response) try: _finish_tracing( status=trace_api.Status(status_code=trace_api.StatusCode.OK), @@ -209,6 +238,7 @@ def __call__( request_parameters ), ) as span: + media = self._capture_input(span, wrapped, args, kwargs) try: response = wrapped(*args, **kwargs) except Exception as exception: @@ -268,6 +298,7 @@ async def __call__( request_parameters ), ) as span: + media = self._capture_input(span, wrapped, args, kwargs) try: response = await wrapped(*args, **kwargs) except Exception as exception: @@ -278,6 +309,9 @@ async def __call__( ) span.finish_tracing(status=status) raise + if media is not None: + # Completed response only (generated images, audio, files); streams stay input-only. + media.gemini_response(response) try: _finish_tracing( status=trace_api.Status(status_code=trace_api.StatusCode.OK), @@ -329,6 +363,7 @@ async def __call__( request_parameters ), ) as span: + media = self._capture_input(span, wrapped, args, kwargs) try: response = await wrapped(*args, **kwargs) except Exception as exception: diff --git a/python/frameworks/groq/traceai_groq/__init__.py b/python/frameworks/groq/traceai_groq/__init__.py index 6544393ca..6ea706c57 100644 --- a/python/frameworks/groq/traceai_groq/__init__.py +++ b/python/frameworks/groq/traceai_groq/__init__.py @@ -66,14 +66,18 @@ def _instrument(self, **kwargs: Any) -> None: wrap_function_wrapper( module="groq.resources.chat.completions", name="Completions.create", - wrapper=_CompletionsWrapper(tracer=self._tracer), + wrapper=_CompletionsWrapper( + tracer=self._tracer, attachments=_attachment_manager(kwargs), config=config + ), ) self._original_async_completions_create = AsyncCompletions.create wrap_function_wrapper( module="groq.resources.chat.completions", name="AsyncCompletions.create", - wrapper=_AsyncCompletionsWrapper(tracer=self._tracer), + wrapper=_AsyncCompletionsWrapper( + tracer=self._tracer, attachments=_attachment_manager(kwargs), config=config + ), ) # Audio spans exist only with an enabled attachment manager: plain tracing is unchanged. diff --git a/python/frameworks/groq/traceai_groq/_audio.py b/python/frameworks/groq/traceai_groq/_audio.py index 00c791ecc..e8407742d 100644 --- a/python/frameworks/groq/traceai_groq/_audio.py +++ b/python/frameworks/groq/traceai_groq/_audio.py @@ -32,10 +32,12 @@ groq_attachment_capabilities = { "schema_version": 1, + # Chat image input (data URIs) is the compatible-message capture in fi_instrumentation's provider_media. "operations": ("audio.speech.create.output", "audio.transcriptions.create.input", - "audio.translations.create.input"), - "sources": ("sdk_buffered_bytes", "inline_bytes"), - "unsupported": ("streamed_speech", "file_path", "open_file_object", "chat_media"), + "audio.translations.create.input", "chat.completions.create.input"), + "sources": ("sdk_buffered_bytes", "inline_bytes", "data_uri"), + "unsupported": ("streamed_speech", "file_path", "open_file_object", "chat_remote_url", "chat_input_audio", + "chat_output_media"), } diff --git a/python/frameworks/groq/traceai_groq/_wrappers.py b/python/frameworks/groq/traceai_groq/_wrappers.py index a239eee46..0702398ac 100644 --- a/python/frameworks/groq/traceai_groq/_wrappers.py +++ b/python/frameworks/groq/traceai_groq/_wrappers.py @@ -51,9 +51,34 @@ class _WithTracer(ABC): Base class for wrappers that need a tracer. """ - def __init__(self, tracer: trace_api.Tracer, *args: Any, **kwargs: Any) -> None: + _media_dialect = "groq" + + def __init__( + self, + tracer: trace_api.Tracer, + *args: Any, + attachments: Any = None, + config: Any = None, + **kwargs: Any, + ) -> None: super().__init__(*args, **kwargs) self._tracer = tracer + # Provider-owned shared attachment manager, or None (no capture, no imports). + self._attachments = attachments + self._config = config + + def _capture_chat_media(self, span: Any, messages: Any) -> None: + """Typed chat media (data-URI images) through the shared attachment manager; never raises.""" + if self._attachments is None: + return + try: + from fi_instrumentation.attachments.provider_media import ProviderMedia + + ProviderMedia( + self._attachments, span, "groq", trace_config=self._config + ).compatible_messages(messages, self._media_dialect) + except Exception: + logger.debug("Attachment capture unavailable", exc_info=True) @contextmanager def _start_as_current_span( @@ -297,6 +322,7 @@ def __call__( request_parameters ), ) as span: + self._capture_chat_media(span._span, kwargs.get("messages")) try: response = wrapped(*args, **kwargs) streaming = request_parameters.get("stream", False) @@ -370,6 +396,7 @@ async def __call__( request_parameters ), ) as span: + self._capture_chat_media(span._span, kwargs.get("messages")) try: response = await wrapped(*args, **kwargs) streaming = request_parameters.get("stream", False) diff --git a/python/frameworks/litellm/traceai_litellm/__init__.py b/python/frameworks/litellm/traceai_litellm/__init__.py index 483cedf88..ed5061f91 100644 --- a/python/frameworks/litellm/traceai_litellm/__init__.py +++ b/python/frameworks/litellm/traceai_litellm/__init__.py @@ -1,5 +1,6 @@ import json from enum import Enum +from contextlib import contextmanager from functools import wraps from inspect import Parameter, signature from typing import ( @@ -11,6 +12,7 @@ Iterable, Iterator, Mapping, + Optional, Tuple, TypeVar, ) @@ -335,6 +337,20 @@ def _finalize_span(span: trace_api.Span, result: Any) -> None: break +def _attachment_manager(kwargs: Any) -> Any: + """The shared manager passed as ``attachments`` (or auto's ``attachment_manager``), if it can admit. + + A missing, disabled or pre-attachments core leaves tracing exactly as before. + """ + if kwargs.get("attachments") is None and kwargs.get("attachment_manager") is None: + return None + try: + from fi_instrumentation.attachments.provider_media import manager_from_kwargs + except ImportError: + return None + return manager_from_kwargs(kwargs) + + class LiteLLMInstrumentor(BaseInstrumentor): # type: ignore original_litellm_funcs: Dict[str, Callable[..., Any]] = ( {} @@ -354,6 +370,9 @@ def _instrument(self, **kwargs: Any) -> None: trace_api.get_tracer(__name__, __version__, tracer_provider), config=config, ) + # Provider-owned shared attachment manager, or None (no capture, no imports). + self._attachments = _attachment_manager(kwargs) + self._media_config = config functions_to_instrument = { "completion": self._completion_wrapper, @@ -396,6 +415,49 @@ def _uninstrument(self, **kwargs: Any) -> None: self.original_litellm_funcs.clear() self._tracer = None + def _media(self, span: trace_api.Span, arguments: Optional[Mapping[str, Any]] = None) -> Any: + """Capture typed chat media of this call; returns the capture (also for output), or None.""" + if getattr(self, "_attachments", None) is None: + return None + try: + from fi_instrumentation.attachments.provider_media import ProviderMedia + + media = ProviderMedia(self._attachments, span, "litellm", trace_config=self._media_config) + if not media.usable: + return None + if arguments is not None: + media.compatible_messages(arguments.get("messages"), "openai") + return media + except Exception: + logger.debug("Attachment capture unavailable", exc_info=True) + return None + + @contextmanager + def _owned(self, media: Any) -> Iterator[None]: + """Run LiteLLM's downstream SDK call as the owner of this call's media. + + A nested provider instrumentor (for example OpenAI under LiteLLM) then + uploads nothing for the same call. Only set while this adapter captures. + """ + if media is None: + yield + return + from fi_instrumentation.attachments.provider_media import claim + + token = context_api.attach(claim()) + try: + yield + finally: + context_api.detach(token) + + def _generated(self, media: Any, result: Any) -> None: + if media is None or not isinstance(result, ImageResponse): + return + try: + media.generated_images(result.data) + except Exception: + logger.debug("Attachment capture unavailable", exc_info=True) + def _get_call_arguments( self, func_name: str, args: Tuple[Any, ...], kwargs: Dict[str, Any] ) -> Dict[str, Any]: @@ -445,7 +507,9 @@ def _completion_wrapper(self, *args: Any, **kwargs: Any) -> Any: ) as span: try: _instrument_func_type_completion(span, arguments) - result = self.original_litellm_funcs["completion"](*args, **kwargs) + media = self._media(span, arguments) + with self._owned(media): + result = self.original_litellm_funcs["completion"](*args, **kwargs) _finalize_span(span, result) span.set_status(trace_api.StatusCode.OK) return result @@ -459,9 +523,11 @@ def _completion_wrapper(self, *args: Any, **kwargs: Any) -> Any: span = span_cm.__enter__() try: _instrument_func_type_completion(span, arguments) - original_iterator = self.original_litellm_funcs["completion"]( - *args, **kwargs - ) + media = self._media(span, arguments) + with self._owned(media): + original_iterator = self.original_litellm_funcs["completion"]( + *args, **kwargs + ) return StreamingIteratorWrapper(original_iterator, span, span_cm) except BaseException as e: span_cm.__exit__(type(e), e, e.__traceback__) @@ -479,9 +545,11 @@ async def _acompletion_wrapper(self, *args: Any, **kwargs: Any) -> Any: ) as span: try: _instrument_func_type_completion(span, arguments) - result = await self.original_litellm_funcs["acompletion"]( - *args, **kwargs - ) + media = self._media(span, arguments) + with self._owned(media): + result = await self.original_litellm_funcs["acompletion"]( + *args, **kwargs + ) _finalize_span(span, result) span.set_status(trace_api.StatusCode.OK) return result @@ -495,9 +563,11 @@ async def _acompletion_wrapper(self, *args: Any, **kwargs: Any) -> Any: span = span_cm.__enter__() try: _instrument_func_type_completion(span, arguments) - original_async_iterator = await self.original_litellm_funcs[ - "acompletion" - ](*args, **kwargs) + media = self._media(span, arguments) + with self._owned(media): + original_async_iterator = await self.original_litellm_funcs[ + "acompletion" + ](*args, **kwargs) return AsyncStreamingIteratorWrapper( original_async_iterator, span, span_cm ) @@ -521,9 +591,11 @@ def _completion_with_retries_wrapper( ) as span: try: _instrument_func_type_completion(span, arguments) - result = self.original_litellm_funcs["completion_with_retries"]( - *args, **kwargs - ) + media = self._media(span, arguments) + with self._owned(media): + result = self.original_litellm_funcs["completion_with_retries"]( + *args, **kwargs + ) _finalize_span(span, result) span.set_status(trace_api.StatusCode.OK) return result # type:ignore @@ -546,9 +618,11 @@ async def _acompletion_with_retries_wrapper( ) as span: try: _instrument_func_type_completion(span, arguments) - result = await self.original_litellm_funcs[ - "acompletion_with_retries" - ](*args, **kwargs) + media = self._media(span, arguments) + with self._owned(media): + result = await self.original_litellm_funcs[ + "acompletion_with_retries" + ](*args, **kwargs) _finalize_span(span, result) span.set_status(trace_api.StatusCode.OK) return result # type:ignore @@ -610,9 +684,12 @@ def _image_generation_wrapper(self, *args: Any, **kwargs: Any) -> ImageResponse: ) as span: try: _instrument_func_type_image_generation(span, arguments, args) - result = self.original_litellm_funcs["image_generation"]( - *args, **kwargs - ) + media = self._media(span) + with self._owned(media): + result = self.original_litellm_funcs["image_generation"]( + *args, **kwargs + ) + self._generated(media, result) _finalize_span(span, result) span.set_status(trace_api.StatusCode.OK) return result # type:ignore @@ -634,9 +711,12 @@ async def _aimage_generation_wrapper( ) as span: try: _instrument_func_type_image_generation(span, arguments, args) - result = await self.original_litellm_funcs["aimage_generation"]( - *args, **kwargs - ) + media = self._media(span) + with self._owned(media): + result = await self.original_litellm_funcs["aimage_generation"]( + *args, **kwargs + ) + self._generated(media, result) _finalize_span(span, result) span.set_status(trace_api.StatusCode.OK) return result # type:ignore @@ -856,3 +936,12 @@ def _process_messages(messages): "eval_input": None, "query": None, } + + +def __getattr__(name: str) -> Any: + # Lazy: the descriptor needs a fi-instrumentation-otel release with attachments. + if name == "litellm_attachment_capabilities": + from fi_instrumentation.attachments.provider_media import capabilities + + return capabilities("traceai_litellm") + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/python/frameworks/mistralai/traceai_mistralai/__init__.py b/python/frameworks/mistralai/traceai_mistralai/__init__.py index c72b6139f..969748bfc 100644 --- a/python/frameworks/mistralai/traceai_mistralai/__init__.py +++ b/python/frameworks/mistralai/traceai_mistralai/__init__.py @@ -17,6 +17,20 @@ _MODULE = "mistralai" +def _attachment_manager(kwargs: Any) -> Any: + """The shared manager passed as ``attachments`` (or auto's ``attachment_manager``), if it can admit. + + A missing, disabled or pre-attachments core leaves tracing exactly as before. + """ + if kwargs.get("attachments") is None and kwargs.get("attachment_manager") is None: + return None + try: + from fi_instrumentation.attachments.provider_media import manager_from_kwargs + except ImportError: + return None + return manager_from_kwargs(kwargs) + + class MistralAIInstrumentor(BaseInstrumentor): # type: ignore """ An instrumentor for mistralai @@ -61,10 +75,13 @@ def _instrument(self, **kwargs: Any) -> None: "traceAI-mistralai supports mistralai >=1.1,<2 or >=3.6,<4" ) self._native = None + media = dict(attachments=_attachment_manager(kwargs), config=config) if sdk_version >= Version("3.6.0"): from traceai_mistralai._native import _NativeInstrumentation - self._native = _NativeInstrumentation(tracer_provider, config) + self._native = _NativeInstrumentation( + tracer_provider, config, attachments=media["attachments"] + ) return from traceai_mistralai._chat_wrapper import ( @@ -104,20 +121,20 @@ def _instrument(self, **kwargs: Any) -> None: wrap_function_wrapper( module="mistralai.chat", name="Chat.complete", - wrapper=_SyncChatWrapper("MistralClient.chat", self._tracer, mistralai), + wrapper=_SyncChatWrapper("MistralClient.chat", self._tracer, mistralai, **media), ) wrap_function_wrapper( module="mistralai.chat", name="Chat.stream", - wrapper=_SyncChatWrapper("MistralClient.chat", self._tracer, mistralai), + wrapper=_SyncChatWrapper("MistralClient.chat", self._tracer, mistralai, **media), ) wrap_function_wrapper( module="mistralai.chat", name="Chat.complete_async", wrapper=_AsyncChatWrapper( - "MistralAsyncClient.chat", self._tracer, mistralai + "MistralAsyncClient.chat", self._tracer, mistralai, **media ), ) @@ -125,7 +142,7 @@ def _instrument(self, **kwargs: Any) -> None: module="mistralai.chat", name="Chat.stream_async", wrapper=_AsyncStreamChatWrapper( - "MistralAsyncClient.chat", self._tracer, mistralai + "MistralAsyncClient.chat", self._tracer, mistralai, **media ), ) @@ -201,3 +218,12 @@ def _uninstrument(self, **kwargs: Any) -> None: Agents.stream = self._original_sync_stream_agent_method # type: ignore Agents.complete_async = self._original_async_agent_method # type: ignore Agents.stream_async = self._original_async_stream_agent_method # type: ignore + + +def __getattr__(name: str) -> Any: + # Lazy: the descriptor needs a fi-instrumentation-otel release with attachments. + if name == "mistralai_attachment_capabilities": + from fi_instrumentation.attachments.provider_media import capabilities + + return capabilities("traceai_mistralai") + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/python/frameworks/mistralai/traceai_mistralai/_chat_wrapper.py b/python/frameworks/mistralai/traceai_mistralai/_chat_wrapper.py index 940ceaa7d..21398b445 100644 --- a/python/frameworks/mistralai/traceai_mistralai/_chat_wrapper.py +++ b/python/frameworks/mistralai/traceai_mistralai/_chat_wrapper.py @@ -102,9 +102,32 @@ async def _async_parse(wrapped, instance, args, kwargs): class _WithTracer(ABC): - def __init__(self, tracer: trace_api.Tracer, *args: Any, **kwargs: Any) -> None: + def __init__( + self, + tracer: trace_api.Tracer, + *args: Any, + attachments: Any = None, + config: Any = None, + **kwargs: Any, + ) -> None: super().__init__(*args, **kwargs) self._tracer = tracer + # Provider-owned shared attachment manager, or None (no capture, no imports). + self._attachments = attachments + self._config = config + + def _capture_chat_media(self, span: Any, messages: Any) -> None: + """Typed chat media (data-URI images) through the shared attachment manager; never raises.""" + if self._attachments is None: + return + try: + from fi_instrumentation.attachments.provider_media import ProviderMedia + + ProviderMedia( + self._attachments, span, "mistralai", trace_config=self._config + ).compatible_messages(messages, "mistral") + except Exception: + logger.debug("Attachment capture unavailable", exc_info=True) @contextmanager def _start_as_current_span( @@ -304,6 +327,7 @@ def __call__( request_parameters ), ) as with_span: + self._capture_chat_media(with_span._span, kwargs.get("messages")) try: response = wrapped(*args, **kwargs) except Exception as exception: @@ -359,6 +383,7 @@ async def __call__( request_parameters ), ) as with_span: + self._capture_chat_media(with_span._span, kwargs.get("messages")) try: response = await wrapped(*args, **kwargs) except (Exception, asyncio.CancelledError) as exception: @@ -414,6 +439,7 @@ async def __call__( request_parameters ), ) as with_span: + self._capture_chat_media(with_span._span, kwargs.get("messages")) try: response = await wrapped(*args, **kwargs) except (Exception, asyncio.CancelledError) as exception: diff --git a/python/frameworks/mistralai/traceai_mistralai/_native.py b/python/frameworks/mistralai/traceai_mistralai/_native.py index d6d19ee9b..38564d911 100644 --- a/python/frameworks/mistralai/traceai_mistralai/_native.py +++ b/python/frameworks/mistralai/traceai_mistralai/_native.py @@ -152,10 +152,17 @@ class _NativeInstrumentation: "_telemetry_auto_disabled", ) - def __init__(self, provider: trace.TracerProvider, config: TraceConfig) -> None: + def __init__( + self, provider: trace.TracerProvider, config: TraceConfig, attachments: Any = None + ) -> None: from mistralai.client._hooks.tracing import TracingHook self._provider = _NativeProvider(provider, config) + # Typed chat messages of the current call, handed to the SDK-created request span + # in before_request. Only set while an attachment manager is configured. + self._attachments = attachments + self._config = config + self._media_messages = ContextVar("mistral_media_messages", default=None) self._hooks = WeakKeyDictionary() self._async_sends = WeakKeyDictionary() self._lock = RLock() @@ -224,13 +231,54 @@ def _call_scope(self): self._response_scope.reset(token) def _call(self, wrapped, instance, args, kwargs): - with self._call_scope(): + with self._call_scope(), self._media_scope(wrapped, instance, kwargs): return wrapped(*args, **kwargs) async def _call_async(self, wrapped, instance, args, kwargs): - with self._call_scope(): + with self._call_scope(), self._media_scope(wrapped, instance, kwargs): return await wrapped(*args, **kwargs) + @contextmanager + def _media_scope(self, wrapped, instance, kwargs): + """Hold chat.complete/stream messages for the first request span of this call.""" + messages = None + if self._attachments is not None and self._media_messages.get() is None: + try: + from mistralai.client.chat import Chat + + if isinstance(instance, Chat) and getattr(wrapped, "__name__", "").split("_")[0] in ( + "complete", + "stream", + ): + messages = kwargs.get("messages") + except Exception: + messages = None + if messages is None: + yield + return + token = self._media_messages.set([messages]) + try: + yield + finally: + self._media_messages.reset(token) + + def _capture_media(self, instance, request) -> None: + holder = self._media_messages.get() + if not holder or holder[0] is None: + return + messages, holder[0] = holder[0], None # Retries never upload the same call twice. + try: + span = instance._get_request_span(request) + if not isinstance(span, _NativeSpan): + return + from fi_instrumentation.attachments.provider_media import ProviderMedia + + ProviderMedia( + self._attachments, span, "mistralai", trace_config=self._config + ).compatible_messages(messages, "mistral") + except Exception: + logger.debug("Attachment capture unavailable") + def _before_request(self, wrapped, instance, args, kwargs): with self._lock: if ( @@ -244,6 +292,8 @@ def _before_request(self, wrapped, instance, args, kwargs): # Do not let SDK auto-export configuration create a second destination. instance._telemetry_auto_disabled = True request = wrapped(*args, **kwargs) + if self._attachments is not None: + self._capture_media(instance, request) hook_ctx = kwargs.get("hook_ctx", args[0] if args else None) client = getattr(getattr(hook_ctx, "config", None), "async_client", None) if client is not None: diff --git a/python/frameworks/ollama/traceai_ollama/__init__.py b/python/frameworks/ollama/traceai_ollama/__init__.py index 2743dbe08..6d44a132e 100644 --- a/python/frameworks/ollama/traceai_ollama/__init__.py +++ b/python/frameworks/ollama/traceai_ollama/__init__.py @@ -32,6 +32,20 @@ _instruments = ("ollama >= 0.3.0",) +def _attachment_manager(kwargs: Any) -> Any: + """The shared manager passed as ``attachments`` (or auto's ``attachment_manager``), if it can admit. + + A missing, disabled or pre-attachments core leaves tracing exactly as before. + """ + if kwargs.get("attachments") is None and kwargs.get("attachment_manager") is None: + return None + try: + from fi_instrumentation.attachments.provider_media import manager_from_kwargs + except ImportError: + return None + return manager_from_kwargs(kwargs) + + class OllamaInstrumentor(BaseInstrumentor): """OpenTelemetry instrumentor for Ollama.""" @@ -64,20 +78,21 @@ def _instrument(self, **kwargs: Any) -> None: ) ollama_module = import_module("ollama") + attachments = _attachment_manager(kwargs) # Wrap synchronous Client methods self._original_chat = ollama_module.Client.chat wrap_function_wrapper( module="ollama", name="Client.chat", - wrapper=_ChatWrapper(tracer=self._tracer), + wrapper=_ChatWrapper(tracer=self._tracer, attachments=attachments), ) self._original_generate = ollama_module.Client.generate wrap_function_wrapper( module="ollama", name="Client.generate", - wrapper=_GenerateWrapper(tracer=self._tracer), + wrapper=_GenerateWrapper(tracer=self._tracer, attachments=attachments), ) # Ollama uses 'embed' for single and 'embeddings' for batch @@ -102,14 +117,14 @@ def _instrument(self, **kwargs: Any) -> None: wrap_function_wrapper( module="ollama", name="AsyncClient.chat", - wrapper=_AsyncChatWrapper(tracer=self._tracer), + wrapper=_AsyncChatWrapper(tracer=self._tracer, attachments=attachments), ) self._original_async_generate = ollama_module.AsyncClient.generate wrap_function_wrapper( module="ollama", name="AsyncClient.generate", - wrapper=_AsyncGenerateWrapper(tracer=self._tracer), + wrapper=_AsyncGenerateWrapper(tracer=self._tracer, attachments=attachments), ) if hasattr(ollama_module.AsyncClient, "embed"): @@ -133,14 +148,14 @@ def _instrument(self, **kwargs: Any) -> None: wrap_function_wrapper( module="ollama", name="chat", - wrapper=_ChatWrapper(tracer=self._tracer), + wrapper=_ChatWrapper(tracer=self._tracer, attachments=attachments), ) if hasattr(ollama_module, "generate"): wrap_function_wrapper( module="ollama", name="generate", - wrapper=_GenerateWrapper(tracer=self._tracer), + wrapper=_GenerateWrapper(tracer=self._tracer, attachments=attachments), ) if hasattr(ollama_module, "embed"): @@ -187,3 +202,12 @@ def _uninstrument(self, **kwargs: Any) -> None: ollama_module.AsyncClient.embed = self._original_async_embed if hasattr(self, "_original_async_embeddings"): ollama_module.AsyncClient.embeddings = self._original_async_embeddings + + +def __getattr__(name: str) -> Any: + # Lazy: the descriptor needs a fi-instrumentation-otel release with attachments. + if name == "ollama_attachment_capabilities": + from fi_instrumentation.attachments.provider_media import capabilities + + return capabilities("traceai_ollama") + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/python/frameworks/ollama/traceai_ollama/_wrappers.py b/python/frameworks/ollama/traceai_ollama/_wrappers.py index b5f33c22e..dcbd43c88 100644 --- a/python/frameworks/ollama/traceai_ollama/_wrappers.py +++ b/python/frameworks/ollama/traceai_ollama/_wrappers.py @@ -23,12 +23,29 @@ class _WithTracer(ABC): """Base class for wrappers that need a tracer.""" - def __init__(self, tracer: trace_api.Tracer, *args: Any, **kwargs: Any) -> None: + def __init__(self, tracer: trace_api.Tracer, *args: Any, attachments: Any = None, **kwargs: Any) -> None: super().__init__(*args, **kwargs) self._tracer = tracer # Use the exact FITracer config: explicit options override environment values. config = getattr(tracer, "_self_config", None) self._config = config if isinstance(config, TraceConfig) else TraceConfig() + # Provider-owned shared attachment manager, or None (no capture, no imports). + self._attachments = attachments + + def _capture_images(self, span: Any, request: Mapping[str, Any]) -> None: + """Ollama ``images`` (bytes or base64; never a path) through the shared manager; never raises.""" + if self._attachments is None: + return + try: + from fi_instrumentation.attachments.provider_media import ProviderMedia + + media = ProviderMedia(self._attachments, span, "ollama", trace_config=self._config) + if "messages" in request: + media.ollama_messages(request.get("messages")) + else: + media.ollama_images(request.get("images")) + except Exception: + logger.debug("Attachment capture unavailable", exc_info=True) @contextmanager def _start_as_current_span( @@ -91,6 +108,7 @@ def __call__( context_attributes=get_attributes_from_context(), extra_attributes=self._request_extractor.get_extra_attributes_from_request(request_parameters), ) as span: + self._capture_images(span._span, request_parameters) try: response = wrapped(*args, **kwargs) except Exception as exception: @@ -222,6 +240,7 @@ async def __call__( context_attributes=get_attributes_from_context(), extra_attributes=self._request_extractor.get_extra_attributes_from_request(request_parameters), ) as span: + self._capture_images(span._span, request_parameters) try: response = await wrapped(*args, **kwargs) except Exception as exception: @@ -350,6 +369,7 @@ def __call__( context_attributes=get_attributes_from_context(), extra_attributes=self._request_extractor.get_extra_attributes_from_request(request_parameters), ) as span: + self._capture_images(span._span, request_parameters) try: response = wrapped(*args, **kwargs) except Exception as exception: @@ -475,6 +495,7 @@ async def __call__( context_attributes=get_attributes_from_context(), extra_attributes=self._request_extractor.get_extra_attributes_from_request(request_parameters), ) as span: + self._capture_images(span._span, request_parameters) try: response = await wrapped(*args, **kwargs) except Exception as exception: diff --git a/python/frameworks/openrouter/traceai_openrouter/__init__.py b/python/frameworks/openrouter/traceai_openrouter/__init__.py index ef653184a..a48a3b65b 100644 --- a/python/frameworks/openrouter/traceai_openrouter/__init__.py +++ b/python/frameworks/openrouter/traceai_openrouter/__init__.py @@ -238,7 +238,7 @@ async def close(self): self._finish(complete=False) -def _start(tracer: Any, operation: str, kwargs: dict[str, Any], config: TraceConfig): +def _start(tracer: Any, operation: str, kwargs: dict[str, Any], config: TraceConfig, attachments: Any = None): kind = FiSpanKindValues.EMBEDDING if operation == "embeddings" else FiSpanKindValues.LLM model = kwargs.get("model") attrs = {A.GEN_AI_PROVIDER_NAME: "openrouter", A.GEN_AI_OPERATION_NAME: operation} @@ -251,10 +251,22 @@ def _start(tracer: Any, operation: str, kwargs: dict[str, Any], config: TraceCon config.hide_inputs or config.hide_input_text or config.hide_input_messages or config.hide_input_images ) _input(span, operation, kwargs, capture_input) + if attachments is not None and operation == "chat": + _capture_chat_media(attachments, span, kwargs.get("messages"), config) return span -def _wrap(tracer: Any, operation: str, config: TraceConfig, is_async: bool): +def _capture_chat_media(attachments: Any, span: Any, messages: Any, config: TraceConfig) -> None: + """Chat ``image_url`` data URIs and ``input_audio`` through the shared manager; never raises.""" + try: + from fi_instrumentation.attachments.provider_media import ProviderMedia + + ProviderMedia(attachments, span, "openrouter", trace_config=config).compatible_messages(messages, "openai") + except Exception: + pass + + +def _wrap(tracer: Any, operation: str, config: TraceConfig, is_async: bool, attachments: Any = None): def complete(span, result): capture_output = not (config.hide_outputs or config.hide_output_text or config.hide_output_messages) if hasattr(result, "__anext__"): @@ -267,7 +279,7 @@ def complete(span, result): if is_async: async def wrapped(wrapped, instance, args, kwargs): - span = _start(tracer, operation, kwargs, config) + span = _start(tracer, operation, kwargs, config, attachments) try: with trace.use_span(span, end_on_exit=False, record_exception=False, set_status_on_exception=False): result = await wrapped(*args, **kwargs) @@ -277,7 +289,7 @@ async def wrapped(wrapped, instance, args, kwargs): raise else: def wrapped(wrapped, instance, args, kwargs): - span = _start(tracer, operation, kwargs, config) + span = _start(tracer, operation, kwargs, config, attachments) try: with trace.use_span(span, end_on_exit=False, record_exception=False, set_status_on_exception=False): result = wrapped(*args, **kwargs) @@ -288,6 +300,20 @@ def wrapped(wrapped, instance, args, kwargs): return wrapped +def _attachment_manager(kwargs: Any) -> Any: + """The shared manager passed as ``attachments`` (or auto's ``attachment_manager``), if it can admit. + + A missing, disabled or pre-attachments core leaves tracing exactly as before. + """ + if kwargs.get("attachments") is None and kwargs.get("attachment_manager") is None: + return None + try: + from fi_instrumentation.attachments.provider_media import manager_from_kwargs + except ImportError: + return None + return manager_from_kwargs(kwargs) + + class OpenRouterInstrumentor(BaseInstrumentor): # type: ignore[misc] """Instrument dedicated OpenRouter chat, embeddings and Responses calls.""" @@ -299,12 +325,13 @@ def _instrument(self, **kwargs: Any) -> None: if not isinstance(config, TraceConfig): raise TypeError("config must be a TraceConfig") tracer = FITracer(trace.get_tracer(__name__, __version__, kwargs.get("tracer_provider")), config) + attachments = _attachment_manager(kwargs) self._originals = [] for module, cls_name, operation, method in _ENDPOINTS: cls = getattr(import_module(module), cls_name) for name, is_async in ((method, False), (method + "_async", True)): original = cls.__dict__[name] - wrap_function_wrapper(module, cls_name + "." + name, _wrap(tracer, operation, config, is_async)) + wrap_function_wrapper(module, cls_name + "." + name, _wrap(tracer, operation, config, is_async, attachments)) self._originals.append((cls, name, original)) def _uninstrument(self, **kwargs: Any) -> None: @@ -314,3 +341,11 @@ def _uninstrument(self, **kwargs: Any) -> None: __all__ = ["OpenRouterInstrumentor", "__version__"] + +def __getattr__(name: str) -> Any: + # Lazy: the descriptor needs a fi-instrumentation-otel release with attachments. + if name == "openrouter_attachment_capabilities": + from fi_instrumentation.attachments.provider_media import capabilities + + return capabilities("traceai_openrouter") + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/python/frameworks/portkey/traceai_portkey/__init__.py b/python/frameworks/portkey/traceai_portkey/__init__.py index 6f0aa70fc..4b7594d9c 100644 --- a/python/frameworks/portkey/traceai_portkey/__init__.py +++ b/python/frameworks/portkey/traceai_portkey/__init__.py @@ -17,6 +17,20 @@ _instruments = ("portkey_ai >= 0.1.0",) +def _attachment_manager(kwargs: Any) -> Any: + """The shared manager passed as ``attachments`` (or auto's ``attachment_manager``), if it can admit. + + A missing, disabled or pre-attachments core leaves tracing exactly as before. + """ + if kwargs.get("attachments") is None and kwargs.get("attachment_manager") is None: + return None + try: + from fi_instrumentation.attachments.provider_media import manager_from_kwargs + except ImportError: + return None + return manager_from_kwargs(kwargs) + + class PortkeyInstrumentor(BaseInstrumentor): # type: ignore[misc] """An instrumentor for the Portkey AI framework.""" @@ -41,7 +55,10 @@ def _instrument(self, **kwargs: Any) -> None: wrap_function_wrapper( module="portkey_ai.api_resources.apis.chat_complete", name="Completions.create", - wrapper=_CompletionsWrapper(tracer=self._tracer), + # Chat only: the generation API carries no typed media. + wrapper=_CompletionsWrapper( + tracer=self._tracer, attachments=_attachment_manager(kwargs), config=config + ), ) wrap_function_wrapper( module="portkey_ai.api_resources.apis.generation", @@ -52,4 +69,12 @@ def _instrument(self, **kwargs: Any) -> None: def _uninstrument(self, **kwargs: Any) -> None: portkey_module = import_module("portkey_ai.api_resources.apis.chat_complete") if self._original_completions_create is not None: - portkey_module.Completions.create = self._original_completions_create \ No newline at end of file + portkey_module.Completions.create = self._original_completions_create + +def __getattr__(name: str) -> Any: + # Lazy: the descriptor needs a fi-instrumentation-otel release with attachments. + if name == "portkey_attachment_capabilities": + from fi_instrumentation.attachments.provider_media import capabilities + + return capabilities("traceai_portkey") + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/python/frameworks/portkey/traceai_portkey/_wrappers.py b/python/frameworks/portkey/traceai_portkey/_wrappers.py index 941c02814..d608bfa8d 100644 --- a/python/frameworks/portkey/traceai_portkey/_wrappers.py +++ b/python/frameworks/portkey/traceai_portkey/_wrappers.py @@ -44,10 +44,35 @@ def _flatten(mapping: Mapping[str, Any]) -> Iterator[Tuple[str, AttributeValue]] class _WithTracer(ABC): """Base class for wrappers that use a tracer.""" - def __init__(self, tracer: trace_api.Tracer, *args: Any, **kwargs: Any) -> None: + _media_dialect = "openai" + + def __init__( + self, + tracer: trace_api.Tracer, + *args: Any, + attachments: Any = None, + config: Any = None, + **kwargs: Any, + ) -> None: self._tracer = tracer + # Provider-owned shared attachment manager, or None (no capture, no imports). + self._attachments = attachments + self._config = config super().__init__(*args, **kwargs) + def _capture_chat_media(self, span: Any, messages: Any) -> None: + """Typed chat media (data-URI images) through the shared attachment manager; never raises.""" + if self._attachments is None: + return + try: + from fi_instrumentation.attachments.provider_media import ProviderMedia + + ProviderMedia( + self._attachments, span, "portkey", trace_config=self._config + ).compatible_messages(messages, self._media_dialect) + except Exception: + logger.debug("Attachment capture unavailable", exc_info=True) + @contextmanager def _start_as_current_span( self, @@ -120,6 +145,7 @@ def __call__( request_parameters ), ) as span: + self._capture_chat_media(span._span, kwargs.get("messages")) try: response = wrapped(*args, **kwargs) except Exception as exception: diff --git a/python/frameworks/together/traceai_together/__init__.py b/python/frameworks/together/traceai_together/__init__.py index 7fca7fb8f..c2a2b284f 100644 --- a/python/frameworks/together/traceai_together/__init__.py +++ b/python/frameworks/together/traceai_together/__init__.py @@ -32,6 +32,20 @@ _instruments = ("together >= 1.0.0",) +def _attachment_manager(kwargs: Any) -> Any: + """The shared manager passed as ``attachments`` (or auto's ``attachment_manager``), if it can admit. + + A missing, disabled or pre-attachments core leaves tracing exactly as before. + """ + if kwargs.get("attachments") is None and kwargs.get("attachment_manager") is None: + return None + try: + from fi_instrumentation.attachments.provider_media import manager_from_kwargs + except ImportError: + return None + return manager_from_kwargs(kwargs) + + class TogetherInstrumentor(BaseInstrumentor): """OpenTelemetry instrumentor for Together AI.""" @@ -61,6 +75,7 @@ def _instrument(self, **kwargs: Any) -> None: trace_api.get_tracer(__name__, __version__, tracer_provider), config=config, ) + media = dict(attachments=_attachment_manager(kwargs), config=config) # Import together client together_module = import_module("together") @@ -88,7 +103,7 @@ def _instrument(self, **kwargs: Any) -> None: wrap_function_wrapper( module="together.resources.chat", name="ChatCompletions.create", - wrapper=_ChatCompletionsWrapper(tracer=self._tracer), + wrapper=_ChatCompletionsWrapper(tracer=self._tracer, **media), ) except (ImportError, AttributeError) as e: logger.debug(f"Could not wrap chat.completions.create: {e}") @@ -130,7 +145,7 @@ def _instrument(self, **kwargs: Any) -> None: wrap_function_wrapper( module="together.resources.chat", name="AsyncChatCompletions.create", - wrapper=_AsyncChatCompletionsWrapper(tracer=self._tracer), + wrapper=_AsyncChatCompletionsWrapper(tracer=self._tracer, **media), ) except (ImportError, AttributeError) as e: logger.debug(f"Could not wrap async chat.completions.create: {e}") @@ -222,3 +237,12 @@ def _uninstrument(self, **kwargs: Any) -> None: except Exception as e: logger.exception(f"Error uninstrumenting Together client: {e}") + + +def __getattr__(name: str) -> Any: + # Lazy: the descriptor needs a fi-instrumentation-otel release with attachments. + if name == "together_attachment_capabilities": + from fi_instrumentation.attachments.provider_media import capabilities + + return capabilities("traceai_together") + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/python/frameworks/together/traceai_together/_wrappers.py b/python/frameworks/together/traceai_together/_wrappers.py index 6e71c5e8f..660a2ebae 100644 --- a/python/frameworks/together/traceai_together/_wrappers.py +++ b/python/frameworks/together/traceai_together/_wrappers.py @@ -32,9 +32,34 @@ class _WithTracer(ABC): """Base class for wrappers that need a tracer.""" - def __init__(self, tracer: trace_api.Tracer, *args: Any, **kwargs: Any) -> None: + _media_dialect = "image" + + def __init__( + self, + tracer: trace_api.Tracer, + *args: Any, + attachments: Any = None, + config: Any = None, + **kwargs: Any, + ) -> None: super().__init__(*args, **kwargs) self._tracer = tracer + # Provider-owned shared attachment manager, or None (no capture, no imports). + self._attachments = attachments + self._config = config + + def _capture_chat_media(self, span: Any, messages: Any) -> None: + """Typed chat media (data-URI images) through the shared attachment manager; never raises.""" + if self._attachments is None: + return + try: + from fi_instrumentation.attachments.provider_media import ProviderMedia + + ProviderMedia( + self._attachments, span, "together", trace_config=self._config + ).compatible_messages(messages, self._media_dialect) + except Exception: + logger.debug("Attachment capture unavailable", exc_info=True) @contextmanager def _start_as_current_span( @@ -97,6 +122,7 @@ def __call__( context_attributes=get_attributes_from_context(), extra_attributes=self._request_extractor.get_extra_attributes_from_request(request_parameters), ) as span: + self._capture_chat_media(span._span, kwargs.get("messages")) span.set_attribute(SpanAttributes.INPUT_VALUE, safe_json_dumps(request_parameters)) try: @@ -204,6 +230,7 @@ async def __call__( context_attributes=get_attributes_from_context(), extra_attributes=self._request_extractor.get_extra_attributes_from_request(request_parameters), ) as span: + self._capture_chat_media(span._span, kwargs.get("messages")) span.set_attribute(SpanAttributes.INPUT_VALUE, safe_json_dumps(request_parameters)) try: diff --git a/python/frameworks/vertexai/traceai_vertexai/__init__.py b/python/frameworks/vertexai/traceai_vertexai/__init__.py index e35531974..34aabfa1d 100644 --- a/python/frameworks/vertexai/traceai_vertexai/__init__.py +++ b/python/frameworks/vertexai/traceai_vertexai/__init__.py @@ -21,6 +21,20 @@ logger.addHandler(logging.NullHandler()) +def _attachment_manager(kwargs: Any) -> Any: + """The shared manager passed as ``attachments`` (or auto's ``attachment_manager``), if it can admit. + + A missing, disabled or pre-attachments core leaves tracing exactly as before. + """ + if kwargs.get("attachments") is None and kwargs.get("attachment_manager") is None: + return None + try: + from fi_instrumentation.attachments.provider_media import manager_from_kwargs + except ImportError: + return None + return manager_from_kwargs(kwargs) + + class VertexAIInstrumentor(BaseInstrumentor): # type: ignore """ An instrumentor for `google-cloud-aiplatform` or the `vertexai` SDK @@ -46,11 +60,12 @@ def _instrument(self, **kwargs: Any) -> None: import google.api_core.gapic_v1 as gapic from traceai_vertexai._wrapper import _Wrapper + media = dict(attachments=_attachment_manager(kwargs), config=config) for method in (gapic.method.wrap_method, gapic.method_async.wrap_method): wrap_function_wrapper( module=method.__module__, name=method.__name__, - wrapper=lambda f, _, args, kwargs: _Wrapper(tracer)(f(*args, **kwargs)), + wrapper=lambda f, _, args, kwargs: _Wrapper(tracer, **media)(f(*args, **kwargs)), ) if Protect is not None: self._original_protect = Protect.protect @@ -69,3 +84,12 @@ def _uninstrument(self, **kwargs: Any) -> None: for module in (gapic.method, gapic.method_async): if wrapped := getattr(module.wrap_method, "__wrapped__", None): setattr(module, "wrap_method", wrapped) + + +def __getattr__(name: str) -> Any: + # Lazy: the descriptor needs a fi-instrumentation-otel release with attachments. + if name == "vertexai_attachment_capabilities": + from fi_instrumentation.attachments.provider_media import capabilities + + return capabilities("traceai_vertexai") + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/python/frameworks/vertexai/traceai_vertexai/_wrapper.py b/python/frameworks/vertexai/traceai_vertexai/_wrapper.py index c4dfcc062..8c26de71a 100644 --- a/python/frameworks/vertexai/traceai_vertexai/_wrapper.py +++ b/python/frameworks/vertexai/traceai_vertexai/_wrapper.py @@ -112,9 +112,19 @@ class _Wrapper: variable `_instrumentation_status._IS_INSTRUMENTED` to False or True, respectively. """ - def __init__(self, tracer: trace_api.Tracer, *args: Any, **kwargs: Any) -> None: + def __init__( + self, + tracer: trace_api.Tracer, + *args: Any, + attachments: Any = None, + config: Any = None, + **kwargs: Any, + ) -> None: super().__init__(*args, **kwargs) self._tracer = tracer + # Provider-owned shared attachment manager, or None (no capture, no imports). + self._attachments = attachments + self._config = config @wrapt.decorator # type: ignore[misc] def __call__( @@ -160,6 +170,7 @@ def __call__( _update_span(request, span) except BaseException: pass + media = _media_input(self._attachments, self._config, span, request) try: with use_span(span, False, False, False): result = wrapped(*args, **kwargs) @@ -169,13 +180,44 @@ def __call__( span.end() raise if isinstance(result, Awaitable): - return _proxy(result, _CallbackForAwaitable(request, span), _use_span(span)) + return _proxy(result, _CallbackForAwaitable(request, span, media), _use_span(span)) if isinstance(result, (Iterable, AsyncIterable)): return _proxy(result, _CallbackForIterable(request, span), _use_span(span)) + _media_output(media, request, result) _finish(span, result) return result +def _media_input(attachments: Any, config: Any, span: Span, request: Any) -> Any: + """Typed inline_data Parts of a GenerateContentRequest; returns the capture for the response.""" + if attachments is None: + return None + try: + from fi_instrumentation.attachments.provider_media import ProviderMedia + + media = ProviderMedia(attachments, span, "vertexai", trace_config=config) + if not media.usable: + return None + if isinstance(request, (v1.GenerateContentRequest, v1beta1.GenerateContentRequest)): + media.gemini_contents(list(request.contents)) + return media + except Exception: + return None + + +def _media_output(media: Any, request: Any, result: Any) -> None: + """Completed responses only: Gemini candidate inline_data and Imagen predictions.""" + if media is None: + return + try: + if isinstance(result, (v1.GenerateContentResponse, v1beta1.GenerateContentResponse)): + media.gemini_response(result) + elif isinstance(result, v1.PredictResponse) and isinstance(request, v1.PredictRequest): + media.imagen_predictions(request.endpoint, result.predictions) + except Exception: + pass + + class _CallbackForAwaitable: """ Callback function for the proxy object when wrapping an Awaitable. This is how we get notified @@ -183,9 +225,10 @@ class _CallbackForAwaitable: Awaitable or when an exception is raised. """ - def __init__(self, request: proto.Message, span: Span) -> None: + def __init__(self, request: proto.Message, span: Span, media: Any = None) -> None: self._request = request self._span = span + self._media = media def __call__(self, obj: _AnyT) -> _AnyT: request, span = self._request, self._span @@ -194,6 +237,8 @@ def __call__(self, obj: _AnyT) -> _AnyT: _AnyT, _proxy(obj, _CallbackForIterable(request, span), _use_span(span)) ) else: + if not isinstance(obj, BaseException): + _media_output(self._media, request, obj) _finish(span, obj) return obj diff --git a/python/frameworks/xai/traceai_xai/__init__.py b/python/frameworks/xai/traceai_xai/__init__.py index 45820a151..952b112ed 100644 --- a/python/frameworks/xai/traceai_xai/__init__.py +++ b/python/frameworks/xai/traceai_xai/__init__.py @@ -58,6 +58,20 @@ ] +def _attachment_manager(kwargs: Any) -> Any: + """The shared manager passed as ``attachments`` (or auto's ``attachment_manager``), if it can admit. + + A missing, disabled or pre-attachments core leaves tracing exactly as before. + """ + if kwargs.get("attachments") is None and kwargs.get("attachment_manager") is None: + return None + try: + from fi_instrumentation.attachments.provider_media import manager_from_kwargs + except ImportError: + return None + return manager_from_kwargs(kwargs) + + class XAIInstrumentor(BaseInstrumentor): """An instrumentor for xAI (Grok) (OpenAI-compatible API).""" @@ -98,7 +112,7 @@ def _instrument(self, **kwargs: Any) -> None: wrap_function_wrapper( module="openai.resources.chat", name="Completions.create", - wrapper=_CompletionsWrapper(tracer=self._tracer), + wrapper=_CompletionsWrapper(tracer=self._tracer, attachments=_attachment_manager(kwargs), config=config), ) # Wrap asynchronous completions @@ -106,7 +120,9 @@ def _instrument(self, **kwargs: Any) -> None: wrap_function_wrapper( module="openai.resources.chat", name="AsyncCompletions.create", - wrapper=_AsyncCompletionsWrapper(tracer=self._tracer), + wrapper=_AsyncCompletionsWrapper( + tracer=self._tracer, attachments=_attachment_manager(kwargs), config=config + ), ) def _uninstrument(self, **kwargs: Any) -> None: @@ -118,3 +134,12 @@ def _uninstrument(self, **kwargs: Any) -> None: openai_module.AsyncCompletions.create = self._original_async_completions_create except ImportError: pass + + +def __getattr__(name: str) -> Any: + # Lazy: the descriptor needs a fi-instrumentation-otel release with attachments. + if name == "xai_attachment_capabilities": + from fi_instrumentation.attachments.provider_media import capabilities + + return capabilities("traceai_xai") + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/python/frameworks/xai/traceai_xai/_wrappers.py b/python/frameworks/xai/traceai_xai/_wrappers.py index 5453706ae..adfd97337 100644 --- a/python/frameworks/xai/traceai_xai/_wrappers.py +++ b/python/frameworks/xai/traceai_xai/_wrappers.py @@ -21,9 +21,34 @@ class _WithTracer(ABC): """Base class for wrappers that need a tracer.""" - def __init__(self, tracer: trace_api.Tracer, *args: Any, **kwargs: Any) -> None: + _media_dialect = "image" + + def __init__( + self, + tracer: trace_api.Tracer, + *args: Any, + attachments: Any = None, + config: Any = None, + **kwargs: Any, + ) -> None: super().__init__(*args, **kwargs) self._tracer = tracer + # Provider-owned shared attachment manager, or None (no capture, no imports). + self._attachments = attachments + self._config = config + + def _capture_chat_media(self, span: Any, messages: Any) -> None: + """Typed chat media (data-URI images) through the shared attachment manager; never raises.""" + if self._attachments is None: + return + try: + from fi_instrumentation.attachments.provider_media import ProviderMedia + + ProviderMedia( + self._attachments, span, "xai", trace_config=self._config + ).compatible_messages(messages, self._media_dialect) + except Exception: + logger.debug("Attachment capture unavailable", exc_info=True) @contextmanager def _start_span( @@ -217,6 +242,7 @@ def __call__( input_attrs = _get_input_attributes(model, messages, **extra_kwargs) with self._start_span("xAI Chat Completions", input_attrs) as span: + self._capture_chat_media(span, messages) try: response = wrapped(*args, **kwargs) @@ -288,6 +314,7 @@ async def __call__( input_attrs = _get_input_attributes(model, messages, **extra_kwargs) with self._start_span("xAI Chat Completions", input_attrs) as span: + self._capture_chat_media(span, messages) try: response = await wrapped(*args, **kwargs) From 620ec552344f9db46dfe17c395f8d4614dcb6590 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek <8347993+nik13@users.noreply.github.com> Date: Wed, 11 Nov 2026 19:33:31 -0800 Subject: [PATCH 03/15] test(python): provider media through real SDKs to a loopback F01 server Co-Authored-By: Claude Opus 5.5 --- .../attachments_compat_providers/conftest.py | 214 ++++++++++++ .../test_bedrock_litellm_attachments.py | 300 ++++++++++++++++ .../test_compat_attachments.py | 268 ++++++++++++++ .../test_google_attachments.py | 201 +++++++++++ .../test_provider_media.py | 327 ++++++++++++++++++ 5 files changed, 1310 insertions(+) create mode 100644 python/tests/attachments_compat_providers/conftest.py create mode 100644 python/tests/attachments_compat_providers/test_bedrock_litellm_attachments.py create mode 100644 python/tests/attachments_compat_providers/test_compat_attachments.py create mode 100644 python/tests/attachments_compat_providers/test_google_attachments.py create mode 100644 python/tests/attachments_compat_providers/test_provider_media.py diff --git a/python/tests/attachments_compat_providers/conftest.py b/python/tests/attachments_compat_providers/conftest.py new file mode 100644 index 000000000..138958f44 --- /dev/null +++ b/python/tests/attachments_compat_providers/conftest.py @@ -0,0 +1,214 @@ +"""Loopback harness for provider-media tests: synthetic data only. + +Reuses the F01 server, span exporter and media builders of +``tests/attachments_providers/conftest.py`` (loaded by path, so both suites +share one harness), and adds a plain-HTTP provider that answers like Gemini, +Vertex (REST), Bedrock Runtime, Ollama and OpenAI-compatible chat APIs, so the +REAL installed SDKs run end to end with outbound network disabled. +""" + +import base64 +import importlib.util +import json +import os +import sys +import threading +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from urllib.parse import unquote + +import pytest + +# LiteLLM fetches its model cost map at import unless told to use the bundled copy. +os.environ.setdefault("LITELLM_LOCAL_MODEL_COST_MAP", "True") +os.environ.setdefault("AWS_EC2_METADATA_DISABLED", "true") + +pytest.importorskip("fi_instrumentation.attachments.provider_media") + +_spec = importlib.util.spec_from_file_location( + "_attachment_provider_harness", Path(__file__).parent.parent / "attachments_providers" / "conftest.py") +harness = importlib.util.module_from_spec(_spec) +sys.modules["_attachment_provider_harness"] = harness +_spec.loader.exec_module(harness) + +# Fixtures (pytest finds them by name in this module) and helpers. +f01 = harness.f01 +spans = harness.spans +clean_env = harness.clean_env +png, wav, PDF, b64, refs, flush, wait_uploads = (harness.png, harness.wav, harness.PDF, harness.b64, harness.refs, + harness.flush, harness.wait_uploads) + + +def jpeg(seed=0) -> bytes: + """Leading bytes of a JPEG (SOI + APP0); the loopback F01 server does not decode it.""" + return b"\xff\xd8\xff\xe0\x00\x10JFIF\x00\x01\x01\x00\x00\x01\x00\x01\x00\x00" + bytes([seed % 256]) * 32 + b"\xff\xd9" + + +MP4 = b"\x00\x00\x00\x18ftypisom\x00\x00\x02\x00isomiso2" + b"\x00" * 24 + + +def data_uri(data: bytes, mime="image/png") -> str: + return "data:%s;base64,%s" % (mime, b64(data)) + + +def uploads_by_purpose(f01_server): + found = {} + for upload in f01_server.uploads: + found.setdefault(upload.get("purpose"), []).append(upload) + return found + + +def span_named(exporter, *names): + matches = [span for span in exporter.get_finished_spans() if span.name in names] + assert matches, [span.name for span in exporter.get_finished_spans()] + return matches[-1] + + +def all_refs(exporter): + found = [] + for span in exporter.get_finished_spans(): + for ref in refs(span): + found.append((span.name, ref)) + return found + + +class MediaProviderState: + def __init__(self): + self.requests = [] + self.image = png(seed=21) + self.generated = png(seed=22) + + +def _openai_chat(body): + if body.get("stream"): + chunks = [{"id": "c1", "object": "chat.completion.chunk", "created": 1, "model": "m", + "choices": [{"index": 0, "delta": {"role": "assistant", "content": "ok"}, "finish_reason": None}]}, + {"id": "c1", "object": "chat.completion.chunk", "created": 1, "model": "m", + "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}}] + return "text/event-stream", "".join("data: %s\n\n" % json.dumps(c) for c in chunks).encode() + b"data: [DONE]\n\n" + return "application/json", json.dumps({ + "id": "c1", "object": "chat.completion", "created": 1, "model": body.get("model", "m"), + "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}}).encode() + + +def _gemini(state, stream): + response = {"candidates": [{"content": {"role": "model", "parts": [ + {"text": "ok"}, {"inlineData": {"mimeType": "image/png", "data": b64(state.generated)}}]}, + "finishReason": "STOP", "index": 0}], + "usageMetadata": {"promptTokenCount": 1, "candidatesTokenCount": 1, "totalTokenCount": 2}, + "modelVersion": "synthetic"} + if stream: + return "text/event-stream", ("data: %s\r\n\r\n" % json.dumps(response)).encode() + return "application/json", json.dumps(response).encode() + + +def _bedrock(state, path, body): + model = unquote(path.split("/model/", 1)[1].rsplit("/", 1)[0]) + if path.endswith("/converse"): + return "application/json", json.dumps({ + "output": {"message": {"role": "assistant", "content": [{"text": "ok"}]}}, "stopReason": "end_turn", + "usage": {"inputTokens": 1, "outputTokens": 1, "totalTokens": 2}, "metrics": {"latencyMs": 1}}).encode() + if "titan-image" in model or "nova-canvas" in model: + return "application/json", json.dumps({"images": [b64(state.generated)], "error": None}).encode() + if "anthropic" in model: + return "application/json", json.dumps({ + "id": "msg_1", "type": "message", "role": "assistant", "model": model, + "content": [{"type": "text", "text": "ok"}], "stop_reason": "end_turn", "stop_sequence": None, + "usage": {"input_tokens": 1, "output_tokens": 1}}).encode() + return "application/json", json.dumps({ + "output": {"message": {"role": "assistant", "content": [{"text": "ok"}]}}, "stopReason": "end_turn", + "usage": {"inputTokens": 1, "outputTokens": 1, "totalTokens": 2}}).encode() + + +def _ollama(path, body): + done = {"model": body.get("model", "m"), "created_at": "2025-01-01T00:00:00Z", "done": True, + "done_reason": "stop", "total_duration": 1, "load_duration": 1, "prompt_eval_count": 1, + "prompt_eval_duration": 1, "eval_count": 1, "eval_duration": 1} + if path.endswith("/api/chat"): + done["message"] = {"role": "assistant", "content": "ok"} + else: + done["response"] = "ok" + if body.get("stream", True): + return "application/x-ndjson", (json.dumps(done) + "\n").encode() + return "application/json", json.dumps(done).encode() + + +@pytest.fixture +def media_provider(): + state = MediaProviderState() + + class Handler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def log_message(self, *args): + pass + + def send(self, content_type, data, status=200): + self.send_response(status) + self.send_header("Content-Type", content_type) + self.send_header("Content-Length", str(len(data))) + self.end_headers() + self.wfile.write(data) + + def body(self): + if self.headers.get("Transfer-Encoding", "").lower() == "chunked": + parts = [] + while True: + size = int(self.rfile.readline().split(b";")[0].strip() or b"0", 16) + if not size: + self.rfile.readline() + return b"".join(parts) + parts.append(self.rfile.read(size)) + self.rfile.readline() + return self.rfile.read(int(self.headers.get("Content-Length") or 0)) + + def do_GET(self): + self.send("application/json", b"{}") + + def do_HEAD(self): + self.send_response(200) + self.send_header("Content-Length", "0") + self.end_headers() + + def do_POST(self): + raw = self.body() + try: + body = json.loads(raw) if raw else {} + except ValueError: + body = {} + state.requests.append((self.path, body, raw)) + path = self.path.split("?")[0] + if path.endswith(":generateContent"): + self.send(*_gemini(state, False)) + elif path.endswith(":streamGenerateContent"): + self.send(*_gemini(state, True)) + elif path.endswith(":predict"): + self.send("application/json", json.dumps({"predictions": [ + {"bytesBase64Encoded": b64(state.generated), "mimeType": "image/png"}], + "deployedModelId": "synthetic"}).encode()) + elif "/model/" in path and path.endswith(("/converse", "/invoke")): + self.send(*_bedrock(state, path, body)) + elif path.endswith(("/api/chat", "/api/generate")): + self.send(*_ollama(path, body)) + elif path.endswith("/chat/completions"): + self.send(*_openai_chat(body)) + elif path.endswith("/images/generations"): + self.send("application/json", json.dumps({"created": 1, "data": [ + {"b64_json": b64(state.generated)}, {"url": "https://images.invalid/x.png"}]}).encode()) + else: + self.send("application/json", b"{}", status=404) + + server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + state.url = "http://127.0.0.1:%d" % server.server_port + yield state + server.shutdown() + server.server_close() + thread.join(2) + + +def decoded(value: str) -> bytes: + return base64.b64decode(value) diff --git a/python/tests/attachments_compat_providers/test_bedrock_litellm_attachments.py b/python/tests/attachments_compat_providers/test_bedrock_litellm_attachments.py new file mode 100644 index 000000000..836fd3d6f --- /dev/null +++ b/python/tests/attachments_compat_providers/test_bedrock_litellm_attachments.py @@ -0,0 +1,300 @@ +"""Real boto3 (Bedrock Runtime) and LiteLLM -> traceAI instrumentors -> loopback F01.""" + +import asyncio +import base64 +import json + +import pytest + +from conftest import MP4, PDF, b64, data_uri, flush, png, refs, span_named, uploads_by_purpose # noqa: E402 +from fi_instrumentation import TraceConfig # noqa: E402 + +IMAGE = png(seed=51) +SHOT_1 = png(seed=52) +SHOT_2 = png(seed=53) +TOOL_IMAGE = png(seed=54) +CLAUDE = "us.anthropic.claude-sonnet-4-20250514-v1:0" +NOVA = "amazon.nova-lite-v1:0" + + +# ------------------------------------------------------------------ Bedrock +@pytest.fixture +def bedrock(spans, media_provider): + pytest.importorskip("boto3") + from traceai_bedrock import BedrockInstrumentor + + tracer_provider, exporter = spans + state = {} + + def install(manager, config=None): + import boto3 + + BedrockInstrumentor().instrument(tracer_provider=tracer_provider, attachments=manager, + config=config or TraceConfig()) + state["on"] = True + # Created AFTER instrument(): the instrumentor wraps client creation. + client = boto3.client("bedrock-runtime", region_name="us-east-1", endpoint_url=media_provider.url, + aws_access_key_id="AKIDSYNTHETIC", aws_secret_access_key="synthetic") + return client, exporter + + yield install + if state.get("on"): + BedrockInstrumentor().uninstrument() + + +def converse_messages(computer=True): + return [ + {"role": "user", "content": [ + {"text": "open the page"}, + {"image": {"format": "png", "source": {"bytes": IMAGE}}}, + {"document": {"format": "pdf", "name": "brief", "source": {"bytes": PDF}}}, + {"image": {"format": "png", "source": {"s3Location": {"uri": "s3://bucket/never-read.png"}}}}, + ]}, + {"role": "assistant", "content": [ + {"toolUse": {"toolUseId": "tooluse_1", "name": "computer", "input": {"action": "screenshot"}}}, + {"toolUse": {"toolUseId": "tooluse_2", "name": "chart", "input": {}}}, + ]}, + {"role": "user", "content": [ + {"toolResult": {"toolUseId": "tooluse_1", "content": [ + {"image": {"format": "png", "source": {"bytes": SHOT_1}}}]}}, + {"toolResult": {"toolUseId": "tooluse_2", "content": [ + {"text": "chart"}, {"image": {"format": "png", "source": {"bytes": TOOL_IMAGE}}}]}}, + ]}, + ] + + +def computer_fields(): + return {"tools": [{"type": "computer_20250124", "name": "computer", "display_width_px": 1024, + "display_height_px": 768}], "anthropic_beta": ["computer-use-2025-01-24"]} + + +def test_converse_vision_document_and_computer_screenshots(f01, bedrock): + manager = f01.manager() + client, exporter = bedrock(manager) + client.converse(modelId=CLAUDE, messages=converse_messages(), + toolConfig={"tools": [{"toolSpec": {"name": "chart", "inputSchema": {"json": {"type": "object"}}}}]}, + additionalModelRequestFields=computer_fields()) + flush(manager) + assert sorted(f01.payloads) == sorted([IMAGE, PDF, SHOT_1, TOOL_IMAGE]) + by_purpose = uploads_by_purpose(f01) + assert [u["tool_call_id"] for u in by_purpose["computer_screenshot"]] == ["tooluse_1"] + assert [u["tool_call_id"] for u in by_purpose["tool_result"]] == ["tooluse_2"] + assert len(by_purpose["document"]) == 1 and len(by_purpose["model_input"]) == 1 + span = span_named(exporter, "bedrock.converse") + shots = [r for r in refs(span) if r["purpose"] == "computer_screenshot"] + assert [(r["message_index"], r["part_index"], r["role"]) for r in shots] == [(2, 0, "tool")] + manager.shutdown(5) + + +def test_converse_screenshot_needs_declared_computer_tool(f01, bedrock): + manager = f01.manager() + client, _ = bedrock(manager) + client.converse(modelId=CLAUDE, messages=converse_messages()) + flush(manager) + purposes = sorted(u["purpose"] for u in f01.uploads) + assert "computer_screenshot" not in purposes and purposes.count("tool_result") == 2 + manager.shutdown(5) + + +def test_converse_unknown_model_family_is_never_read(f01, bedrock): + manager = f01.manager() + client, _ = bedrock(manager) + client.converse(modelId="meta.llama3-2-90b-instruct-v1:0", messages=converse_messages()) + flush(manager) + assert f01.payloads == [] + manager.shutdown(5) + + +def test_nova_converse_video_block(f01, bedrock): + manager = f01.manager() + client, _ = bedrock(manager) + client.converse(modelId=NOVA, messages=[{"role": "user", "content": [ + {"video": {"format": "mp4", "source": {"bytes": MP4}}}, {"text": "summarise"}]}]) + client.converse(modelId=CLAUDE, messages=[{"role": "user", "content": [ + {"video": {"format": "mp4", "source": {"bytes": MP4}}}, {"text": "summarise"}]}]) + flush(manager) + assert f01.payloads == [MP4] # video is qualified for Nova only + manager.shutdown(5) + + +def test_converse_hide_input_images_keeps_documents(f01, bedrock): + manager = f01.manager() + client, _ = bedrock(manager, TraceConfig(hide_input_images=True)) + client.converse(modelId=CLAUDE, messages=converse_messages(), additionalModelRequestFields=computer_fields()) + flush(manager) + assert f01.payloads == [PDF] + manager.shutdown(5) + + +def test_invoke_model_anthropic_body(f01, bedrock): + manager = f01.manager() + client, _ = bedrock(manager) + body = {"anthropic_version": "bedrock-2023-05-31", "max_tokens": 8, + "tools": [{"type": "computer_20250124", "name": "computer", "display_width_px": 800, + "display_height_px": 600}], + "messages": [ + {"role": "user", "content": [ + {"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": b64(IMAGE)}}]}, + {"role": "assistant", "content": [{"type": "tool_use", "id": "toolu_1", "name": "computer", + "input": {"action": "screenshot"}}]}, + {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "toolu_1", "content": [ + {"type": "image", "source": {"type": "base64", "media_type": "image/png", + "data": b64(SHOT_2)}}]}]}, + ]} + response = client.invoke_model(modelId="anthropic.claude-3-5-sonnet-20240620-v1:0", body=json.dumps(body)) + assert json.loads(response["body"].read())["content"][0]["text"] == "ok" + flush(manager) + assert sorted(f01.payloads) == sorted([IMAGE, SHOT_2]) + assert sorted(u["purpose"] for u in f01.uploads) == ["computer_screenshot", "model_input"] + manager.shutdown(5) + + +def test_invoke_model_nova_body(f01, bedrock): + manager = f01.manager() + client, _ = bedrock(manager) + body = {"schemaVersion": "messages-v1", "messages": [{"role": "user", "content": [ + {"image": {"format": "png", "source": {"bytes": b64(IMAGE)}}}, + {"document": {"format": "pdf", "name": "brief", "source": {"bytes": b64(PDF)}}}, + {"text": "describe"}]}]} + client.invoke_model(modelId=NOVA, body=json.dumps(body)) + flush(manager) + assert sorted(f01.payloads) == sorted([IMAGE, PDF]) + manager.shutdown(5) + + +@pytest.mark.parametrize("model", ["amazon.titan-image-generator-v2:0", "amazon.nova-canvas-v1:0"]) +def test_invoke_model_image_generation_both_directions(f01, bedrock, media_provider, model): + manager = f01.manager() + client, exporter = bedrock(manager) + body = {"taskType": "INPAINTING", "inPaintingParams": {"image": b64(IMAGE), "text": "add a hat", + "maskPrompt": "head"}, + "imageGenerationConfig": {"numberOfImages": 1}} + response = client.invoke_model(modelId=model, body=json.dumps(body)) + # The output came from the body the instrumentor already buffered; the caller still reads it whole. + assert json.loads(response["body"].read())["images"] == [b64(media_provider.generated)] + flush(manager) + assert sorted(f01.payloads) == sorted([IMAGE, media_provider.generated]) + directions = sorted((u["direction"], u["purpose"]) for u in f01.uploads) + assert directions == [("input", "model_input"), ("output", "model_output")] + span = span_named(exporter, "bedrock.invoke_model") + assert len(refs(span)) == 2 + manager.shutdown(5) + + +def test_invoke_model_unknown_body_is_never_scraped(f01, bedrock): + manager = f01.manager() + client, _ = bedrock(manager) + body = {"prompt": "x", "images": [b64(IMAGE)], "messages": [{"role": "user", "content": [ + {"image": {"format": "png", "source": {"bytes": b64(IMAGE)}}}]}]} + client.invoke_model(modelId="meta.llama3-2-90b-instruct-v1:0", body=json.dumps(body)) + flush(manager) + assert f01.payloads == [] + manager.shutdown(5) + + +# ------------------------------------------------------------------ LiteLLM +@pytest.fixture +def litellm_instrument(spans): + pytest.importorskip("litellm") + from traceai_litellm import LiteLLMInstrumentor + + tracer_provider, exporter = spans + state = {} + + def install(manager, config=None, with_openai=False): + LiteLLMInstrumentor().instrument(tracer_provider=tracer_provider, attachments=manager, + config=config or TraceConfig()) + state["litellm"] = True + if with_openai: + from traceai_openai import OpenAIInstrumentor + + OpenAIInstrumentor().instrument(tracer_provider=tracer_provider, attachments=manager) + state["openai"] = True + return exporter + + yield install + if state.get("openai"): + from traceai_openai import OpenAIInstrumentor + + OpenAIInstrumentor().uninstrument() + if state.get("litellm"): + LiteLLMInstrumentor().uninstrument() + + +def chat_messages(): + return [{"role": "user", "content": [ + {"type": "text", "text": "what is this"}, + {"type": "image_url", "image_url": {"url": data_uri(IMAGE)}}, + {"type": "image_url", "image_url": {"url": "https://example.invalid/x.png"}}, + ]}] + + +@pytest.mark.parametrize("mode", ["sync", "async", "stream"]) +def test_litellm_completion_input_media(f01, media_provider, litellm_instrument, mode): + import litellm + + manager = f01.manager() + exporter = litellm_instrument(manager) + options = dict(model="openai/gpt-synthetic", messages=chat_messages(), api_base=media_provider.url, + api_key="sk-synthetic") + if mode == "async": + asyncio.run(litellm.acompletion(**options)) + elif mode == "stream": + for _ in litellm.completion(stream=True, **options): + pass + else: + litellm.completion(**options) + flush(manager) + assert f01.payloads == [IMAGE] + span = span_named(exporter, "completion", "acompletion") + assert [r["purpose"] for r in refs(span)] == ["model_input"] + manager.shutdown(5) + + +def test_litellm_owns_media_under_nested_openai_instrumentor(f01, media_provider, litellm_instrument): + """LiteLLM marks the call as owned, so the nested OpenAI instrumentor uploads nothing more.""" + pytest.importorskip("traceai_openai") + import litellm + + manager = f01.manager() + exporter = litellm_instrument(manager, with_openai=True) + litellm.completion(model="openai/gpt-synthetic", messages=chat_messages(), api_base=media_provider.url, + api_key="sk-synthetic") + flush(manager) + assert f01.payloads == [IMAGE] + owners = [name for name, _ in ((s.name, r) for s in exporter.get_finished_spans() for r in refs(s))] + assert owners == ["completion"], owners + manager.shutdown(5) + + +@pytest.mark.parametrize("async_", [False, True]) +def test_litellm_image_generation_output(f01, media_provider, litellm_instrument, async_): + import litellm + + manager = f01.manager() + exporter = litellm_instrument(manager) + options = dict(model="openai/dall-e-3", prompt="a cat", api_base=media_provider.url, api_key="sk-synthetic", + response_format="b64_json") + if async_: + result = asyncio.run(litellm.aimage_generation(**options)) + else: + result = litellm.image_generation(**options) + assert base64.b64decode(result.data[0].b64_json) == media_provider.generated + flush(manager) + assert f01.payloads == [media_provider.generated] # the URL result is never fetched + assert f01.uploads[0]["direction"] == "output" + span = span_named(exporter, "image_generation", "aimage_generation") + assert len(refs(span)) == 1 + manager.shutdown(5) + + +def test_litellm_hide_input_images(f01, media_provider, litellm_instrument): + import litellm + + manager = f01.manager() + litellm_instrument(manager, TraceConfig(hide_input_images=True)) + litellm.completion(model="openai/gpt-synthetic", messages=chat_messages(), api_base=media_provider.url, + api_key="sk-synthetic") + flush(manager) + assert f01.payloads == [] + manager.shutdown(5) diff --git a/python/tests/attachments_compat_providers/test_compat_attachments.py b/python/tests/attachments_compat_providers/test_compat_attachments.py new file mode 100644 index 000000000..95f4a874f --- /dev/null +++ b/python/tests/attachments_compat_providers/test_compat_attachments.py @@ -0,0 +1,268 @@ +"""Real compatible-chat SDKs -> thin traceAI bindings -> loopback F01. + +One typed fixture per wrapper plus a capability-negative row; DeepSeek and +Cerebras have no qualified media surface and must upload nothing. +""" + +import asyncio + +import pytest + +from conftest import b64, data_uri, flush, png, refs, wav # noqa: E402 +from fi_instrumentation import TraceConfig # noqa: E402 + +IMAGE = png(seed=61) + + +def image_messages(url_form="object"): + image = data_uri(IMAGE) + return [{"role": "user", "content": [ + {"type": "text", "text": "what is this"}, + {"type": "image_url", "image_url": image if url_form == "string" else {"url": image}}, + {"type": "image_url", "image_url": {"url": "https://example.invalid/x.png"}}, + ]}] + + +def audio_messages(sound): + return [{"role": "user", "content": [ + {"type": "input_audio", "input_audio": {"data": b64(sound), "format": "wav"}}]}] + + +@pytest.fixture +def instrumented(spans): + tracer_provider, exporter = spans + installed = [] + + def install(instrumentor, manager, config=None): + instrumentor.instrument(tracer_provider=tracer_provider, attachments=manager, config=config or TraceConfig()) + installed.append(instrumentor) + return exporter + + yield install + for instrumentor in installed: + instrumentor.uninstrument() + + +def media_refs(exporter): + return [r for s in exporter.get_finished_spans() for r in refs(s)] + + +# ------------------------------------------------------------------ groq (image-only chat) +def test_groq_chat_images(f01, media_provider, instrumented): + groq = pytest.importorskip("groq") + from traceai_groq import GroqInstrumentor + + manager = f01.manager() + exporter = instrumented(GroqInstrumentor(), manager) + client = groq.Groq(api_key="gsk-synthetic", base_url=media_provider.url, max_retries=0) + client.chat.completions.create(model="m", messages=image_messages()) + client.chat.completions.create(model="m", messages=audio_messages(wav())) # Groq chat: no audio parts + asyncio.run(groq.AsyncGroq(api_key="gsk-synthetic", base_url=media_provider.url, max_retries=0) + .chat.completions.create(model="m", messages=image_messages())) + flush(manager) + assert f01.payloads == [IMAGE, IMAGE] + assert [r["purpose"] for r in media_refs(exporter)] == ["model_input", "model_input"] + manager.shutdown(5) + + +# ------------------------------------------------------------------ together +def test_together_chat_images(f01, media_provider, instrumented): + together = pytest.importorskip("together") + from traceai_together import TogetherInstrumentor + + manager = f01.manager() + exporter = instrumented(TogetherInstrumentor(), manager) + client = together.Together(api_key="synthetic", base_url=media_provider.url + "/v1") + client.chat.completions.create(model="m", messages=image_messages()) + flush(manager) + assert f01.payloads == [IMAGE] + assert len(media_refs(exporter)) == 1 + manager.shutdown(5) + + +# ------------------------------------------------------------------ OpenAI-SDK hosted routes +@pytest.mark.parametrize("package", ["fireworks", "xai"]) +def test_openai_sdk_routes_capture_images(f01, media_provider, instrumented, monkeypatch, package): + openai = pytest.importorskip("openai") + if package == "fireworks": + from traceai_fireworks import FireworksInstrumentor as Instrumentor + from traceai_fireworks import _wrappers + monkeypatch.setattr(_wrappers, "_is_fireworks_client", lambda instance: True) + if hasattr(_wrappers, "_shared_instrumentor_enabled"): + monkeypatch.setattr(_wrappers, "_shared_instrumentor_enabled", lambda: False) + else: + from traceai_xai import XAIInstrumentor as Instrumentor + from traceai_xai import _wrappers + monkeypatch.setattr(_wrappers, "_is_xai_client", lambda instance: True) + + manager = f01.manager() + exporter = instrumented(Instrumentor(), manager) + client = openai.OpenAI(api_key="sk-synthetic", base_url=media_provider.url + "/v1", max_retries=0) + client.chat.completions.create(model="m", messages=image_messages()) + client.chat.completions.create(model="m", messages=audio_messages(wav())) # image-only routes + flush(manager) + assert f01.payloads == [IMAGE] + assert len(media_refs(exporter)) == 1 + manager.shutdown(5) + + +@pytest.mark.parametrize("package", ["deepseek", "cerebras"]) +def test_routes_without_media_surface_capture_nothing(f01, media_provider, instrumented, monkeypatch, package): + if package == "deepseek": + openai = pytest.importorskip("openai") + import traceai_deepseek + from traceai_deepseek import DeepSeekInstrumentor as Instrumentor + + for name in ("_is_deepseek_client", "_is_deepseek"): + if hasattr(traceai_deepseek._wrappers, name): + monkeypatch.setattr(traceai_deepseek._wrappers, name, lambda *a, **k: True) + manager = f01.manager() + exporter = instrumented(Instrumentor(), manager) + client = openai.OpenAI(api_key="sk-synthetic", base_url=media_provider.url + "/v1", max_retries=0) + client.chat.completions.create(model="deepseek-chat", messages=image_messages()) + descriptor = traceai_deepseek.deepseek_attachment_capabilities + else: + sdk = pytest.importorskip("cerebras.cloud.sdk") + import traceai_cerebras + from traceai_cerebras import CerebrasInstrumentor as Instrumentor + + manager = f01.manager() + exporter = instrumented(Instrumentor(), manager) + client = sdk.Cerebras(api_key="synthetic", base_url=media_provider.url, max_retries=0, + warm_tcp_connection=False) + client.chat.completions.create(model="llama", messages=image_messages()) + descriptor = traceai_cerebras.cerebras_attachment_capabilities + flush(manager) + assert f01.payloads == [] and media_refs(exporter) == [] + assert descriptor["operations"] == () and descriptor["note"] + manager.shutdown(5) + + +# ------------------------------------------------------------------ mistralai (legacy 1.x and native 3.6+) +@pytest.mark.parametrize("async_", [False, True]) +def test_mistral_chat_images(f01, media_provider, instrumented, async_): + mistralai = pytest.importorskip("mistralai") + from traceai_mistralai import MistralAIInstrumentor + + manager = f01.manager() + exporter = instrumented(MistralAIInstrumentor(), manager) + Mistral = getattr(mistralai, "Mistral", None) + if Mistral is None: + from mistralai.client import Mistral + client = Mistral(api_key="synthetic", server_url=media_provider.url) + messages = image_messages("string") + [{"role": "user", "content": [ + {"type": "image_url", "image_url": {"url": data_uri(png(seed=62))}}]}] + if async_: + asyncio.run(client.chat.complete_async(model="pixtral-synthetic", messages=messages)) + else: + client.chat.complete(model="pixtral-synthetic", messages=messages) + flush(manager) + assert sorted(f01.payloads) == sorted([IMAGE, png(seed=62)]) + assert len(media_refs(exporter)) == 2 + manager.shutdown(5) + + +def test_mistral_stream_and_hide_input_images(f01, media_provider, instrumented): + mistralai = pytest.importorskip("mistralai") + from traceai_mistralai import MistralAIInstrumentor + + manager = f01.manager() + instrumented(MistralAIInstrumentor(), manager, TraceConfig(hide_input_images=True)) + Mistral = getattr(mistralai, "Mistral", None) + if Mistral is None: + from mistralai.client import Mistral + client = Mistral(api_key="synthetic", server_url=media_provider.url) + client.chat.complete(model="pixtral-synthetic", messages=image_messages("string")) + flush(manager) + assert f01.payloads == [] + manager.shutdown(5) + + +# ------------------------------------------------------------------ openrouter +def test_openrouter_chat_images_and_audio(f01, media_provider, instrumented): + openrouter = pytest.importorskip("openrouter") + from traceai_openrouter import OpenRouterInstrumentor + + manager = f01.manager() + exporter = instrumented(OpenRouterInstrumentor(), manager) + sound = wav(seed=8) + with openrouter.OpenRouter(api_key="sk-or-synthetic", server_url=media_provider.url) as client: + client.chat.send(model="m", messages=image_messages() + audio_messages(sound)) + flush(manager) + assert sorted(f01.payloads) == sorted([IMAGE, sound]) + assert len(media_refs(exporter)) == 2 + manager.shutdown(5) + + +# ------------------------------------------------------------------ portkey +def test_portkey_chat_images(f01, media_provider, instrumented): + portkey_ai = pytest.importorskip("portkey_ai") + from traceai_portkey import PortkeyInstrumentor + + manager = f01.manager() + exporter = instrumented(PortkeyInstrumentor(), manager) + client = portkey_ai.Portkey(api_key="synthetic", base_url=media_provider.url + "/v1", provider="openai", + Authorization="Bearer sk-synthetic") + client.chat.completions.create(model="m", messages=image_messages()) + flush(manager) + assert f01.payloads == [IMAGE] + assert len(media_refs(exporter)) == 1 + manager.shutdown(5) + + +# ------------------------------------------------------------------ ollama +def test_ollama_chat_and_generate_images(f01, media_provider, instrumented, tmp_path): + ollama = pytest.importorskip("ollama") + from traceai_ollama import OllamaInstrumentor + + manager = f01.manager() + exporter = instrumented(OllamaInstrumentor(), manager) + second = png(seed=63) + path = tmp_path / "local.png" + path.write_bytes(png(seed=64)) + client = ollama.Client(host=media_provider.url) + client.chat(model="llava", messages=[{"role": "user", "content": "describe", "images": [IMAGE]}], stream=False) + client.generate(model="llava", prompt="describe", images=[b64(second)], stream=False) + # A local path: the SDK reads it for its own request; the adapter never opens it or uploads it. + client.generate(model="llava", prompt="describe", images=[str(path)], stream=False) + flush(manager) + assert sorted(f01.payloads) == sorted([IMAGE, second]) + assert len(media_refs(exporter)) == 2 + manager.shutdown(5) + + +def test_ollama_async_chat(f01, media_provider, instrumented): + ollama = pytest.importorskip("ollama") + from traceai_ollama import OllamaInstrumentor + + manager = f01.manager() + instrumented(OllamaInstrumentor(), manager) + asyncio.run(ollama.AsyncClient(host=media_provider.url).chat( + model="llava", messages=[{"role": "user", "content": "describe", "images": [IMAGE]}], stream=False)) + flush(manager) + assert f01.payloads == [IMAGE] + manager.shutdown(5) + + +# ------------------------------------------------------------------ descriptors exported per package +@pytest.mark.parametrize("module,name", [ + ("traceai_google_genai", "google_genai_attachment_capabilities"), + ("traceai_vertexai", "vertexai_attachment_capabilities"), + ("traceai_bedrock", "bedrock_attachment_capabilities"), + ("traceai_litellm", "litellm_attachment_capabilities"), + ("traceai_together", "together_attachment_capabilities"), + ("traceai_fireworks", "fireworks_attachment_capabilities"), + ("traceai_xai", "xai_attachment_capabilities"), + ("traceai_mistralai", "mistralai_attachment_capabilities"), + ("traceai_openrouter", "openrouter_attachment_capabilities"), + ("traceai_ollama", "ollama_attachment_capabilities"), + ("traceai_portkey", "portkey_attachment_capabilities"), + ("traceai_deepseek", "deepseek_attachment_capabilities"), + ("traceai_cerebras", "cerebras_attachment_capabilities"), +]) +def test_package_descriptor(module, name): + package = pytest.importorskip(module) + descriptor = getattr(package, name) + assert descriptor["schema_version"] == 1 and isinstance(descriptor["operations"], tuple) + with pytest.raises(AttributeError): + getattr(package, "no_such_attachment_capabilities") diff --git a/python/tests/attachments_compat_providers/test_google_attachments.py b/python/tests/attachments_compat_providers/test_google_attachments.py new file mode 100644 index 000000000..bbb544ddc --- /dev/null +++ b/python/tests/attachments_compat_providers/test_google_attachments.py @@ -0,0 +1,201 @@ +"""Real google-genai and Vertex (gapic REST) SDKs -> traceAI instrumentors -> loopback F01.""" + +import asyncio +import json + +import pytest + +from conftest import MP4, PDF, b64, flush, png, refs, span_named, uploads_by_purpose, wav # noqa: E402 +from fi_instrumentation import TraceConfig # noqa: E402 + +IMAGE = png(seed=41) + + +# ------------------------------------------------------------------ google-genai +@pytest.fixture +def genai_instrument(spans): + pytest.importorskip("google.genai") + from traceai_google_genai import GoogleGenAIInstrumentor + + tracer_provider, exporter = spans + state = {} + + def install(manager, config=None): + GoogleGenAIInstrumentor().instrument(tracer_provider=tracer_provider, attachments=manager, + config=config or TraceConfig()) + state["on"] = True + return exporter + + yield install + if state.get("on"): + GoogleGenAIInstrumentor().uninstrument() + + +def genai_client(media_provider): + from google import genai + from google.genai import types + + return genai.Client(api_key="synthetic-key", http_options=types.HttpOptions(base_url=media_provider.url + "/")) + + +def gemini_contents(): + from google.genai import types + + return [types.Content(role="user", parts=[ + types.Part(text="describe"), + types.Part(inline_data=types.Blob(mime_type="image/png", data=IMAGE)), + types.Part(inline_data=types.Blob(mime_type="application/pdf", data=PDF)), + types.Part(file_data=types.FileData(file_uri="gs://bucket/never-read.png", mime_type="image/png")), + types.Part(inline_data=types.Blob(mime_type="audio/L16;codec=pcm;rate=24000", data=b"\x00\x01" * 64)), + ])] + + +@pytest.mark.parametrize("async_", [False, True]) +def test_genai_generate_content_input_and_generated_output(f01, media_provider, genai_instrument, async_): + manager = f01.manager() + exporter = genai_instrument(manager) + client = genai_client(media_provider) + if async_: + response = asyncio.run(client.aio.models.generate_content(model="gemini-synthetic", contents=gemini_contents())) + else: + response = client.models.generate_content(model="gemini-synthetic", contents=gemini_contents()) + assert response.candidates[0].content.parts[1].inline_data.data == media_provider.generated + flush(manager) + assert sorted(f01.payloads) == sorted([IMAGE, PDF, media_provider.generated]) + by_purpose = uploads_by_purpose(f01) + assert sorted(by_purpose) == ["document", "model_input", "model_output"] + assert by_purpose["model_output"][0]["direction"] == "output" + span = span_named(exporter, "GenerateContent", "AsyncGenerateContent") + positions = sorted((r["direction"], r["message_index"], r["part_index"]) for r in refs(span)) + assert positions == [("input", 0, 1), ("input", 0, 2), ("output", 0, 1)] + # The application's request reached the provider unchanged (base64 on the wire, as the SDK sends it). + sent = json.dumps(media_provider.requests[-1][1]) + assert b64(IMAGE) in sent + manager.shutdown(5) + + +def test_genai_stream_captures_input_only(f01, media_provider, genai_instrument): + manager = f01.manager() + genai_instrument(manager) + client = genai_client(media_provider) + for _ in client.models.generate_content_stream(model="gemini-synthetic", contents=gemini_contents()): + pass + flush(manager) + assert sorted(f01.payloads) == sorted([IMAGE, PDF]) + manager.shutdown(5) + + +def test_genai_bare_parts_and_audio(f01, media_provider, genai_instrument): + from google.genai import types + + manager = f01.manager() + genai_instrument(manager) + sound = wav(seed=5) + client = genai_client(media_provider) + client.models.generate_content(model="gemini-synthetic", contents=[ + types.Part.from_bytes(data=sound, mime_type="audio/wav"), + types.Part.from_bytes(data=MP4, mime_type="video/mp4"), + "transcribe", + ]) + flush(manager) + assert sorted(f01.payloads) == sorted([sound, MP4, media_provider.generated]) + manager.shutdown(5) + + +@pytest.mark.parametrize("config", [TraceConfig(hide_input_images=True), TraceConfig(hide_outputs=True)]) +def test_genai_local_restrictions(f01, media_provider, genai_instrument, config): + manager = f01.manager() + genai_instrument(manager, config) + genai_client(media_provider).models.generate_content(model="gemini-synthetic", contents=gemini_contents()) + flush(manager) + if config.hide_input_images: + assert sorted(f01.payloads) == sorted([PDF, media_provider.generated]) + else: + assert sorted(f01.payloads) == sorted([IMAGE, PDF]) + manager.shutdown(5) + + +def test_genai_without_manager_is_unchanged(media_provider, genai_instrument): + exporter = genai_instrument(None) + genai_client(media_provider).models.generate_content(model="gemini-synthetic", contents=gemini_contents()) + span = span_named(exporter, "GenerateContent") + assert "fi.attachment.refs" not in span.attributes + + +# ------------------------------------------------------------------ Vertex (gapic, REST transport) +@pytest.fixture +def vertex_instrument(spans): + pytest.importorskip("google.cloud.aiplatform_v1") + from traceai_vertexai import VertexAIInstrumentor + + tracer_provider, exporter = spans + state = {} + + def install(manager, config=None): + VertexAIInstrumentor().instrument(tracer_provider=tracer_provider, attachments=manager, + config=config or TraceConfig()) + state["on"] = True + return exporter + + yield install + if state.get("on"): + VertexAIInstrumentor().uninstrument() + + +def vertex_client(media_provider): + from google.auth.credentials import AnonymousCredentials + from google.cloud import aiplatform_v1 + + # Created AFTER instrument(): the instrumentor wraps gapic's wrap_method. + return aiplatform_v1.PredictionServiceClient( + credentials=AnonymousCredentials(), transport="rest", + client_options={"api_endpoint": media_provider.url}) + + +MODEL = "projects/synthetic/locations/us-central1/publishers/google/models/" + + +def test_vertex_generate_content_input_and_output(f01, media_provider, vertex_instrument): + from google.cloud import aiplatform_v1 as v1 + + manager = f01.manager() + exporter = vertex_instrument(manager) + request = v1.GenerateContentRequest(model=MODEL + "gemini-synthetic", contents=[v1.Content(role="user", parts=[ + v1.Part(text="describe"), + v1.Part(inline_data=v1.Blob(mime_type="image/png", data=IMAGE)), + v1.Part(inline_data=v1.Blob(mime_type="application/pdf", data=PDF)), + v1.Part(file_data=v1.FileData(file_uri="gs://bucket/never-read.png", mime_type="image/png")), + ])]) + response = vertex_client(media_provider).generate_content(request=request) + assert response.candidates[0].content.parts[1].inline_data.data == media_provider.generated + flush(manager) + assert sorted(f01.payloads) == sorted([IMAGE, PDF, media_provider.generated]) + assert sorted(uploads_by_purpose(f01)) == ["document", "model_input", "model_output"] + assert len(refs(exporter.get_finished_spans()[-1])) == 3 + manager.shutdown(5) + + +def test_vertex_imagen_predictions_are_generated_images(f01, media_provider, vertex_instrument): + from google.protobuf import json_format, struct_pb2 + + manager = f01.manager() + vertex_instrument(manager) + instance = json_format.ParseDict({"prompt": "a cat"}, struct_pb2.Value()) + vertex_client(media_provider).predict(endpoint=MODEL + "imagen-3.0-generate-002", instances=[instance]) + flush(manager) + assert f01.payloads == [media_provider.generated] + assert f01.uploads[0]["purpose"] == "model_output" and f01.uploads[0]["direction"] == "output" + manager.shutdown(5) + + +def test_vertex_other_prediction_endpoints_are_never_read(f01, media_provider, vertex_instrument): + from google.protobuf import json_format, struct_pb2 + + manager = f01.manager() + vertex_instrument(manager) + instance = json_format.ParseDict({"content": "x"}, struct_pb2.Value()) + vertex_client(media_provider).predict(endpoint="projects/synthetic/locations/us-central1/endpoints/123", + instances=[instance]) + flush(manager) + assert f01.payloads == [] + manager.shutdown(5) diff --git a/python/tests/attachments_compat_providers/test_provider_media.py b/python/tests/attachments_compat_providers/test_provider_media.py new file mode 100644 index 000000000..624bfa67c --- /dev/null +++ b/python/tests/attachments_compat_providers/test_provider_media.py @@ -0,0 +1,327 @@ +"""provider_media helper rules, through the real manager to a loopback F01 server.""" + +import builtins +import dataclasses +import io +import os +import subprocess +import sys +import textwrap +import traceback + +import pytest +from opentelemetry import context as otel_context + +from conftest import MP4, PDF, b64, data_uri, flush, jpeg, png, refs, wav # noqa: E402 +from fi_instrumentation import TraceConfig # noqa: E402 +from fi_instrumentation.attachments import AttachmentConfig, CapturePolicy, create_attachment_manager # noqa: E402 +from fi_instrumentation.attachments import media_capture, provider_media # noqa: E402 +from fi_instrumentation.attachments.provider_media import ( # noqa: E402 + PROVIDER_MEDIA_CAPABILITIES, + ProviderMedia, + capabilities, + consistent, + sniff, +) + +IMAGE = png(seed=31) +OTHER = png(seed=32) + + +def capture(manager, tracer_provider, fn, *, trace_config=None, name="llm"): + with tracer_provider.get_tracer("t").start_as_current_span(name) as span: + media = ProviderMedia(manager, span, "test", trace_config=trace_config) + fn(media) + return media, span + + +def test_compatible_messages_capture_data_uri_images_and_audio(f01, spans): + tracer_provider, exporter = spans + manager = f01.manager() + sound = wav(seed=3) + messages = [ + {"role": "system", "content": "be brief"}, + {"role": "user", "content": [ + {"type": "text", "text": "what is this"}, + {"type": "image_url", "image_url": {"url": data_uri(IMAGE)}}, + {"type": "image_url", "image_url": {"url": "https://example.invalid/x.png"}}, + {"type": "input_audio", "input_audio": {"data": b64(sound), "format": "wav"}}, + ]}, + ] + media, _ = capture(manager, tracer_provider, lambda m: m.compatible_messages(messages, "openai")) + flush(manager) + assert sorted(f01.payloads) == sorted([IMAGE, sound]) + assert set(media.captured) == {"chat/1/1", "chat/1/3"} + span = exporter.get_finished_spans()[-1] + by_mime = {ref["mime_type"]: ref for ref in refs(span)} + assert by_mime["image/png"]["message_index"] == 1 and by_mime["image/png"]["part_index"] == 1 + assert by_mime["audio/wav"]["role"] == "user" and by_mime["audio/wav"]["direction"] == "input" + manager.shutdown(5) + + +@pytest.mark.parametrize("dialect", ["groq", "mistral", "image"]) +def test_image_only_dialects_never_read_audio(f01, spans, dialect): + tracer_provider, _ = spans + manager = f01.manager() + messages = [{"role": "user", "content": [ + {"type": "input_audio", "input_audio": {"data": b64(wav()), "format": "wav"}}, + {"type": "image_url", "image_url": data_uri(IMAGE) if dialect == "mistral" else {"url": data_uri(IMAGE)}}, + ]}] + capture(manager, tracer_provider, lambda m: m.compatible_messages(messages, dialect)) + flush(manager) + assert f01.payloads == [IMAGE] + manager.shutdown(5) + + +def test_wrong_label_is_never_uploaded_and_cannot_bypass_hide_input_images(f01, spans): + """A PNG labelled as WAV audio would dodge hide_input_images; the magic-byte check refuses it.""" + tracer_provider, _ = spans + manager = f01.manager() + messages = [{"role": "user", "content": [ + {"type": "input_audio", "input_audio": {"data": b64(IMAGE), "format": "wav"}}]}] + contents = [{"role": "user", "parts": [{"inline_data": {"mime_type": "audio/wav", "data": OTHER}}]}] + capture(manager, tracer_provider, lambda m: (m.compatible_messages(messages, "openai"), m.gemini_contents(contents)), + trace_config=TraceConfig(hide_input_images=True)) + flush(manager) + assert f01.payloads == [] + assert f01.uploads == [] + manager.shutdown(5) + + +def test_declared_pdf_and_video_need_matching_bytes(f01, spans): + tracer_provider, _ = spans + manager = f01.manager() + contents = [{"role": "user", "parts": [ + {"inline_data": {"mime_type": "application/pdf", "data": PDF}}, + {"inline_data": {"mime_type": "video/mp4", "data": MP4}}, + {"inline_data": {"mime_type": "application/pdf", "data": IMAGE}}, # mislabelled: refused + {"inline_data": {"mime_type": "image/jpeg", "data": jpeg(4)}}, + ]}] + capture(manager, tracer_provider, lambda m: m.gemini_contents(contents)) + flush(manager) + assert sorted(f01.payloads) == sorted([PDF, MP4, jpeg(4)]) + purposes = sorted(u["purpose"] for u in f01.uploads) + assert purposes == ["document", "model_input", "model_input"] + manager.shutdown(5) + + +def test_preflight_runs_before_any_read(f01, spans): + """A denied part is never read: the data-URI getter is not called under hide_input_images.""" + tracer_provider, _ = spans + manager = f01.manager() + reads = [] + + class Part(dict): + def get(self, key, default=None): + if key == "image_url": + reads.append(key) + return super().get(key, default) + + messages = [{"role": "user", "content": [Part(type="image_url", image_url={"url": data_uri(IMAGE)})]}] + capture(manager, tracer_provider, lambda m: m.compatible_messages(messages, "openai"), + trace_config=TraceConfig(hide_input_images=True)) + flush(manager) + assert reads == [] and f01.payloads == [] + capture(manager, tracer_provider, lambda m: m.compatible_messages(messages, "openai")) + flush(manager) + assert reads == ["image_url"] and f01.payloads == [IMAGE] + manager.shutdown(5) + + +def test_small_server_cap_resizes_data_uri_gate(f01, spans): + """The data-URI gate asks for 1 MiB; a server cap below that re-admits at the exact size.""" + tracer_provider, _ = spans + f01.state["limits"] = {"max_upload_bytes": 4096} + manager = f01.manager() + messages = [{"role": "user", "content": [{"type": "image_url", "image_url": {"url": data_uri(IMAGE)}}]}] + capture(manager, tracer_provider, lambda m: m.compatible_messages(messages, "openai")) + flush(manager) + assert f01.payloads == [IMAGE] + manager.shutdown(5) + + +def test_untyped_images_sniff_their_type_and_skip_paths(f01, spans, tmp_path, monkeypatch): + """Ollama/Titan images: bytes or base64 only. A path string is never opened or decoded into an upload.""" + tracer_provider, _ = spans + manager = f01.manager() + path = tmp_path / "picture.png" + path.write_bytes(OTHER) + opened = [] + real_open, real_io_open = builtins.open, io.open + + def watch(opener): + def wrapped(file, *args, **kwargs): + if any(frame.filename.endswith("provider_media.py") for frame in traceback.extract_stack()): + opened.append(str(file)) + return opener(file, *args, **kwargs) + return wrapped + + monkeypatch.setattr(builtins, "open", watch(real_open)) + monkeypatch.setattr(io, "open", watch(real_io_open)) + images = [IMAGE, b64(jpeg(7)), str(path), path, "not base64!"] + media, _ = capture(manager, tracer_provider, lambda m: m.ollama_images(images)) + flush(manager) + monkeypatch.undo() + assert sorted(f01.payloads) == sorted([IMAGE, jpeg(7)]) + assert opened == [] + assert sorted(u["mime_type"] for u in f01.uploads) == ["image/jpeg", "image/png"] + manager.shutdown(5) + + +def test_context_policy_and_owner_keys_deny(f01, spans): + """The shared keys media_capture.CAPTURE_POLICY_KEY / OWNER_KEY apply to provider media (deny wins).""" + tracer_provider, _ = spans + manager = f01.manager() + messages = [{"role": "user", "content": [{"type": "image_url", "image_url": {"url": data_uri(IMAGE)}}]}] + for key, value in ((media_capture.CAPTURE_POLICY_KEY, CapturePolicy(capture_content=False)), + (media_capture.OWNER_KEY, True)): + token = otel_context.attach(otel_context.set_value(key, value)) + try: + media, _ = capture(manager, tracer_provider, lambda m: m.compatible_messages(messages, "openai")) + finally: + otel_context.detach(token) + assert media.captured == {} + flush(manager) + assert f01.payloads == [] + # claim() marks exactly that owner key. + assert otel_context.get_value(media_capture.OWNER_KEY, provider_media.claim()) is True + manager.shutdown(5) + + +@pytest.mark.parametrize("config", [ + TraceConfig(hide_input_images=True), TraceConfig(hide_inputs=True), TraceConfig(pii_redaction=True), + TraceConfig(hide_input_messages=True), TraceConfig(hide_input_text=True), +]) +def test_local_input_restrictions_deny(f01, spans, config): + tracer_provider, _ = spans + manager = f01.manager(allow_unredacted_media_with_text_pii=True) + messages = [{"role": "user", "content": [{"type": "image_url", "image_url": {"url": data_uri(IMAGE)}}]}] + contents = [{"role": "user", "parts": [{"inline_data": {"mime_type": "image/png", "data": OTHER}}]}] + capture(manager, tracer_provider, lambda m: (m.compatible_messages(messages), m.gemini_contents(contents)), + trace_config=config) + flush(manager) + assert f01.payloads == [] + manager.shutdown(5) + + +@pytest.mark.parametrize("config", [TraceConfig(hide_outputs=True), TraceConfig(hide_output_messages=True), + TraceConfig(hide_output_text=True), TraceConfig(pii_redaction=True)]) +def test_local_output_restrictions_deny_generated_media(f01, spans, config): + tracer_provider, _ = spans + manager = f01.manager(allow_unredacted_media_with_text_pii=True) + response = {"candidates": [{"content": {"role": "model", "parts": [ + {"inline_data": {"mime_type": "image/png", "data": IMAGE}}]}}]} + capture(manager, tracer_provider, lambda m: (m.gemini_response(response), + m.generated_images([{"b64_json": b64(OTHER)}])), + trace_config=config) + flush(manager) + assert f01.payloads == [] + manager.shutdown(5) + + +def test_unsampled_disabled_and_missing_managers_capture_nothing(f01, spans): + from opentelemetry.sdk.trace import TracerProvider + from opentelemetry.sdk.trace.sampling import ALWAYS_OFF + + manager = f01.manager() + messages = [{"role": "user", "content": [{"type": "image_url", "image_url": {"url": data_uri(IMAGE)}}]}] + unsampled = TracerProvider(sampler=ALWAYS_OFF) + media, _ = capture(manager, unsampled, lambda m: m.compatible_messages(messages)) + assert not media.usable + disabled = create_attachment_manager(config=AttachmentConfig(enabled=False)) + tracer_provider, _ = spans + for candidate in (None, disabled, object()): + media, _ = capture(candidate, tracer_provider, lambda m: m.compatible_messages(messages)) + assert not media.usable and media.captured == {} + flush(manager) + assert f01.payloads == [] + manager.shutdown(5) + + +def test_second_observer_of_same_span_uploads_nothing(f01, spans): + tracer_provider, _ = spans + manager = f01.manager() + messages = [{"role": "user", "content": [{"type": "image_url", "image_url": {"url": data_uri(IMAGE)}}]}] + with tracer_provider.get_tracer("t").start_as_current_span("llm") as span: + first = ProviderMedia(manager, span, "ns").compatible_messages(messages).captured + second = ProviderMedia(manager, span, "ns").compatible_messages(messages).captured + flush(manager) + assert first == second and len(first) == 1 and f01.payloads == [IMAGE] + manager.shutdown(5) + + +def test_part_cap_bounds_work(f01, spans): + tracer_provider, _ = spans + manager = f01.manager() + content = [{"type": "image_url", "image_url": {"url": data_uri(png(seed=i))}} for i in range(80)] + media, _ = capture(manager, tracer_provider, lambda m: m.compatible_messages([{"role": "user", "content": content}])) + flush(manager) + assert len(f01.payloads) == provider_media.MAX_PARTS == len(media.captured) + manager.shutdown(5) + + +def test_sniff_and_consistent(): + assert sniff(IMAGE) == "image/png" and sniff(jpeg()) == "image/jpeg" and sniff(PDF) == "application/pdf" + assert sniff(MP4) == "video/mp4" and sniff(wav()) == "audio/wav" and sniff(b"RIFF\x00\x00\x00\x00WEBPVP8 ") == "image/webp" + assert sniff(b"hello") is None and sniff("not bytes") is None + assert consistent("image/png", IMAGE) and not consistent("audio/wav", IMAGE) + assert consistent("video/webm", b"\x1a\x45\xdf\xa3" + b"\x00" * 12) + assert consistent("audio/webm", b"\x1a\x45\xdf\xa3" + b"\x00" * 12) + + +def test_capability_matrix_is_explicit(): + for package in ("traceai_deepseek", "traceai_cerebras"): + assert PROVIDER_MEDIA_CAPABILITIES[package]["operations"] == () + assert PROVIDER_MEDIA_CAPABILITIES[package]["note"] + for package, descriptor in PROVIDER_MEDIA_CAPABILITIES.items(): + assert descriptor["schema_version"] == 1 and isinstance(descriptor["operations"], tuple) + assert "remote_url" in descriptor["unsupported"] or package in ( + "traceai_google_genai", "traceai_vertexai", "traceai_bedrock", "traceai_ollama", "traceai_deepseek", + "traceai_cerebras") + assert capabilities("traceai_unknown")["operations"] == () + with pytest.raises(TypeError): + PROVIDER_MEDIA_CAPABILITIES["traceai_xai"] = {} # read-only + + +PROBE = textwrap.dedent(""" + import sys + from opentelemetry.sdk.trace import TracerProvider + names = sys.argv[1:] + import importlib + for name in names: + try: + module = importlib.import_module(name) + except ImportError: + continue + instrumentor = [getattr(module, attr) for attr in dir(module) if attr.endswith("Instrumentor") + and attr != "BaseInstrumentor"] + if instrumentor: + try: + instrumentor[0]().instrument(tracer_provider=TracerProvider()) + except Exception as error: + print("SKIP", name, type(error).__name__) + loaded = sorted(m for m in sys.modules if m.startswith("fi_instrumentation.attachments")) + print("LOADED", ",".join(loaded)) +""") + + +def test_disabled_path_never_imports_attachment_modules(tmp_path): + """Without a manager, instrumenting every provider package imports no attachment module.""" + script = tmp_path / "probe.py" + script.write_text(PROBE) + names = ["traceai_google_genai", "traceai_vertexai", "traceai_bedrock", "traceai_litellm", "traceai_groq", + "traceai_together", "traceai_fireworks", "traceai_xai", "traceai_mistralai", "traceai_openrouter", + "traceai_ollama", "traceai_portkey", "traceai_deepseek", "traceai_cerebras"] + result = subprocess.run([sys.executable, str(script), *names], capture_output=True, text=True, timeout=120, + env=dict(os.environ, LITELLM_LOCAL_MODEL_COST_MAP="True")) + assert result.returncode == 0, result.stderr[-2000:] + line = [row for row in result.stdout.splitlines() if row.startswith("LOADED")][-1] + loaded = line.split(" ", 1)[1] if " " in line else "" + assert "provider_media" not in loaded and "media_capture" not in loaded, line + + +def test_dataclass_policy_unchanged_by_helper(): + """local_capture_policy is the only fold; the helper adds no permissive default.""" + policy = media_capture.local_capture_policy(None, direction="input", image=True) + assert dataclasses.asdict(policy) == dataclasses.asdict( + dataclasses.replace(CapturePolicy(), model_input=True)) From 862548de8e0fdd708c2ec042ae99c0398e7cc0b9 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek <8347993+nik13@users.noreply.github.com> Date: Wed, 11 Nov 2026 19:52:49 -0800 Subject: [PATCH 04/15] fix(python): walk proto repeated fields; together claims no chat media; CI workflow Vertex Imagen predictions are a proto repeated field that does not slice, so the bounded walk uses islice. traceai_together installs no chat wrapper today, so its descriptor claims nothing. New workflow runs every provider binding at its declared floor and latest SDK with outbound network off. Co-Authored-By: Claude Opus 5.5 --- ...achments-python-compat-providers-tests.yml | 131 ++++++++++++++++++ .../attachments/provider_media.py | 12 +- .../attachments_compat_providers/conftest.py | 1 + .../test_bedrock_litellm_attachments.py | 7 +- .../test_compat_attachments.py | 26 +++- .../test_google_attachments.py | 9 +- .../test_provider_media.py | 13 +- 7 files changed, 186 insertions(+), 13 deletions(-) create mode 100644 .github/workflows/attachments-python-compat-providers-tests.yml diff --git a/.github/workflows/attachments-python-compat-providers-tests.yml b/.github/workflows/attachments-python-compat-providers-tests.yml new file mode 100644 index 000000000..bee9db83c --- /dev/null +++ b/.github/workflows/attachments-python-compat-providers-tests.yml @@ -0,0 +1,131 @@ +name: Python provider media (Gemini, Vertex, Bedrock, LiteLLM, compatible chat) + +# Real provider SDKs (declared floor and latest) through the Python +# instrumentors and fi_instrumentation.attachments.provider_media to a TLS +# loopback F01 attachment server. Outbound network is disabled while the tests run. + +on: + pull_request: + paths: + - 'python/fi_instrumentation/attachments/**' + - 'python/frameworks/google-genai/**' + - 'python/frameworks/vertexai/**' + - 'python/frameworks/bedrock/**' + - 'python/frameworks/litellm/**' + - 'python/frameworks/groq/**' + - 'python/frameworks/together/**' + - 'python/frameworks/fireworks/**' + - 'python/frameworks/xai/**' + - 'python/frameworks/deepseek/**' + - 'python/frameworks/cerebras/**' + - 'python/frameworks/mistralai/**' + - 'python/frameworks/openrouter/**' + - 'python/frameworks/portkey/**' + - 'python/frameworks/ollama/**' + - 'python/tests/attachments_providers/conftest.py' + - 'python/tests/attachments_compat_providers/**' + - '.github/workflows/attachments-python-compat-providers-tests.yml' + push: + branches: [dev] + paths: + - 'python/fi_instrumentation/attachments/**' + - 'python/frameworks/google-genai/**' + - 'python/frameworks/vertexai/**' + - 'python/frameworks/bedrock/**' + - 'python/frameworks/litellm/**' + - 'python/frameworks/groq/**' + - 'python/frameworks/together/**' + - 'python/frameworks/fireworks/**' + - 'python/frameworks/xai/**' + - 'python/frameworks/deepseek/**' + - 'python/frameworks/cerebras/**' + - 'python/frameworks/mistralai/**' + - 'python/frameworks/openrouter/**' + - 'python/frameworks/portkey/**' + - 'python/frameworks/ollama/**' + - 'python/tests/attachments_providers/conftest.py' + - 'python/tests/attachments_compat_providers/**' + - '.github/workflows/attachments-python-compat-providers-tests.yml' + workflow_dispatch: + +permissions: + contents: read + +jobs: + media: + name: Provider media (${{ matrix.package }} ${{ matrix.tier }}) + runs-on: ubuntu-24.04 + timeout-minutes: 25 + strategy: + fail-fast: false + matrix: + include: + - {package: core, tier: latest, python: '3.12', sdks: '', tests: core} + - {package: google-genai, tier: floor, python: '3.10', sdks: 'google-genai==1.8.0', tests: google} + - {package: google-genai, tier: latest, python: '3.12', sdks: 'google-genai==3.2.0', tests: google} + - {package: vertexai, tier: floor, python: '3.10', sdks: 'google-cloud-aiplatform==1.61.0', tests: google} + - {package: vertexai, tier: latest, python: '3.12', sdks: 'google-cloud-aiplatform==1.101.0', tests: google} + - {package: bedrock, tier: floor, python: '3.10', sdks: 'boto3==1.38.17', tests: aws} + - {package: bedrock, tier: latest, python: '3.12', sdks: 'boto3==1.44.1', tests: aws} + - {package: litellm, tier: floor, python: '3.10', sdks: '-e python/frameworks/openai litellm==1.53.1 openai==1.69.0', tests: aws} + - {package: litellm, tier: latest, python: '3.12', sdks: '-e python/frameworks/openai litellm==1.109.1', tests: aws} + - {package: groq, tier: floor, python: '3.10', sdks: 'groq==0.9.0 httpx<0.28', tests: compat} + - {package: groq, tier: latest, python: '3.12', sdks: 'groq==1.7.0', tests: compat} + - {package: together, tier: floor, python: '3.10', sdks: 'together==1.3.0', tests: compat} + - {package: together, tier: latest, python: '3.12', sdks: 'together==3.1.1', tests: compat} + - {package: fireworks, tier: floor, python: '3.10', sdks: 'openai==1.69.0 httpx<0.28', tests: compat} + - {package: fireworks, tier: latest, python: '3.12', sdks: 'openai==3.49.0', tests: compat} + - {package: xai, tier: floor, python: '3.10', sdks: 'openai==1.69.0 httpx<0.28', tests: compat} + - {package: xai, tier: latest, python: '3.12', sdks: 'openai==3.49.0', tests: compat} + - {package: deepseek, tier: floor, python: '3.10', sdks: 'openai==1.69.0 httpx<0.28', tests: compat} + - {package: deepseek, tier: latest, python: '3.12', sdks: 'openai==3.49.0', tests: compat} + - {package: cerebras, tier: floor, python: '3.10', sdks: 'cerebras-cloud-sdk==1.0.0', tests: compat} + - {package: cerebras, tier: latest, python: '3.12', sdks: 'cerebras-cloud-sdk==1.91.0', tests: compat} + - {package: mistralai, tier: floor, python: '3.10', sdks: 'mistralai==1.1.0', tests: compat} + - {package: mistralai, tier: latest, python: '3.12', sdks: 'mistralai==3.7.0', tests: compat} + - {package: openrouter, tier: floor, python: '3.10', sdks: 'openrouter==1.3.0', tests: compat} + - {package: openrouter, tier: latest, python: '3.12', sdks: 'openrouter==1.3.107', tests: compat} + - {package: portkey, tier: floor, python: '3.10', sdks: 'portkey-ai==1.9.0 httpx<0.28', tests: compat} + - {package: portkey, tier: latest, python: '3.12', sdks: 'portkey-ai==2.3.4', tests: compat} + - {package: ollama, tier: floor, python: '3.10', sdks: 'ollama==0.3.0', tests: compat} + - {package: ollama, tier: latest, python: '3.12', sdks: 'ollama==0.6.3', tests: compat} + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: false + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: ${{ matrix.python }} + - name: Install the core, the integration and the selected SDKs + env: + PACKAGE: ${{ matrix.package }} + SDKS: ${{ matrix.sdks }} + run: | + python -m venv .venv + integration="" + if [ "$PACKAGE" != core ]; then integration="-e python/frameworks/$PACKAGE"; fi + # shellcheck disable=SC2086 + .venv/bin/pip install -e python $integration opentelemetry-sdk 'pytest>=8,<9' 'pytest-asyncio<0.25' $SDKS + .venv/bin/pip freeze | grep -iE '^(google-genai|google-cloud-aiplatform|boto3|litellm|openai|groq|together|cerebras-cloud-sdk|mistralai|openrouter|portkey-ai|ollama|httpx)==' || true + - name: Run with outbound network disabled + env: + SUITE: ${{ matrix.tests }} + run: | + case "$SUITE" in + core) tests="tests/attachments_compat_providers/test_provider_media.py" ;; + google) tests="tests/attachments_compat_providers/test_google_attachments.py" ;; + aws) tests="tests/attachments_compat_providers/test_bedrock_litellm_attachments.py" ;; + compat) tests="tests/attachments_compat_providers/test_compat_attachments.py" ;; + esac + if [ "$SUITE" != core ]; then + tests="$tests tests/attachments_compat_providers/test_compat_attachments.py::test_package_descriptor" + tests="$tests tests/attachments_compat_providers/test_provider_media.py::test_disabled_path_never_imports_attachment_modules" + fi + sudo unshare --net -- bash -e -c ' + ip link set lo up + cd "$2/python" + export FI_BASE_URL=http://127.0.0.1:9 FI_GRPC_URL=http://127.0.0.1:9 + export LITELLM_LOCAL_MODEL_COST_MAP=True AWS_EC2_METADATA_DISABLED=true + # shellcheck disable=SC2086 + PYTHONPATH=. "$1" -m pytest $3 -v -rs -p no:cacheprovider -o addopts="" -o faulthandler_timeout=180 + ' _ "$PWD/.venv/bin/python" "$PWD" "$tests" diff --git a/python/fi_instrumentation/attachments/provider_media.py b/python/fi_instrumentation/attachments/provider_media.py index a07208456..a003967d1 100644 --- a/python/fi_instrumentation/attachments/provider_media.py +++ b/python/fi_instrumentation/attachments/provider_media.py @@ -28,6 +28,7 @@ import re from collections.abc import Sequence +from itertools import islice from types import MappingProxyType from typing import Any, Callable, Mapping, Optional, Tuple @@ -577,10 +578,13 @@ def bedrock_invoke_response(self, model_id: Any, body: Any) -> "ProviderMedia": def _items(value: Any) -> list: """A bounded list of a list, tuple or other sequence (proto repeated fields); never a str or bytes.""" + if isinstance(value, (list, tuple)): + return list(value[:MAX_PARTS]) if isinstance(value, (str, bytes, bytearray, memoryview, Mapping)) or not isinstance(value, Sequence): return [] try: - return list(value[:MAX_PARTS]) + # Proto repeated fields do not all support slicing; bound the walk instead. + return list(islice(value, MAX_PARTS)) except Exception: return [] @@ -669,8 +673,10 @@ def _descriptor(operations: Tuple[str, ...], sources: Tuple[str, ...], unsupport ("remote_url", "provider_file", "streamed_output", "image_generation.url", "image_edit", "speech", "transcription")), "traceai_groq": _descriptor(("chat.completions.create.input",), ("data_uri",), _COMPAT_UNSUPPORTED), - "traceai_together": _descriptor(("chat.completions.create.input",), ("data_uri",), - _COMPAT_UNSUPPORTED + ("images.generate",)), + # The binding is wired, but traceai_together installs no chat wrapper today (its install check looks + # for `chat` on the Together class, where it is an instance attribute), so nothing is claimed. + "traceai_together": _descriptor((), (), _COMPAT_UNSUPPORTED + ("chat.completions.create", "images.generate"), + "traceai_together does not trace chat.completions.create yet; no media claim."), "traceai_fireworks": _descriptor(("chat.completions.create.input",), ("data_uri",), _COMPAT_UNSUPPORTED), "traceai_xai": _descriptor(("chat.completions.create.input",), ("data_uri",), _COMPAT_UNSUPPORTED), "traceai_mistralai": _descriptor(("chat.complete.input", "chat.stream.input"), ("data_uri",), diff --git a/python/tests/attachments_compat_providers/conftest.py b/python/tests/attachments_compat_providers/conftest.py index 138958f44..67aef6d2e 100644 --- a/python/tests/attachments_compat_providers/conftest.py +++ b/python/tests/attachments_compat_providers/conftest.py @@ -89,6 +89,7 @@ def _openai_chat(body): return "text/event-stream", "".join("data: %s\n\n" % json.dumps(c) for c in chunks).encode() + b"data: [DONE]\n\n" return "application/json", json.dumps({ "id": "c1", "object": "chat.completion", "created": 1, "model": body.get("model", "m"), + "system_fingerprint": "fp_synthetic", "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}], "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}}).encode() diff --git a/python/tests/attachments_compat_providers/test_bedrock_litellm_attachments.py b/python/tests/attachments_compat_providers/test_bedrock_litellm_attachments.py index 836fd3d6f..cc138e21c 100644 --- a/python/tests/attachments_compat_providers/test_bedrock_litellm_attachments.py +++ b/python/tests/attachments_compat_providers/test_bedrock_litellm_attachments.py @@ -21,6 +21,7 @@ @pytest.fixture def bedrock(spans, media_provider): pytest.importorskip("boto3") + pytest.importorskip("traceai_bedrock") from traceai_bedrock import BedrockInstrumentor tracer_provider, exporter = spans @@ -106,6 +107,7 @@ def test_converse_unknown_model_family_is_never_read(f01, bedrock): def test_nova_converse_video_block(f01, bedrock): + f01.state["policy"] = {"allowed_modalities": ["audio", "image", "document", "file", "video"]} manager = f01.manager() client, _ = bedrock(manager) client.converse(modelId=NOVA, messages=[{"role": "user", "content": [ @@ -196,6 +198,7 @@ def test_invoke_model_unknown_body_is_never_scraped(f01, bedrock): @pytest.fixture def litellm_instrument(spans): pytest.importorskip("litellm") + pytest.importorskip("traceai_litellm") from traceai_litellm import LiteLLMInstrumentor tracer_provider, exporter = spans @@ -273,8 +276,8 @@ def test_litellm_image_generation_output(f01, media_provider, litellm_instrument manager = f01.manager() exporter = litellm_instrument(manager) - options = dict(model="openai/dall-e-3", prompt="a cat", api_base=media_provider.url, api_key="sk-synthetic", - response_format="b64_json") + # gpt-image-1 always answers with b64_json; the loopback also returns a URL result, which is never fetched. + options = dict(model="openai/gpt-image-1", prompt="a cat", api_base=media_provider.url, api_key="sk-synthetic") if async_: result = asyncio.run(litellm.aimage_generation(**options)) else: diff --git a/python/tests/attachments_compat_providers/test_compat_attachments.py b/python/tests/attachments_compat_providers/test_compat_attachments.py index 95f4a874f..df21eaa09 100644 --- a/python/tests/attachments_compat_providers/test_compat_attachments.py +++ b/python/tests/attachments_compat_providers/test_compat_attachments.py @@ -50,6 +50,7 @@ def media_refs(exporter): # ------------------------------------------------------------------ groq (image-only chat) def test_groq_chat_images(f01, media_provider, instrumented): groq = pytest.importorskip("groq") + pytest.importorskip("traceai_groq") from traceai_groq import GroqInstrumentor manager = f01.manager() @@ -66,8 +67,11 @@ def test_groq_chat_images(f01, media_provider, instrumented): # ------------------------------------------------------------------ together -def test_together_chat_images(f01, media_provider, instrumented): +def test_together_chat_is_not_claimed(f01, media_provider, instrumented): + """traceai_together does not trace chat today, so it must not claim (or upload) chat media.""" together = pytest.importorskip("together") + pytest.importorskip("traceai_together") + import traceai_together from traceai_together import TogetherInstrumentor manager = f01.manager() @@ -75,8 +79,9 @@ def test_together_chat_images(f01, media_provider, instrumented): client = together.Together(api_key="synthetic", base_url=media_provider.url + "/v1") client.chat.completions.create(model="m", messages=image_messages()) flush(manager) - assert f01.payloads == [IMAGE] - assert len(media_refs(exporter)) == 1 + assert f01.payloads == [] and media_refs(exporter) == [] + descriptor = traceai_together.together_attachment_capabilities + assert descriptor["operations"] == () and "chat.completions.create" in descriptor["unsupported"] manager.shutdown(5) @@ -84,6 +89,7 @@ def test_together_chat_images(f01, media_provider, instrumented): @pytest.mark.parametrize("package", ["fireworks", "xai"]) def test_openai_sdk_routes_capture_images(f01, media_provider, instrumented, monkeypatch, package): openai = pytest.importorskip("openai") + pytest.importorskip("traceai_" + package) if package == "fireworks": from traceai_fireworks import FireworksInstrumentor as Instrumentor from traceai_fireworks import _wrappers @@ -108,6 +114,7 @@ def test_openai_sdk_routes_capture_images(f01, media_provider, instrumented, mon @pytest.mark.parametrize("package", ["deepseek", "cerebras"]) def test_routes_without_media_surface_capture_nothing(f01, media_provider, instrumented, monkeypatch, package): + pytest.importorskip("traceai_" + package) if package == "deepseek": openai = pytest.importorskip("openai") import traceai_deepseek @@ -128,8 +135,11 @@ def test_routes_without_media_surface_capture_nothing(f01, media_provider, instr manager = f01.manager() exporter = instrumented(Instrumentor(), manager) - client = sdk.Cerebras(api_key="synthetic", base_url=media_provider.url, max_retries=0, - warm_tcp_connection=False) + try: + client = sdk.Cerebras(api_key="synthetic", base_url=media_provider.url, max_retries=0, + warm_tcp_connection=False) + except TypeError: # cerebras-cloud-sdk 1.0.0 has no warm_tcp_connection option + client = sdk.Cerebras(api_key="synthetic", base_url=media_provider.url, max_retries=0) client.chat.completions.create(model="llama", messages=image_messages()) descriptor = traceai_cerebras.cerebras_attachment_capabilities flush(manager) @@ -142,6 +152,7 @@ def test_routes_without_media_surface_capture_nothing(f01, media_provider, instr @pytest.mark.parametrize("async_", [False, True]) def test_mistral_chat_images(f01, media_provider, instrumented, async_): mistralai = pytest.importorskip("mistralai") + pytest.importorskip("traceai_mistralai") from traceai_mistralai import MistralAIInstrumentor manager = f01.manager() @@ -164,6 +175,7 @@ def test_mistral_chat_images(f01, media_provider, instrumented, async_): def test_mistral_stream_and_hide_input_images(f01, media_provider, instrumented): mistralai = pytest.importorskip("mistralai") + pytest.importorskip("traceai_mistralai") from traceai_mistralai import MistralAIInstrumentor manager = f01.manager() @@ -181,6 +193,7 @@ def test_mistral_stream_and_hide_input_images(f01, media_provider, instrumented) # ------------------------------------------------------------------ openrouter def test_openrouter_chat_images_and_audio(f01, media_provider, instrumented): openrouter = pytest.importorskip("openrouter") + pytest.importorskip("traceai_openrouter") from traceai_openrouter import OpenRouterInstrumentor manager = f01.manager() @@ -197,6 +210,7 @@ def test_openrouter_chat_images_and_audio(f01, media_provider, instrumented): # ------------------------------------------------------------------ portkey def test_portkey_chat_images(f01, media_provider, instrumented): portkey_ai = pytest.importorskip("portkey_ai") + pytest.importorskip("traceai_portkey") from traceai_portkey import PortkeyInstrumentor manager = f01.manager() @@ -213,6 +227,7 @@ def test_portkey_chat_images(f01, media_provider, instrumented): # ------------------------------------------------------------------ ollama def test_ollama_chat_and_generate_images(f01, media_provider, instrumented, tmp_path): ollama = pytest.importorskip("ollama") + pytest.importorskip("traceai_ollama") from traceai_ollama import OllamaInstrumentor manager = f01.manager() @@ -233,6 +248,7 @@ def test_ollama_chat_and_generate_images(f01, media_provider, instrumented, tmp_ def test_ollama_async_chat(f01, media_provider, instrumented): ollama = pytest.importorskip("ollama") + pytest.importorskip("traceai_ollama") from traceai_ollama import OllamaInstrumentor manager = f01.manager() diff --git a/python/tests/attachments_compat_providers/test_google_attachments.py b/python/tests/attachments_compat_providers/test_google_attachments.py index bbb544ddc..478ffeba6 100644 --- a/python/tests/attachments_compat_providers/test_google_attachments.py +++ b/python/tests/attachments_compat_providers/test_google_attachments.py @@ -15,6 +15,7 @@ @pytest.fixture def genai_instrument(spans): pytest.importorskip("google.genai") + pytest.importorskip("traceai_google_genai") from traceai_google_genai import GoogleGenAIInstrumentor tracer_provider, exporter = spans @@ -88,6 +89,7 @@ def test_genai_stream_captures_input_only(f01, media_provider, genai_instrument) def test_genai_bare_parts_and_audio(f01, media_provider, genai_instrument): from google.genai import types + f01.state["policy"] = {"allowed_modalities": ["audio", "image", "document", "file", "video"]} manager = f01.manager() genai_instrument(manager) sound = wav(seed=5) @@ -106,7 +108,8 @@ def test_genai_bare_parts_and_audio(f01, media_provider, genai_instrument): def test_genai_local_restrictions(f01, media_provider, genai_instrument, config): manager = f01.manager() genai_instrument(manager, config) - genai_client(media_provider).models.generate_content(model="gemini-synthetic", contents=gemini_contents()) + client = genai_client(media_provider) # keep the client alive: it closes its transport when collected + client.models.generate_content(model="gemini-synthetic", contents=gemini_contents()) flush(manager) if config.hide_input_images: assert sorted(f01.payloads) == sorted([PDF, media_provider.generated]) @@ -117,7 +120,8 @@ def test_genai_local_restrictions(f01, media_provider, genai_instrument, config) def test_genai_without_manager_is_unchanged(media_provider, genai_instrument): exporter = genai_instrument(None) - genai_client(media_provider).models.generate_content(model="gemini-synthetic", contents=gemini_contents()) + client = genai_client(media_provider) # keep the client alive: it closes its transport when collected + client.models.generate_content(model="gemini-synthetic", contents=gemini_contents()) span = span_named(exporter, "GenerateContent") assert "fi.attachment.refs" not in span.attributes @@ -126,6 +130,7 @@ def test_genai_without_manager_is_unchanged(media_provider, genai_instrument): @pytest.fixture def vertex_instrument(spans): pytest.importorskip("google.cloud.aiplatform_v1") + pytest.importorskip("traceai_vertexai") from traceai_vertexai import VertexAIInstrumentor tracer_provider, exporter = spans diff --git a/python/tests/attachments_compat_providers/test_provider_media.py b/python/tests/attachments_compat_providers/test_provider_media.py index 624bfa67c..71f4dec64 100644 --- a/python/tests/attachments_compat_providers/test_provider_media.py +++ b/python/tests/attachments_compat_providers/test_provider_media.py @@ -90,6 +90,7 @@ def test_wrong_label_is_never_uploaded_and_cannot_bypass_hide_input_images(f01, def test_declared_pdf_and_video_need_matching_bytes(f01, spans): tracer_provider, _ = spans + f01.state["policy"] = {"allowed_modalities": ["audio", "image", "document", "file", "video"]} manager = f01.manager() contents = [{"role": "user", "parts": [ {"inline_data": {"mime_type": "application/pdf", "data": PDF}}, @@ -105,6 +106,16 @@ def test_declared_pdf_and_video_need_matching_bytes(f01, spans): manager.shutdown(5) +def test_server_policy_without_video_denies_video(f01, spans): + tracer_provider, _ = spans + manager = f01.manager() # the harness policy allows audio, image, document and file only + contents = [{"role": "user", "parts": [{"inline_data": {"mime_type": "video/mp4", "data": MP4}}]}] + capture(manager, tracer_provider, lambda m: m.gemini_contents(contents)) + flush(manager) + assert f01.payloads == [] + manager.shutdown(5) + + def test_preflight_runs_before_any_read(f01, spans): """A denied part is never read: the data-URI getter is not called under hide_input_images.""" tracer_provider, _ = spans @@ -270,7 +281,7 @@ def test_sniff_and_consistent(): def test_capability_matrix_is_explicit(): - for package in ("traceai_deepseek", "traceai_cerebras"): + for package in ("traceai_deepseek", "traceai_cerebras", "traceai_together"): assert PROVIDER_MEDIA_CAPABILITIES[package]["operations"] == () assert PROVIDER_MEDIA_CAPABILITIES[package]["note"] for package, descriptor in PROVIDER_MEDIA_CAPABILITIES.items(): From d0ea4e89cc5dd808ea43e2f0bcbb0dc110bda565 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek <8347993+nik13@users.noreply.github.com> Date: Wed, 11 Nov 2026 20:03:33 -0800 Subject: [PATCH 05/15] fix(python): keep LiteLLM media helpers off the instrumentor instance The LiteLLM package suite calls the wrapper methods with a stand-in self, so the media helpers are module functions that read the manager with getattr (no manager: unchanged behavior). Co-Authored-By: Claude Opus 5.5 --- .../litellm/traceai_litellm/__init__.py | 126 +++++++++--------- 1 file changed, 65 insertions(+), 61 deletions(-) diff --git a/python/frameworks/litellm/traceai_litellm/__init__.py b/python/frameworks/litellm/traceai_litellm/__init__.py index ed5061f91..7f11ebcb1 100644 --- a/python/frameworks/litellm/traceai_litellm/__init__.py +++ b/python/frameworks/litellm/traceai_litellm/__init__.py @@ -337,6 +337,53 @@ def _finalize_span(span: trace_api.Span, result: Any) -> None: break +def _media_capture(instrumentor: Any, span: trace_api.Span, arguments: Optional[Mapping[str, Any]] = None) -> Any: + """Capture typed chat media of this call; returns the capture (also for output), or None.""" + attachments = getattr(instrumentor, "_attachments", None) + if attachments is None: + return None + try: + from fi_instrumentation.attachments.provider_media import ProviderMedia + + media = ProviderMedia(attachments, span, "litellm", trace_config=getattr(instrumentor, "_media_config", None)) + if not media.usable: + return None + if arguments is not None: + media.compatible_messages(arguments.get("messages"), "openai") + return media + except Exception: + logger.debug("Attachment capture unavailable", exc_info=True) + return None + + +@contextmanager +def _media_owned(media: Any) -> Iterator[None]: + """Run LiteLLM's downstream SDK call as the owner of this call's media. + + A nested provider instrumentor (for example OpenAI under LiteLLM) then + uploads nothing for the same call. Only set while this adapter captures. + """ + if media is None: + yield + return + from fi_instrumentation.attachments.provider_media import claim + + token = context_api.attach(claim()) + try: + yield + finally: + context_api.detach(token) + + +def _media_generated(media: Any, result: Any) -> None: + if media is None or not isinstance(result, ImageResponse): + return + try: + media.generated_images(result.data) + except Exception: + logger.debug("Attachment capture unavailable", exc_info=True) + + def _attachment_manager(kwargs: Any) -> Any: """The shared manager passed as ``attachments`` (or auto's ``attachment_manager``), if it can admit. @@ -415,49 +462,6 @@ def _uninstrument(self, **kwargs: Any) -> None: self.original_litellm_funcs.clear() self._tracer = None - def _media(self, span: trace_api.Span, arguments: Optional[Mapping[str, Any]] = None) -> Any: - """Capture typed chat media of this call; returns the capture (also for output), or None.""" - if getattr(self, "_attachments", None) is None: - return None - try: - from fi_instrumentation.attachments.provider_media import ProviderMedia - - media = ProviderMedia(self._attachments, span, "litellm", trace_config=self._media_config) - if not media.usable: - return None - if arguments is not None: - media.compatible_messages(arguments.get("messages"), "openai") - return media - except Exception: - logger.debug("Attachment capture unavailable", exc_info=True) - return None - - @contextmanager - def _owned(self, media: Any) -> Iterator[None]: - """Run LiteLLM's downstream SDK call as the owner of this call's media. - - A nested provider instrumentor (for example OpenAI under LiteLLM) then - uploads nothing for the same call. Only set while this adapter captures. - """ - if media is None: - yield - return - from fi_instrumentation.attachments.provider_media import claim - - token = context_api.attach(claim()) - try: - yield - finally: - context_api.detach(token) - - def _generated(self, media: Any, result: Any) -> None: - if media is None or not isinstance(result, ImageResponse): - return - try: - media.generated_images(result.data) - except Exception: - logger.debug("Attachment capture unavailable", exc_info=True) - def _get_call_arguments( self, func_name: str, args: Tuple[Any, ...], kwargs: Dict[str, Any] ) -> Dict[str, Any]: @@ -507,8 +511,8 @@ def _completion_wrapper(self, *args: Any, **kwargs: Any) -> Any: ) as span: try: _instrument_func_type_completion(span, arguments) - media = self._media(span, arguments) - with self._owned(media): + media = _media_capture(self, span, arguments) + with _media_owned(media): result = self.original_litellm_funcs["completion"](*args, **kwargs) _finalize_span(span, result) span.set_status(trace_api.StatusCode.OK) @@ -523,8 +527,8 @@ def _completion_wrapper(self, *args: Any, **kwargs: Any) -> Any: span = span_cm.__enter__() try: _instrument_func_type_completion(span, arguments) - media = self._media(span, arguments) - with self._owned(media): + media = _media_capture(self, span, arguments) + with _media_owned(media): original_iterator = self.original_litellm_funcs["completion"]( *args, **kwargs ) @@ -545,8 +549,8 @@ async def _acompletion_wrapper(self, *args: Any, **kwargs: Any) -> Any: ) as span: try: _instrument_func_type_completion(span, arguments) - media = self._media(span, arguments) - with self._owned(media): + media = _media_capture(self, span, arguments) + with _media_owned(media): result = await self.original_litellm_funcs["acompletion"]( *args, **kwargs ) @@ -563,8 +567,8 @@ async def _acompletion_wrapper(self, *args: Any, **kwargs: Any) -> Any: span = span_cm.__enter__() try: _instrument_func_type_completion(span, arguments) - media = self._media(span, arguments) - with self._owned(media): + media = _media_capture(self, span, arguments) + with _media_owned(media): original_async_iterator = await self.original_litellm_funcs[ "acompletion" ](*args, **kwargs) @@ -591,8 +595,8 @@ def _completion_with_retries_wrapper( ) as span: try: _instrument_func_type_completion(span, arguments) - media = self._media(span, arguments) - with self._owned(media): + media = _media_capture(self, span, arguments) + with _media_owned(media): result = self.original_litellm_funcs["completion_with_retries"]( *args, **kwargs ) @@ -618,8 +622,8 @@ async def _acompletion_with_retries_wrapper( ) as span: try: _instrument_func_type_completion(span, arguments) - media = self._media(span, arguments) - with self._owned(media): + media = _media_capture(self, span, arguments) + with _media_owned(media): result = await self.original_litellm_funcs[ "acompletion_with_retries" ](*args, **kwargs) @@ -684,12 +688,12 @@ def _image_generation_wrapper(self, *args: Any, **kwargs: Any) -> ImageResponse: ) as span: try: _instrument_func_type_image_generation(span, arguments, args) - media = self._media(span) - with self._owned(media): + media = _media_capture(self, span) + with _media_owned(media): result = self.original_litellm_funcs["image_generation"]( *args, **kwargs ) - self._generated(media, result) + _media_generated(media, result) _finalize_span(span, result) span.set_status(trace_api.StatusCode.OK) return result # type:ignore @@ -711,12 +715,12 @@ async def _aimage_generation_wrapper( ) as span: try: _instrument_func_type_image_generation(span, arguments, args) - media = self._media(span) - with self._owned(media): + media = _media_capture(self, span) + with _media_owned(media): result = await self.original_litellm_funcs["aimage_generation"]( *args, **kwargs ) - self._generated(media, result) + _media_generated(media, result) _finalize_span(span, result) span.set_status(trace_api.StatusCode.OK) return result # type:ignore From af66415a1f0590336e2bfb644cd979a484406cd0 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek <8347993+nik13@users.noreply.github.com> Date: Wed, 11 Nov 2026 20:16:46 -0800 Subject: [PATCH 06/15] ci(python): run every compatible-chat row and fail when SDK rows skip A file plus a node ID in the same file collapsed to the node ID, so the compatible-chat jobs only ran the descriptor rows. Each job now also requires its minimum number of passed rows, and the step runs with pipefail. Co-Authored-By: Claude Opus 5.5 --- ...achments-python-compat-providers-tests.yml | 72 +++++++++++-------- 1 file changed, 41 insertions(+), 31 deletions(-) diff --git a/.github/workflows/attachments-python-compat-providers-tests.yml b/.github/workflows/attachments-python-compat-providers-tests.yml index bee9db83c..90f27dee7 100644 --- a/.github/workflows/attachments-python-compat-providers-tests.yml +++ b/.github/workflows/attachments-python-compat-providers-tests.yml @@ -60,35 +60,35 @@ jobs: fail-fast: false matrix: include: - - {package: core, tier: latest, python: '3.12', sdks: '', tests: core} - - {package: google-genai, tier: floor, python: '3.10', sdks: 'google-genai==1.8.0', tests: google} - - {package: google-genai, tier: latest, python: '3.12', sdks: 'google-genai==3.2.0', tests: google} - - {package: vertexai, tier: floor, python: '3.10', sdks: 'google-cloud-aiplatform==1.61.0', tests: google} - - {package: vertexai, tier: latest, python: '3.12', sdks: 'google-cloud-aiplatform==1.101.0', tests: google} - - {package: bedrock, tier: floor, python: '3.10', sdks: 'boto3==1.38.17', tests: aws} - - {package: bedrock, tier: latest, python: '3.12', sdks: 'boto3==1.44.1', tests: aws} - - {package: litellm, tier: floor, python: '3.10', sdks: '-e python/frameworks/openai litellm==1.53.1 openai==1.69.0', tests: aws} - - {package: litellm, tier: latest, python: '3.12', sdks: '-e python/frameworks/openai litellm==1.109.1', tests: aws} - - {package: groq, tier: floor, python: '3.10', sdks: 'groq==0.9.0 httpx<0.28', tests: compat} - - {package: groq, tier: latest, python: '3.12', sdks: 'groq==1.7.0', tests: compat} - - {package: together, tier: floor, python: '3.10', sdks: 'together==1.3.0', tests: compat} - - {package: together, tier: latest, python: '3.12', sdks: 'together==3.1.1', tests: compat} - - {package: fireworks, tier: floor, python: '3.10', sdks: 'openai==1.69.0 httpx<0.28', tests: compat} - - {package: fireworks, tier: latest, python: '3.12', sdks: 'openai==3.49.0', tests: compat} - - {package: xai, tier: floor, python: '3.10', sdks: 'openai==1.69.0 httpx<0.28', tests: compat} - - {package: xai, tier: latest, python: '3.12', sdks: 'openai==3.49.0', tests: compat} - - {package: deepseek, tier: floor, python: '3.10', sdks: 'openai==1.69.0 httpx<0.28', tests: compat} - - {package: deepseek, tier: latest, python: '3.12', sdks: 'openai==3.49.0', tests: compat} - - {package: cerebras, tier: floor, python: '3.10', sdks: 'cerebras-cloud-sdk==1.0.0', tests: compat} - - {package: cerebras, tier: latest, python: '3.12', sdks: 'cerebras-cloud-sdk==1.91.0', tests: compat} - - {package: mistralai, tier: floor, python: '3.10', sdks: 'mistralai==1.1.0', tests: compat} - - {package: mistralai, tier: latest, python: '3.12', sdks: 'mistralai==3.7.0', tests: compat} - - {package: openrouter, tier: floor, python: '3.10', sdks: 'openrouter==1.3.0', tests: compat} - - {package: openrouter, tier: latest, python: '3.12', sdks: 'openrouter==1.3.107', tests: compat} - - {package: portkey, tier: floor, python: '3.10', sdks: 'portkey-ai==1.9.0 httpx<0.28', tests: compat} - - {package: portkey, tier: latest, python: '3.12', sdks: 'portkey-ai==2.3.4', tests: compat} - - {package: ollama, tier: floor, python: '3.10', sdks: 'ollama==0.3.0', tests: compat} - - {package: ollama, tier: latest, python: '3.12', sdks: 'ollama==0.6.3', tests: compat} + - {package: core, tier: latest, python: '3.12', sdks: '', tests: core, min: 27} + - {package: google-genai, tier: floor, python: '3.10', sdks: 'google-genai==1.8.0', tests: google, min: 9} + - {package: google-genai, tier: latest, python: '3.12', sdks: 'google-genai==3.2.0', tests: google, min: 9} + - {package: vertexai, tier: floor, python: '3.10', sdks: 'google-cloud-aiplatform==1.61.0', tests: google, min: 5} + - {package: vertexai, tier: latest, python: '3.12', sdks: 'google-cloud-aiplatform==1.101.0', tests: google, min: 5} + - {package: bedrock, tier: floor, python: '3.10', sdks: 'boto3==1.38.17', tests: aws, min: 12} + - {package: bedrock, tier: latest, python: '3.12', sdks: 'boto3==1.44.1', tests: aws, min: 12} + - {package: litellm, tier: floor, python: '3.10', sdks: '-e python/frameworks/openai litellm==1.53.1 openai==1.69.0', tests: aws, min: 9} + - {package: litellm, tier: latest, python: '3.12', sdks: '-e python/frameworks/openai litellm==1.109.1', tests: aws, min: 9} + - {package: groq, tier: floor, python: '3.10', sdks: 'groq==0.9.0 httpx<0.28', tests: compat, min: 2} + - {package: groq, tier: latest, python: '3.12', sdks: 'groq==1.7.0', tests: compat, min: 2} + - {package: together, tier: floor, python: '3.10', sdks: 'together==1.3.0', tests: compat, min: 3} + - {package: together, tier: latest, python: '3.12', sdks: 'together==3.1.1', tests: compat, min: 3} + - {package: fireworks, tier: floor, python: '3.10', sdks: 'openai==1.69.0 httpx<0.28', tests: compat, min: 3} + - {package: fireworks, tier: latest, python: '3.12', sdks: 'openai==3.49.0', tests: compat, min: 3} + - {package: xai, tier: floor, python: '3.10', sdks: 'openai==1.69.0 httpx<0.28', tests: compat, min: 3} + - {package: xai, tier: latest, python: '3.12', sdks: 'openai==3.49.0', tests: compat, min: 3} + - {package: deepseek, tier: floor, python: '3.10', sdks: 'openai==1.69.0 httpx<0.28', tests: compat, min: 3} + - {package: deepseek, tier: latest, python: '3.12', sdks: 'openai==3.49.0', tests: compat, min: 3} + - {package: cerebras, tier: floor, python: '3.10', sdks: 'cerebras-cloud-sdk==1.0.0 httpx<0.28', tests: compat, min: 3} + - {package: cerebras, tier: latest, python: '3.12', sdks: 'cerebras-cloud-sdk==1.91.0', tests: compat, min: 3} + - {package: mistralai, tier: floor, python: '3.10', sdks: 'mistralai==1.1.0', tests: compat, min: 5} + - {package: mistralai, tier: latest, python: '3.12', sdks: 'mistralai==3.7.0', tests: compat, min: 5} + - {package: openrouter, tier: floor, python: '3.10', sdks: 'openrouter==1.3.0', tests: compat, min: 3} + - {package: openrouter, tier: latest, python: '3.12', sdks: 'openrouter==1.3.107', tests: compat, min: 3} + - {package: portkey, tier: floor, python: '3.10', sdks: 'portkey-ai==1.9.0 httpx<0.28', tests: compat, min: 3} + - {package: portkey, tier: latest, python: '3.12', sdks: 'portkey-ai==2.3.4', tests: compat, min: 3} + - {package: ollama, tier: floor, python: '3.10', sdks: 'ollama==0.3.0', tests: compat, min: 4} + - {package: ollama, tier: latest, python: '3.12', sdks: 'ollama==0.6.3', tests: compat, min: 4} steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: @@ -108,8 +108,11 @@ jobs: .venv/bin/pip install -e python $integration opentelemetry-sdk 'pytest>=8,<9' 'pytest-asyncio<0.25' $SDKS .venv/bin/pip freeze | grep -iE '^(google-genai|google-cloud-aiplatform|boto3|litellm|openai|groq|together|cerebras-cloud-sdk|mistralai|openrouter|portkey-ai|ollama|httpx)==' || true - name: Run with outbound network disabled + # bash -eo pipefail: a failing pytest is not hidden by tee. + shell: bash env: SUITE: ${{ matrix.tests }} + MIN: ${{ matrix.min }} run: | case "$SUITE" in core) tests="tests/attachments_compat_providers/test_provider_media.py" ;; @@ -117,8 +120,12 @@ jobs: aws) tests="tests/attachments_compat_providers/test_bedrock_litellm_attachments.py" ;; compat) tests="tests/attachments_compat_providers/test_compat_attachments.py" ;; esac + # A file and a node ID inside the same file would collapse to the node ID only, + # so the descriptor rows are added only where the compat file is not already selected. + case "$SUITE" in + google|aws) tests="$tests tests/attachments_compat_providers/test_compat_attachments.py::test_package_descriptor" ;; + esac if [ "$SUITE" != core ]; then - tests="$tests tests/attachments_compat_providers/test_compat_attachments.py::test_package_descriptor" tests="$tests tests/attachments_compat_providers/test_provider_media.py::test_disabled_path_never_imports_attachment_modules" fi sudo unshare --net -- bash -e -c ' @@ -128,4 +135,7 @@ jobs: export LITELLM_LOCAL_MODEL_COST_MAP=True AWS_EC2_METADATA_DISABLED=true # shellcheck disable=SC2086 PYTHONPATH=. "$1" -m pytest $3 -v -rs -p no:cacheprovider -o addopts="" -o faulthandler_timeout=180 - ' _ "$PWD/.venv/bin/python" "$PWD" "$tests" + ' _ "$PWD/.venv/bin/python" "$PWD" "$tests" | tee pytest.log + # A job whose SDK rows silently skipped must not pass: require the rows this package proves. + passed=$(grep -Eo '[0-9]+ passed' pytest.log | tail -1 | cut -d' ' -f1) + if [ "${passed:-0}" -lt "$MIN" ]; then echo "expected at least $MIN passed, got ${passed:-0}"; exit 1; fi From 15693afecffb046a24b42657df4fe063e63a00d0 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek <8347993+nik13@users.noreply.github.com> Date: Wed, 11 Nov 2026 20:30:54 -0800 Subject: [PATCH 07/15] test(python): untyped sources stay image-only; per-observation part cap is exact Co-Authored-By: Claude Opus 5.5 --- .../test_provider_media.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/python/tests/attachments_compat_providers/test_provider_media.py b/python/tests/attachments_compat_providers/test_provider_media.py index 71f4dec64..ded7f3e07 100644 --- a/python/tests/attachments_compat_providers/test_provider_media.py +++ b/python/tests/attachments_compat_providers/test_provider_media.py @@ -169,7 +169,8 @@ def wrapped(file, *args, **kwargs): monkeypatch.setattr(builtins, "open", watch(real_open)) monkeypatch.setattr(io, "open", watch(real_io_open)) - images = [IMAGE, b64(jpeg(7)), str(path), path, "not base64!"] + # A PDF has no image signature: untyped sources are images only, so it is never uploaded. + images = [IMAGE, b64(jpeg(7)), str(path), path, "not base64!", PDF, b64(PDF)] media, _ = capture(manager, tracer_provider, lambda m: m.ollama_images(images)) flush(manager) monkeypatch.undo() @@ -264,10 +265,25 @@ def test_second_observer_of_same_span_uploads_nothing(f01, spans): def test_part_cap_bounds_work(f01, spans): tracer_provider, _ = spans manager = f01.manager() + calls = [] + preflight = manager.preflight + + def counted(*args, **kwargs): + calls.append(kwargs.get("mime_type")) + return preflight(*args, **kwargs) + + manager.preflight = counted content = [{"type": "image_url", "image_url": {"url": data_uri(png(seed=i))}} for i in range(80)] media, _ = capture(manager, tracer_provider, lambda m: m.compatible_messages([{"role": "user", "content": content}])) flush(manager) assert len(f01.payloads) == provider_media.MAX_PARTS == len(media.captured) + # One preflight per admitted part (exact type and size already match the gate), none past the cap. + assert len(calls) == provider_media.MAX_PARTS + del manager.preflight + media, _ = capture(manager, tracer_provider, lambda m: m.compatible_messages( + [{"role": "user", "content": [{"type": "input_audio", "input_audio": {"data": b64(wav()), "format": "wav"}}] * 80}])) + flush(manager) + assert len(media.captured) == provider_media.MAX_PARTS manager.shutdown(5) From 73727e7aed07ec4eefba54951888da6e1876a56a Mon Sep 17 00:00:00 2001 From: Nikhil Pareek <8347993+nik13@users.noreply.github.com> Date: Wed, 11 Nov 2026 20:34:08 -0800 Subject: [PATCH 08/15] test(python): exercise the per-observation part cap across messages Co-Authored-By: Claude Opus 5.5 --- .../attachments_compat_providers/test_provider_media.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/python/tests/attachments_compat_providers/test_provider_media.py b/python/tests/attachments_compat_providers/test_provider_media.py index ded7f3e07..991f48b66 100644 --- a/python/tests/attachments_compat_providers/test_provider_media.py +++ b/python/tests/attachments_compat_providers/test_provider_media.py @@ -273,15 +273,18 @@ def counted(*args, **kwargs): return preflight(*args, **kwargs) manager.preflight = counted - content = [{"type": "image_url", "image_url": {"url": data_uri(png(seed=i))}} for i in range(80)] - media, _ = capture(manager, tracer_provider, lambda m: m.compatible_messages([{"role": "user", "content": content}])) + # 80 parts over two messages: each list is within its own bound, so only the per-observation cap stops it. + messages = [{"role": "user", "content": [{"type": "image_url", "image_url": {"url": data_uri(png(seed=m * 40 + i))}} + for i in range(40)]} for m in range(2)] + media, _ = capture(manager, tracer_provider, lambda m: m.compatible_messages(messages)) flush(manager) assert len(f01.payloads) == provider_media.MAX_PARTS == len(media.captured) # One preflight per admitted part (exact type and size already match the gate), none past the cap. assert len(calls) == provider_media.MAX_PARTS del manager.preflight media, _ = capture(manager, tracer_provider, lambda m: m.compatible_messages( - [{"role": "user", "content": [{"type": "input_audio", "input_audio": {"data": b64(wav()), "format": "wav"}}] * 80}])) + [{"role": "user", "content": [{"type": "input_audio", "input_audio": {"data": b64(wav(seed=i)), "format": "wav"}} + for i in range(40)]}] * 2)) flush(manager) assert len(media.captured) == provider_media.MAX_PARTS manager.shutdown(5) From b68e06566b8ca274bafbe1e1e67d66cd189ce99c Mon Sep 17 00:00:00 2001 From: Nikhil Pareek <8347993+nik13@users.noreply.github.com> Date: Wed, 11 Nov 2026 22:13:22 -0800 Subject: [PATCH 09/15] fix(python): provider media review findings (LiteLLM PDF parts, xAI owner, vLLM/MiniMax bindings, Vertex descriptor, MP4 brand, data-URI bound, InvokeModel part index) - LiteLLM claims the call from a nested OpenAI instrumentor, so it now captures PDF file/input_file data URIs itself (documents=True, LiteLLM only): one upload. - traceai_xai defers to an enabled traceai_openai like fireworks: one owner per call. - Thin vLLM (OpenAI dialect) and MiniMax (image dialect) bindings with descriptors; MiniMax skips media while traceai_openai is enabled. - The vertexai descriptor advertises the proven PredictionServiceClient paths only. - sniff() accepts video/mp4 only for MP4 ISO-BMFF major brands (not HEIC/AVIF/QT/M4A). - Data URIs go through media_capture.split_inline: bounded before slicing. - Bedrock InvokeModel tool_result children carry the block index, as Converse does. Co-Authored-By: Claude Opus 5.5 --- ...achments-python-compat-providers-tests.yml | 20 ++- .../attachments/provider_media.py | 73 +++++++---- .../litellm/traceai_litellm/__init__.py | 3 +- .../minimax/traceai_minimax/__init__.py | 31 ++++- .../minimax/traceai_minimax/_wrappers.py | 39 +++++- .../frameworks/vllm/traceai_vllm/__init__.py | 33 ++++- .../frameworks/vllm/traceai_vllm/_wrappers.py | 19 +++ .../frameworks/xai/traceai_xai/_wrappers.py | 20 ++- .../test_bedrock_litellm_attachments.py | 35 +++++- .../test_compat_attachments.py | 117 ++++++++++++++++++ .../test_provider_media.py | 97 ++++++++++++++- 11 files changed, 446 insertions(+), 41 deletions(-) diff --git a/.github/workflows/attachments-python-compat-providers-tests.yml b/.github/workflows/attachments-python-compat-providers-tests.yml index 90f27dee7..df49c0861 100644 --- a/.github/workflows/attachments-python-compat-providers-tests.yml +++ b/.github/workflows/attachments-python-compat-providers-tests.yml @@ -22,6 +22,9 @@ on: - 'python/frameworks/openrouter/**' - 'python/frameworks/portkey/**' - 'python/frameworks/ollama/**' + - 'python/frameworks/vllm/**' + - 'python/frameworks/minimax/**' + - 'python/frameworks/openai/**' - 'python/tests/attachments_providers/conftest.py' - 'python/tests/attachments_compat_providers/**' - '.github/workflows/attachments-python-compat-providers-tests.yml' @@ -43,6 +46,9 @@ on: - 'python/frameworks/openrouter/**' - 'python/frameworks/portkey/**' - 'python/frameworks/ollama/**' + - 'python/frameworks/vllm/**' + - 'python/frameworks/minimax/**' + - 'python/frameworks/openai/**' - 'python/tests/attachments_providers/conftest.py' - 'python/tests/attachments_compat_providers/**' - '.github/workflows/attachments-python-compat-providers-tests.yml' @@ -60,23 +66,27 @@ jobs: fail-fast: false matrix: include: - - {package: core, tier: latest, python: '3.12', sdks: '', tests: core, min: 27} + - {package: core, tier: latest, python: '3.12', sdks: '', tests: core, min: 45} - {package: google-genai, tier: floor, python: '3.10', sdks: 'google-genai==1.8.0', tests: google, min: 9} - {package: google-genai, tier: latest, python: '3.12', sdks: 'google-genai==3.2.0', tests: google, min: 9} - {package: vertexai, tier: floor, python: '3.10', sdks: 'google-cloud-aiplatform==1.61.0', tests: google, min: 5} - {package: vertexai, tier: latest, python: '3.12', sdks: 'google-cloud-aiplatform==1.101.0', tests: google, min: 5} - {package: bedrock, tier: floor, python: '3.10', sdks: 'boto3==1.38.17', tests: aws, min: 12} - {package: bedrock, tier: latest, python: '3.12', sdks: 'boto3==1.44.1', tests: aws, min: 12} - - {package: litellm, tier: floor, python: '3.10', sdks: '-e python/frameworks/openai litellm==1.53.1 openai==1.69.0', tests: aws, min: 9} - - {package: litellm, tier: latest, python: '3.12', sdks: '-e python/frameworks/openai litellm==1.109.1', tests: aws, min: 9} + - {package: litellm, tier: floor, python: '3.10', sdks: '-e python/frameworks/openai litellm==1.53.1 openai==1.69.0', tests: aws, min: 10} + - {package: litellm, tier: latest, python: '3.12', sdks: '-e python/frameworks/openai litellm==1.109.1', tests: aws, min: 10} - {package: groq, tier: floor, python: '3.10', sdks: 'groq==0.9.0 httpx<0.28', tests: compat, min: 2} - {package: groq, tier: latest, python: '3.12', sdks: 'groq==1.7.0', tests: compat, min: 2} - {package: together, tier: floor, python: '3.10', sdks: 'together==1.3.0', tests: compat, min: 3} - {package: together, tier: latest, python: '3.12', sdks: 'together==3.1.1', tests: compat, min: 3} - {package: fireworks, tier: floor, python: '3.10', sdks: 'openai==1.69.0 httpx<0.28', tests: compat, min: 3} - {package: fireworks, tier: latest, python: '3.12', sdks: 'openai==3.49.0', tests: compat, min: 3} - - {package: xai, tier: floor, python: '3.10', sdks: 'openai==1.69.0 httpx<0.28', tests: compat, min: 3} - - {package: xai, tier: latest, python: '3.12', sdks: 'openai==3.49.0', tests: compat, min: 3} + - {package: xai, tier: floor, python: '3.10', sdks: '-e python/frameworks/openai openai==1.69.0 httpx<0.28', tests: compat, min: 5} + - {package: xai, tier: latest, python: '3.12', sdks: '-e python/frameworks/openai openai==3.49.0', tests: compat, min: 5} + - {package: vllm, tier: floor, python: '3.10', sdks: '-e python/frameworks/openai openai==1.69.0 httpx<0.28', tests: compat, min: 4} + - {package: vllm, tier: latest, python: '3.12', sdks: '-e python/frameworks/openai openai==3.49.0', tests: compat, min: 4} + - {package: minimax, tier: floor, python: '3.10', sdks: '-e python/frameworks/openai openai==1.69.0 httpx<0.28', tests: compat, min: 6} + - {package: minimax, tier: latest, python: '3.12', sdks: '-e python/frameworks/openai openai==3.49.0', tests: compat, min: 6} - {package: deepseek, tier: floor, python: '3.10', sdks: 'openai==1.69.0 httpx<0.28', tests: compat, min: 3} - {package: deepseek, tier: latest, python: '3.12', sdks: 'openai==3.49.0', tests: compat, min: 3} - {package: cerebras, tier: floor, python: '3.10', sdks: 'cerebras-cloud-sdk==1.0.0 httpx<0.28', tests: compat, min: 3} diff --git a/python/fi_instrumentation/attachments/provider_media.py b/python/fi_instrumentation/attachments/provider_media.py index a003967d1..8ee0ae2fa 100644 --- a/python/fi_instrumentation/attachments/provider_media.py +++ b/python/fi_instrumentation/attachments/provider_media.py @@ -73,6 +73,11 @@ _IMAGE_GENERATION_MODELS = re.compile( r"(?:arn:aws[\w-]*:bedrock:[\w-]+:\d*:(?:inference-profile|foundation-model)/)?" r"(?:(?:us|eu|apac|global)\.)?amazon\.(?:titan-image-generator-v[12]|nova-canvas-v1)", re.IGNORECASE) +# ISO-BMFF major brands (``ftyp`` bytes 8-12) that name an MP4 video file. HEIF/HEIC and AVIF still +# images, QuickTime, 3GPP and audio-only M4A share the box layout but are not video/mp4, so they are +# not admitted under that label (fail closed: an unlisted brand is unknown). +_MP4_BRANDS = frozenset((b"isom", b"iso2", b"iso3", b"iso4", b"iso5", b"iso6", b"mp41", b"mp42", b"avc1", + b"dash", b"M4V ", b"mmp4")) _COMPUTER_TOOL = re.compile(r"computer_\d{8}") # Vertex publisher image-generation models (Imagen). Other prediction endpoints are never read. _IMAGEN_ENDPOINT = re.compile(r"/models/(?:imagen-|imagegeneration)") @@ -127,7 +132,7 @@ def sniff(head: Any) -> Optional[str]: if head[:5] == b"%PDF-": return "application/pdf" if len(head) >= 12 and head[4:8] == b"ftyp": - return "video/mp4" + return "video/mp4" if head[8:12] in _MP4_BRANDS else None return mc.sniff_audio(head) @@ -193,14 +198,13 @@ def decode(max_bytes: int) -> Optional[bytes]: return _Source(mime, size, decode) -def _data_uri_source(value: Any) -> Optional[_Source]: - if not isinstance(value, str): +def _data_uri_source(value: Any, types: Tuple[str, ...] = DATA_URI_TYPES) -> Optional[_Source]: + # split_inline bounds the length before it slices, so an oversized string is never copied. + found = mc.split_inline(value, types, False, MAX_DECODED_BYTES) + if found is None: return None - for mime in DATA_URI_TYPES: - prefix = "data:%s;base64," % mime - if value.startswith(prefix): - return _base64_source(value[len(prefix):], mime) - return None + mime, encoded, _ = found + return _base64_source(encoded, mime) # ------------------------------------------------------------------ capture @@ -311,10 +315,11 @@ def _untyped_image(self, position: str, data: Any, **fields) -> None: if size: self._admit(position, "image/png", size, lambda: _image_only(_base64_source(data, None)), **fields) - def _data_uri(self, position: str, read: Callable[[], Any], **fields) -> None: - # Every candidate type is an image, so one image gate decides the part before the + def _data_uri(self, position: str, read: Callable[[], Any], types: Tuple[str, ...] = DATA_URI_TYPES, + **fields) -> None: + # Every candidate type shares one modality, so one gate decides the part before the # string is parsed. A remote or signed URL is never kept as fallback metadata. - self._admit(position, DATA_URI_TYPES[0], MAX_DECODED_BYTES, lambda: _data_uri_source(read()), exact=False, + self._admit(position, types[0], MAX_DECODED_BYTES, lambda: _data_uri_source(read(), types), exact=False, **fields) # -------------------------------------------------------------- Gemini @@ -356,11 +361,14 @@ def gemini_response(self, response: Any) -> "ProviderMedia": return self # -------------------------------------------------------------- OpenAI-compatible chat - def compatible_messages(self, messages: Any, dialect: str = "openai") -> "ProviderMedia": + def compatible_messages(self, messages: Any, dialect: str = "openai", *, + documents: bool = False) -> "ProviderMedia": """Chat ``image_url`` data URIs, plus ``input_audio`` (wav/mp3) for OpenAI-dialect routes. ``groq``, ``mistral`` and ``image`` dialects are image only; ``mistral`` also accepts - ``image_url`` as a plain string. + ``image_url`` as a plain string. ``documents=True`` (OpenAI dialect only, for a route whose + PDF parts are qualified) also captures PDF ``file`` (``file.file_data``) and ``input_file`` + (``file_data``) data URIs. Provider file IDs and URLs are never resolved. """ if not self.usable: return self @@ -383,6 +391,12 @@ def read(part=part): mime = _COMPAT_AUDIO.get(mc.get(audio, "format")) if mime is not None: self._typed("chat/%d/%d" % (m, p), mc.get(audio, "data"), mime, **fields) + elif kind in ("file", "input_file") and documents and dialect == "openai": + def read_file(part=part, kind=kind): + return mc.get(mc.get(part, "file") if kind == "file" else part, "file_data") + + fields["purpose"] = "document" + self._data_uri("chat/%d/%d" % (m, p), read_file, DOCUMENT_TYPES, **fields) except Exception: pass return self @@ -493,8 +507,9 @@ def anthropic_messages(self, messages: Any, tools: Any = None) -> "ProviderMedia if not isinstance(tool_id, str) or not mc.TOOL_ID.fullmatch(tool_id): continue for q, child in enumerate(_items(mc.get(block, "content"))[:MAX_PARTS]): + # part_index is the tool_result block, as for Converse; q only keeps positions apart. self._anthropic_block("anthropic/%d/%d/%d" % (m, p, q), child, role="tool", - message_index=m, part_index=q, tool_call_id=tool_id, + message_index=m, part_index=p, tool_call_id=tool_id, screenshot=tool_id in computer, error_detail=mc.get(block, "is_error") is True) else: @@ -655,12 +670,19 @@ def _descriptor(operations: Tuple[str, ...], sources: Tuple[str, ...], unsupport ("sdk_bytes", "inline_base64"), ("file_data_uri", "gs_uri", "provider_file", "pil_image", "streamed_output", "models.generate_images", "models.generate_videos", "live", "audio/L16")), + # Proven: the sync gapic PredictionServiceClient (REST and gRPC share the wrapped method). The + # high-level vertexai GenerativeModel / ImageGenerationModel routes run through the same client + # but have no fixture yet, so they are not claimed. "traceai_vertexai": _descriptor( - ("GenerativeModel.generate_content.input", "GenerativeModel.generate_content.output", - "ImageGenerationModel.generate_images.output"), - ("sdk_bytes",), - ("file_data_uri", "gs_uri", "streamed_output", "ImageGenerationModel.edit_image", - "prediction_service_raw", "audio/L16")), + ("PredictionServiceClient.generate_content.input", "PredictionServiceClient.generate_content.output", + "PredictionServiceClient.predict.imagen.output"), + ("sdk_bytes", "inline_base64"), + ("file_data_uri", "gs_uri", "streamed_output", "PredictionServiceAsyncClient", + "PredictionServiceClient.stream_generate_content", "PredictionServiceClient.predict.other_endpoints", + "GenerativeModel.generate_content", "ImageGenerationModel.generate_images", + "ImageGenerationModel.edit_image", "audio/L16"), + "GenerativeModel and ImageGenerationModel are not qualified yet; only the PredictionServiceClient " + "paths above are proven."), "traceai_bedrock": _descriptor( ("converse.input", "invoke_model.anthropic.input", "invoke_model.nova.input", "invoke_model.image_generation.input", "invoke_model.image_generation.output"), @@ -670,8 +692,10 @@ def _descriptor(operations: Tuple[str, ...], sources: Tuple[str, ...], unsupport "traceai_litellm": _descriptor( ("completion.input", "acompletion.input", "image_generation.output", "aimage_generation.output"), ("data_uri", "inline_base64"), - ("remote_url", "provider_file", "streamed_output", "image_generation.url", "image_edit", - "speech", "transcription")), + ("remote_url", "provider_file", "file_id", "streamed_output", "image_generation.url", "image_edit", + "responses", "speech", "transcription"), + "Chat input covers image_url, input_audio and PDF file/input_file data URIs. LiteLLM claims the call, " + "so a nested OpenAI instrumentor uploads nothing more; application/octet-stream files stay uncaptured."), "traceai_groq": _descriptor(("chat.completions.create.input",), ("data_uri",), _COMPAT_UNSUPPORTED), # The binding is wired, but traceai_together installs no chat wrapper today (its install check looks # for `chat` on the Together class, where it is an instance attribute), so nothing is claimed. @@ -686,6 +710,13 @@ def _descriptor(operations: Tuple[str, ...], sources: Tuple[str, ...], unsupport ("image_path", "streamed_output", "output_media")), "traceai_portkey": _descriptor(("chat.completions.create.input",), ("data_uri", "inline_base64"), _COMPAT_UNSUPPORTED), + # Legacy VLLMInstrumentor (OpenAI-dialect chat: image_url and input_audio). It already defers to + # traceai_openai while that is enabled, which then owns the call's media. + "traceai_vllm": _descriptor(("chat.completions.create.input",), ("data_uri", "inline_base64"), + _COMPAT_UNSUPPORTED + ("video_url",)), + # MiniMax chat takes image_url (M3 / M3.1-Flash); no audio input. Native speech/image/video is separate. + "traceai_minimax": _descriptor(("chat.completions.create.input",), ("data_uri",), + _COMPAT_UNSUPPORTED + ("video_url", "input_audio", "native_media")), "traceai_deepseek": _descriptor((), (), ("chat.completions.create",), "DeepSeek chat has no qualified media input or output; tracing only."), "traceai_cerebras": _descriptor((), (), ("chat.completions.create",), diff --git a/python/frameworks/litellm/traceai_litellm/__init__.py b/python/frameworks/litellm/traceai_litellm/__init__.py index 7f11ebcb1..5a4704592 100644 --- a/python/frameworks/litellm/traceai_litellm/__init__.py +++ b/python/frameworks/litellm/traceai_litellm/__init__.py @@ -349,7 +349,8 @@ def _media_capture(instrumentor: Any, span: trace_api.Span, arguments: Optional[ if not media.usable: return None if arguments is not None: - media.compatible_messages(arguments.get("messages"), "openai") + # PDF file parts too: LiteLLM claims the call, so the nested OpenAI instrumentor skips them. + media.compatible_messages(arguments.get("messages"), "openai", documents=True) return media except Exception: logger.debug("Attachment capture unavailable", exc_info=True) diff --git a/python/frameworks/minimax/traceai_minimax/__init__.py b/python/frameworks/minimax/traceai_minimax/__init__.py index f3b1e6fe2..8517357b6 100644 --- a/python/frameworks/minimax/traceai_minimax/__init__.py +++ b/python/frameworks/minimax/traceai_minimax/__init__.py @@ -21,6 +21,20 @@ _instruments = ("openai >= 1.0.0",) +def _attachment_manager(kwargs: Any) -> Any: + """The shared manager passed as ``attachments`` (or auto's ``attachment_manager``), if it can admit. + + A missing, disabled or pre-attachments core leaves tracing exactly as before. + """ + if kwargs.get("attachments") is None and kwargs.get("attachment_manager") is None: + return None + try: + from fi_instrumentation.attachments.provider_media import manager_from_kwargs + except ImportError: + return None + return manager_from_kwargs(kwargs) + + class MiniMaxInstrumentor(BaseInstrumentor): """ OpenTelemetry instrumentor for MiniMax. @@ -78,7 +92,9 @@ def _instrument(self, **kwargs: Any) -> None: wrap_function_wrapper( module="openai.resources.chat.completions", name="Completions.create", - wrapper=_ChatCompletionWrapper(tracer=self._tracer), + wrapper=_ChatCompletionWrapper( + tracer=self._tracer, attachments=_attachment_manager(kwargs), config=config + ), ) # Async client @@ -89,7 +105,9 @@ def _instrument(self, **kwargs: Any) -> None: wrap_function_wrapper( module="openai.resources.chat.completions", name="AsyncCompletions.create", - wrapper=_AsyncChatCompletionWrapper(tracer=self._tracer), + wrapper=_AsyncChatCompletionWrapper( + tracer=self._tracer, attachments=_attachment_manager(kwargs), config=config + ), ) except Exception as e: logger.warning(f"Failed to instrument OpenAI for MiniMax: {e}") @@ -104,3 +122,12 @@ def _uninstrument(self, **kwargs: Any) -> None: completions.AsyncCompletions.create = self._original_async_chat_create except Exception as e: logger.warning(f"Failed to uninstrument: {e}") + + +def __getattr__(name: str) -> Any: + # Lazy: the descriptor needs a fi-instrumentation-otel release with attachments. + if name == "minimax_attachment_capabilities": + from fi_instrumentation.attachments.provider_media import capabilities + + return capabilities("traceai_minimax") + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/python/frameworks/minimax/traceai_minimax/_wrappers.py b/python/frameworks/minimax/traceai_minimax/_wrappers.py index 6e2eea238..50f64427a 100644 --- a/python/frameworks/minimax/traceai_minimax/_wrappers.py +++ b/python/frameworks/minimax/traceai_minimax/_wrappers.py @@ -1,4 +1,5 @@ import logging +import sys from abc import ABC from contextlib import contextmanager from typing import Any, Callable, Dict, Iterable, Iterator, List, Mapping, Tuple @@ -24,12 +25,46 @@ logger.addHandler(logging.NullHandler()) +def _shared_instrumentor_enabled() -> bool: + """True while traceai_openai's OpenAIInstrumentor is enabled (it then owns the call's media). + + Do not import the optional package; inspect live state on each call, so either + installation order works. + """ + module = sys.modules.get("traceai_openai") + instrumentor = getattr(module, "OpenAIInstrumentor", None) + return bool(instrumentor and instrumentor().is_instrumented_by_opentelemetry) + + class _WithTracer(ABC): """Base class for wrappers that need a tracer.""" - def __init__(self, tracer: trace_api.Tracer, *args: Any, **kwargs: Any) -> None: + def __init__( + self, tracer: trace_api.Tracer, *args: Any, attachments: Any = None, config: Any = None, **kwargs: Any + ) -> None: super().__init__(*args, **kwargs) self._tracer = tracer + # Shared attachment manager, or None (no capture, no imports). + self._attachments = attachments + self._config = config + + def _capture_chat_media(self, span: _WithSpan, messages: Any) -> None: + """Typed chat media (data-URI images) through the shared manager; never raises. + + The OpenAI instrumentor wraps the same SDK method; while it is enabled it captures this + call's media, so this binding uploads nothing (one upload per part, either order). + """ + if self._attachments is None: + return + try: + if _shared_instrumentor_enabled(): + return + from fi_instrumentation.attachments.provider_media import ProviderMedia + + ProviderMedia(self._attachments, span._span, "minimax", trace_config=self._config).compatible_messages( + messages, "image") + except Exception: + logger.debug("Attachment capture unavailable", exc_info=True) @contextmanager def _start_as_current_span( @@ -99,6 +134,7 @@ def __call__( context_attributes=get_attributes_from_context(), extra_attributes=self._request_extractor.get_extra_attributes_from_request(request_parameters), ) as span: + self._capture_chat_media(span, request_parameters.get("messages")) try: response = wrapped(*args, **kwargs) except Exception as exception: @@ -206,6 +242,7 @@ async def __call__( context_attributes=get_attributes_from_context(), extra_attributes=self._request_extractor.get_extra_attributes_from_request(request_parameters), ) as span: + self._capture_chat_media(span, request_parameters.get("messages")) try: response = await wrapped(*args, **kwargs) except Exception as exception: diff --git a/python/frameworks/vllm/traceai_vllm/__init__.py b/python/frameworks/vllm/traceai_vllm/__init__.py index 94413cd86..04753bf3a 100644 --- a/python/frameworks/vllm/traceai_vllm/__init__.py +++ b/python/frameworks/vllm/traceai_vllm/__init__.py @@ -33,6 +33,20 @@ ] +def _attachment_manager(kwargs: Any) -> Any: + """The shared manager passed as ``attachments`` (or auto's ``attachment_manager``), if it can admit. + + A missing, disabled or pre-attachments core leaves tracing exactly as before. + """ + if kwargs.get("attachments") is None and kwargs.get("attachment_manager") is None: + return None + try: + from fi_instrumentation.attachments.provider_media import manager_from_kwargs + except ImportError: + return None + return manager_from_kwargs(kwargs) + + class VLLMInstrumentor(BaseInstrumentor): """An instrumentor for vLLM (OpenAI-compatible API).""" @@ -93,7 +107,10 @@ def _instrument(self, **kwargs: Any) -> None: wrap_function_wrapper( module="openai.resources.chat", name="Completions.create", - wrapper=_CompletionsWrapper(tracer=self._tracer, vllm_base_urls=self._vllm_base_urls), + wrapper=_CompletionsWrapper( + tracer=self._tracer, vllm_base_urls=self._vllm_base_urls, + attachments=_attachment_manager(kwargs), config=config, + ), ) # Wrap asynchronous completions @@ -101,7 +118,10 @@ def _instrument(self, **kwargs: Any) -> None: wrap_function_wrapper( module="openai.resources.chat", name="AsyncCompletions.create", - wrapper=_AsyncCompletionsWrapper(tracer=self._tracer, vllm_base_urls=self._vllm_base_urls), + wrapper=_AsyncCompletionsWrapper( + tracer=self._tracer, vllm_base_urls=self._vllm_base_urls, + attachments=_attachment_manager(kwargs), config=config, + ), ) def _uninstrument(self, **kwargs: Any) -> None: @@ -113,3 +133,12 @@ def _uninstrument(self, **kwargs: Any) -> None: openai_module.AsyncCompletions.create = self._original_async_completions_create except ImportError: pass + + +def __getattr__(name: str) -> Any: + # Lazy: the descriptor needs a fi-instrumentation-otel release with attachments. + if name == "vllm_attachment_capabilities": + from fi_instrumentation.attachments.provider_media import capabilities + + return capabilities("traceai_vllm") + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/python/frameworks/vllm/traceai_vllm/_wrappers.py b/python/frameworks/vllm/traceai_vllm/_wrappers.py index 29a3ab580..2b3aebacc 100644 --- a/python/frameworks/vllm/traceai_vllm/_wrappers.py +++ b/python/frameworks/vllm/traceai_vllm/_wrappers.py @@ -44,11 +44,28 @@ def __init__( tracer: trace_api.Tracer, vllm_base_urls: List[str], *args: Any, + attachments: Any = None, + config: Any = None, **kwargs: Any, ) -> None: super().__init__(*args, **kwargs) self._tracer = tracer self._vllm_base_urls = vllm_base_urls + # Shared attachment manager, or None (no capture, no imports). + self._attachments = attachments + self._config = config + + def _capture_chat_media(self, span: Any, messages: Any) -> None: + """Typed chat media (image_url and input_audio data) through the shared manager; never raises.""" + if self._attachments is None: + return + try: + from fi_instrumentation.attachments.provider_media import ProviderMedia + + ProviderMedia(self._attachments, span, "vllm", trace_config=self._config).compatible_messages( + messages, "openai") + except Exception: + logger.debug("Attachment capture unavailable", exc_info=True) def _is_vllm_client(self, instance: Any) -> bool: """Check if the client instance is configured for vLLM.""" @@ -268,6 +285,7 @@ def __call__( input_attrs = _get_input_attributes(model, messages, **extra_kwargs) with self._start_span("vLLM Chat Completions", input_attrs) as span: + self._capture_chat_media(span, messages) try: response = wrapped(*args, **kwargs) @@ -312,6 +330,7 @@ async def __call__( input_attrs = _get_input_attributes(model, messages, **extra_kwargs) with self._start_span("vLLM Chat Completions", input_attrs) as span: + self._capture_chat_media(span, messages) try: response = await wrapped(*args, **kwargs) diff --git a/python/frameworks/xai/traceai_xai/_wrappers.py b/python/frameworks/xai/traceai_xai/_wrappers.py index adfd97337..a4f335aa7 100644 --- a/python/frameworks/xai/traceai_xai/_wrappers.py +++ b/python/frameworks/xai/traceai_xai/_wrappers.py @@ -1,6 +1,7 @@ """Wrapper classes for xAI (Grok) instrumentation.""" import logging +import sys from abc import ABC from contextlib import contextmanager from typing import Any, Dict, Iterator, Tuple @@ -84,6 +85,17 @@ def _is_xai_client(instance: Any) -> bool: return False +def _shared_instrumentor_enabled() -> bool: + """Let the request-level OpenAI instrumentor own the call when both are enabled. + + Same rule as the Fireworks route: one request has one owner (span and media), + whatever the setup order. Do not import the optional package; inspect live state. + """ + module = sys.modules.get("traceai_openai") + instrumentor = getattr(module, "OpenAIInstrumentor", None) + return bool(instrumentor and instrumentor().is_instrumented_by_opentelemetry) + + def _get_input_attributes( model: str, messages: list, @@ -228,8 +240,8 @@ def __call__( args: Tuple[Any, ...], kwargs: Dict[str, Any], ) -> Any: - # Check if this is an xAI client - if not _is_xai_client(instance): + # One request must have one owner, regardless of setup order. + if _shared_instrumentor_enabled() or not _is_xai_client(instance): return wrapped(*args, **kwargs) # Extract parameters @@ -300,8 +312,8 @@ async def __call__( args: Tuple[Any, ...], kwargs: Dict[str, Any], ) -> Any: - # Check if this is an xAI client - if not _is_xai_client(instance): + # One request must have one owner, regardless of setup order. + if _shared_instrumentor_enabled() or not _is_xai_client(instance): return await wrapped(*args, **kwargs) # Extract parameters diff --git a/python/tests/attachments_compat_providers/test_bedrock_litellm_attachments.py b/python/tests/attachments_compat_providers/test_bedrock_litellm_attachments.py index cc138e21c..3ba6084fb 100644 --- a/python/tests/attachments_compat_providers/test_bedrock_litellm_attachments.py +++ b/python/tests/attachments_compat_providers/test_bedrock_litellm_attachments.py @@ -130,7 +130,7 @@ def test_converse_hide_input_images_keeps_documents(f01, bedrock): def test_invoke_model_anthropic_body(f01, bedrock): manager = f01.manager() - client, _ = bedrock(manager) + client, exporter = bedrock(manager) body = {"anthropic_version": "bedrock-2023-05-31", "max_tokens": 8, "tools": [{"type": "computer_20250124", "name": "computer", "display_width_px": 800, "display_height_px": 600}], @@ -139,15 +139,22 @@ def test_invoke_model_anthropic_body(f01, bedrock): {"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": b64(IMAGE)}}]}, {"role": "assistant", "content": [{"type": "tool_use", "id": "toolu_1", "name": "computer", "input": {"action": "screenshot"}}]}, - {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "toolu_1", "content": [ - {"type": "image", "source": {"type": "base64", "media_type": "image/png", - "data": b64(SHOT_2)}}]}]}, + {"role": "user", "content": [ + {"type": "text", "text": "result follows"}, + {"type": "tool_result", "tool_use_id": "toolu_1", "content": [ + {"type": "text", "text": "screenshot taken"}, + {"type": "text", "text": "below"}, + {"type": "image", "source": {"type": "base64", "media_type": "image/png", + "data": b64(SHOT_2)}}]}]}, ]} response = client.invoke_model(modelId="anthropic.claude-3-5-sonnet-20240620-v1:0", body=json.dumps(body)) assert json.loads(response["body"].read())["content"][0]["text"] == "ok" flush(manager) assert sorted(f01.payloads) == sorted([IMAGE, SHOT_2]) assert sorted(u["purpose"] for u in f01.uploads) == ["computer_screenshot", "model_input"] + # As for Converse, a tool_result child carries the tool_result block's index (1), not its own (2). + shot = [r for s in exporter.get_finished_spans() for r in refs(s) if r.get("purpose") == "computer_screenshot"] + assert [(r["message_index"], r["part_index"], r["tool_call_id"]) for r in shot] == [(2, 1, "toolu_1")] manager.shutdown(5) @@ -270,6 +277,26 @@ def test_litellm_owns_media_under_nested_openai_instrumentor(f01, media_provider manager.shutdown(5) +def test_litellm_pdf_file_part_uploads_once_under_nested_openai(f01, media_provider, litellm_instrument): + """LiteLLM claims the call, so it captures PDF ``file`` parts itself: exactly one upload, on its span.""" + pytest.importorskip("traceai_openai") + import litellm + + manager = f01.manager() + exporter = litellm_instrument(manager, TraceConfig(hide_input_images=True), with_openai=True) + content = [{"type": "text", "text": "summarize"}, + {"type": "file", "file": {"filename": "a.pdf", "file_data": data_uri(PDF, "application/pdf")}}] + litellm.completion(model="openai/gpt-synthetic", messages=[{"role": "user", "content": content}], + api_base=media_provider.url, api_key="sk-synthetic") + flush(manager) + assert f01.payloads == [PDF] + owners = [(s.name, r["purpose"]) for s in exporter.get_finished_spans() for r in refs(s)] + assert owners == [("completion", "document")], owners + sent = [raw for path, _, raw in media_provider.requests if path.endswith("/chat/completions")] + assert sent and b"file_data" in sent[-1] # the SDK request itself is unchanged + manager.shutdown(5) + + @pytest.mark.parametrize("async_", [False, True]) def test_litellm_image_generation_output(f01, media_provider, litellm_instrument, async_): import litellm diff --git a/python/tests/attachments_compat_providers/test_compat_attachments.py b/python/tests/attachments_compat_providers/test_compat_attachments.py index df21eaa09..27231425b 100644 --- a/python/tests/attachments_compat_providers/test_compat_attachments.py +++ b/python/tests/attachments_compat_providers/test_compat_attachments.py @@ -100,6 +100,8 @@ def test_openai_sdk_routes_capture_images(f01, media_provider, instrumented, mon from traceai_xai import XAIInstrumentor as Instrumentor from traceai_xai import _wrappers monkeypatch.setattr(_wrappers, "_is_xai_client", lambda instance: True) + if hasattr(_wrappers, "_shared_instrumentor_enabled"): + monkeypatch.setattr(_wrappers, "_shared_instrumentor_enabled", lambda: False) manager = f01.manager() exporter = instrumented(Instrumentor(), manager) @@ -112,6 +114,119 @@ def test_openai_sdk_routes_capture_images(f01, media_provider, instrumented, mon manager.shutdown(5) +def media_owners(exporter): + return [(s.name, r["purpose"]) for s in exporter.get_finished_spans() for r in refs(s)] + + +@pytest.mark.parametrize("order", ["openai_first", "binding_first"]) +@pytest.mark.parametrize("package", ["xai", "minimax"]) +def test_binding_with_openai_instrumentor_uploads_once(f01, media_provider, instrumented, monkeypatch, package, + order): + """The OpenAI instrumentor wraps the same SDK call; one image is uploaded once, whatever the order.""" + openai = pytest.importorskip("openai") + pytest.importorskip("traceai_openai") + pytest.importorskip("traceai_" + package) + from traceai_openai import OpenAIInstrumentor + + if package == "xai": + from traceai_xai import XAIInstrumentor as Instrumentor + from traceai_xai import _wrappers + monkeypatch.setattr(_wrappers, "_is_xai_client", lambda instance: True) + else: + from traceai_minimax import MiniMaxInstrumentor as Instrumentor + from traceai_minimax import _wrappers + monkeypatch.setattr(_wrappers, "is_minimax_client", lambda instance: True) + + manager = f01.manager() + pair = [OpenAIInstrumentor(), Instrumentor()] + if order == "binding_first": + pair.reverse() + for instrumentor in pair: + exporter = instrumented(instrumentor, manager) + client = openai.OpenAI(api_key="sk-synthetic", base_url=media_provider.url + "/v1", max_retries=0) + client.chat.completions.create(model="m", messages=image_messages()) + flush(manager) + assert f01.payloads == [IMAGE] + owners = media_owners(exporter) + assert len(owners) == 1 and owners[0][1] == "model_input", owners + manager.shutdown(5) + + +def test_minimax_chat_images(f01, media_provider, instrumented, monkeypatch): + openai = pytest.importorskip("openai") + pytest.importorskip("traceai_minimax") + from traceai_minimax import MiniMaxInstrumentor + from traceai_minimax import _wrappers + + monkeypatch.setattr(_wrappers, "is_minimax_client", lambda instance: True) + manager = f01.manager() + exporter = instrumented(MiniMaxInstrumentor(), manager) + client = openai.OpenAI(api_key="sk-synthetic", base_url=media_provider.url + "/v1", max_retries=0) + client.chat.completions.create(model="MiniMax-M3", messages=image_messages()) + client.chat.completions.create(model="MiniMax-M3", messages=audio_messages(wav())) # no audio input + asyncio.run(openai.AsyncOpenAI(api_key="sk-synthetic", base_url=media_provider.url + "/v1", max_retries=0) + .chat.completions.create(model="MiniMax-M3", messages=image_messages())) + flush(manager) + assert f01.payloads == [IMAGE, IMAGE] + assert media_owners(exporter) == [("minimax.chat.completions", "model_input")] * 2 + manager.shutdown(5) + + +def test_minimax_hide_input_images(f01, media_provider, instrumented, monkeypatch): + openai = pytest.importorskip("openai") + pytest.importorskip("traceai_minimax") + from traceai_minimax import MiniMaxInstrumentor + from traceai_minimax import _wrappers + + monkeypatch.setattr(_wrappers, "is_minimax_client", lambda instance: True) + manager = f01.manager() + instrumented(MiniMaxInstrumentor(), manager, TraceConfig(hide_input_images=True)) + client = openai.OpenAI(api_key="sk-synthetic", base_url=media_provider.url + "/v1", max_retries=0) + client.chat.completions.create(model="MiniMax-M3", messages=image_messages()) + flush(manager) + assert f01.payloads == [] + manager.shutdown(5) + + +# ------------------------------------------------------------------ vllm (legacy OpenAI-dialect chat binding) +def test_vllm_chat_images_and_audio(f01, media_provider, instrumented): + openai = pytest.importorskip("openai") + pytest.importorskip("traceai_vllm") + from traceai_vllm import VLLMInstrumentor + + manager = f01.manager() + sound = wav(seed=9) + exporter = instrumented(VLLMInstrumentor(vllm_base_urls=[media_provider.url]), manager) + client = openai.OpenAI(api_key="EMPTY", base_url=media_provider.url + "/v1", max_retries=0) + client.chat.completions.create(model="m", messages=image_messages() + audio_messages(sound)) + asyncio.run(openai.AsyncOpenAI(api_key="EMPTY", base_url=media_provider.url + "/v1", max_retries=0) + .chat.completions.create(model="m", messages=image_messages())) + flush(manager) + assert sorted(f01.payloads) == sorted([IMAGE, sound, IMAGE]) + assert sorted(media_owners(exporter)) == [("vLLM Chat Completions", "model_input")] * 3 + manager.shutdown(5) + + +def test_vllm_defers_to_openai_instrumentor(f01, media_provider, instrumented): + """VLLMInstrumentor already defers the whole call to traceai_openai while it is enabled: one upload.""" + openai = pytest.importorskip("openai") + pytest.importorskip("traceai_openai") + pytest.importorskip("traceai_vllm") + from traceai_openai import OpenAIInstrumentor + from traceai_vllm import VLLMInstrumentor + + manager = f01.manager() + instrumented(VLLMInstrumentor(vllm_base_urls=[media_provider.url]), manager) + exporter = instrumented(OpenAIInstrumentor(), manager) + client = openai.OpenAI(api_key="EMPTY", base_url=media_provider.url + "/v1", max_retries=0) + client.chat.completions.create(model="m", messages=image_messages()) + flush(manager) + assert f01.payloads == [IMAGE] + owners = media_owners(exporter) + assert len(owners) == 1 and owners[0][0] != "vLLM Chat Completions", owners + manager.shutdown(5) + + @pytest.mark.parametrize("package", ["deepseek", "cerebras"]) def test_routes_without_media_surface_capture_nothing(f01, media_provider, instrumented, monkeypatch, package): pytest.importorskip("traceai_" + package) @@ -275,6 +390,8 @@ def test_ollama_async_chat(f01, media_provider, instrumented): ("traceai_portkey", "portkey_attachment_capabilities"), ("traceai_deepseek", "deepseek_attachment_capabilities"), ("traceai_cerebras", "cerebras_attachment_capabilities"), + ("traceai_vllm", "vllm_attachment_capabilities"), + ("traceai_minimax", "minimax_attachment_capabilities"), ]) def test_package_descriptor(module, name): package = pytest.importorskip(module) diff --git a/python/tests/attachments_compat_providers/test_provider_media.py b/python/tests/attachments_compat_providers/test_provider_media.py index 991f48b66..13874951c 100644 --- a/python/tests/attachments_compat_providers/test_provider_media.py +++ b/python/tests/attachments_compat_providers/test_provider_media.py @@ -299,6 +299,100 @@ def test_sniff_and_consistent(): assert consistent("audio/webm", b"\x1a\x45\xdf\xa3" + b"\x00" * 12) +@pytest.mark.parametrize("brand,expected", [ + (b"isom", "video/mp4"), (b"mp42", "video/mp4"), (b"avc1", "video/mp4"), + # HEIF/HEIC and AVIF still images, QuickTime and audio-only M4A share the ftyp box: never video/mp4. + (b"heic", None), (b"heix", None), (b"mif1", None), (b"msf1", None), (b"avif", None), (b"avis", None), + (b"qt ", None), (b"M4A ", None), (b"zzzz", None), +]) +def test_sniff_checks_iso_bmff_major_brand(brand, expected): + head = b"\x00\x00\x00\x18ftyp" + brand + b"\x00\x00\x02\x00" + b"\x00" * 16 + assert sniff(head) == expected + assert consistent("video/mp4", head) is (expected == "video/mp4") + + +def test_heic_labelled_video_mp4_is_never_uploaded(f01, spans): + """A HEIC still image declared as video/mp4 would dodge hide_input_images; the brand check refuses it.""" + tracer_provider, _ = spans + f01.state["policy"] = {"allowed_modalities": ["audio", "image", "document", "file", "video"]} + manager = f01.manager() + heic = b"\x00\x00\x00\x18ftypheic\x00\x00\x00\x00mif1heic" + b"\x00" * 24 + contents = [{"role": "user", "parts": [{"inline_data": {"mime_type": "video/mp4", "data": heic}}, + {"inline_data": {"mime_type": "video/mp4", "data": MP4}}]}] + capture(manager, tracer_provider, lambda m: m.gemini_contents(contents), + trace_config=TraceConfig(hide_input_images=True)) + flush(manager) + assert f01.payloads == [MP4] + manager.shutdown(5) + + +def test_openai_dialect_file_parts_capture_pdf_once(f01, spans): + """Chat ``file`` (file.file_data) and Responses-style ``input_file`` PDF data URIs; IDs and URLs never read.""" + tracer_provider, exporter = spans + manager = f01.manager() + other_pdf = PDF + b"%%second\n" + messages = [{"role": "user", "content": [ + {"type": "text", "text": "summarize"}, + {"type": "file", "file": {"filename": "a.pdf", "file_data": data_uri(PDF, "application/pdf")}}, + {"type": "input_file", "filename": "b.pdf", "file_data": data_uri(other_pdf, "application/pdf")}, + {"type": "file", "file": {"file_id": "file-synthetic"}}, + {"type": "file", "file": {"file_data": data_uri(IMAGE, "application/pdf")}}, # mislabelled: refused + {"type": "file", "file": {"file_data": data_uri(PDF, "text/plain")}}, # not a captured type + ]}] + # hide_input_images does not hide documents. + # Without documents=True (every route but LiteLLM) no file part is read. + capture(manager, tracer_provider, lambda m: m.compatible_messages(messages, "openai")) + flush(manager) + assert f01.payloads == [] + media, span = capture(manager, tracer_provider, + lambda m: m.compatible_messages(messages, "openai", documents=True), + trace_config=TraceConfig(hide_input_images=True)) + flush(manager) + assert sorted(f01.payloads) == sorted([PDF, other_pdf]) + assert set(media.captured) == {"chat/0/1", "chat/0/2"} + span = exporter.get_finished_spans()[-1] + assert sorted((r["part_index"], r["purpose"], r["mime_type"]) for r in refs(span)) == [ + (1, "document", "application/pdf"), (2, "document", "application/pdf")] + manager.shutdown(5) + + +@pytest.mark.parametrize("dialect", ["groq", "mistral", "image"]) +def test_image_only_dialects_never_read_file_parts(f01, spans, dialect): + tracer_provider, _ = spans + manager = f01.manager() + messages = [{"role": "user", "content": [ + {"type": "file", "file": {"file_data": data_uri(PDF, "application/pdf")}}, + {"type": "input_file", "file_data": data_uri(PDF, "application/pdf")}]}] + capture(manager, tracer_provider, lambda m: m.compatible_messages(messages, dialect, documents=True)) + flush(manager) + assert f01.payloads == [] + manager.shutdown(5) + + +def test_data_uri_length_is_bounded_before_slicing(f01, spans): + """An oversized data URI is refused from its length alone: the string is never sliced or copied.""" + tracer_provider, _ = spans + manager = f01.manager() + slices = [] + + class Tracked(str): + def __getitem__(self, key): + slices.append(key) + return str.__getitem__(self, key) + + oversized = Tracked("data:image/png;base64," + "A" * (4 * provider_media.MAX_DECODED_BYTES // 3 + 8)) + exact = Tracked(data_uri(IMAGE)) + messages = [{"role": "user", "content": [{"type": "image_url", "image_url": {"url": oversized}}]}] + capture(manager, tracer_provider, lambda m: m.compatible_messages(messages, "openai")) + flush(manager) + assert f01.payloads == [] and slices == [] + messages = [{"role": "user", "content": [{"type": "image_url", "image_url": {"url": exact}}]}] + capture(manager, tracer_provider, lambda m: m.compatible_messages(messages, "openai")) + flush(manager) + assert f01.payloads == [IMAGE] and slices # in-bound values are sliced once, after the bound check + manager.shutdown(5) + + def test_capability_matrix_is_explicit(): for package in ("traceai_deepseek", "traceai_cerebras", "traceai_together"): assert PROVIDER_MEDIA_CAPABILITIES[package]["operations"] == () @@ -341,7 +435,8 @@ def test_disabled_path_never_imports_attachment_modules(tmp_path): script.write_text(PROBE) names = ["traceai_google_genai", "traceai_vertexai", "traceai_bedrock", "traceai_litellm", "traceai_groq", "traceai_together", "traceai_fireworks", "traceai_xai", "traceai_mistralai", "traceai_openrouter", - "traceai_ollama", "traceai_portkey", "traceai_deepseek", "traceai_cerebras"] + "traceai_ollama", "traceai_portkey", "traceai_deepseek", "traceai_cerebras", "traceai_vllm", + "traceai_minimax"] result = subprocess.run([sys.executable, str(script), *names], capture_output=True, text=True, timeout=120, env=dict(os.environ, LITELLM_LOCAL_MODEL_COST_MAP="True")) assert result.returncode == 0, result.stderr[-2000:] From 41f508f839708dbbb8926492a97752bc298e8673 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek <8347993+nik13@users.noreply.github.com> Date: Wed, 11 Nov 2026 22:51:57 -0800 Subject: [PATCH 10/15] test(python): drop tool_call_id from the InvokeModel ref check and skip LiteLLM releases that reject file parts The span ref carries message and part indexes only. litellm 1.53.x rejects chat file parts before sending, so the floor job proves the PDF row on releases that accept them (latest) and keeps its 9-row minimum. Co-Authored-By: Claude Opus 5.5 --- .../attachments-python-compat-providers-tests.yml | 2 +- .../test_bedrock_litellm_attachments.py | 13 ++++++++++--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/.github/workflows/attachments-python-compat-providers-tests.yml b/.github/workflows/attachments-python-compat-providers-tests.yml index df49c0861..c537e29d4 100644 --- a/.github/workflows/attachments-python-compat-providers-tests.yml +++ b/.github/workflows/attachments-python-compat-providers-tests.yml @@ -73,7 +73,7 @@ jobs: - {package: vertexai, tier: latest, python: '3.12', sdks: 'google-cloud-aiplatform==1.101.0', tests: google, min: 5} - {package: bedrock, tier: floor, python: '3.10', sdks: 'boto3==1.38.17', tests: aws, min: 12} - {package: bedrock, tier: latest, python: '3.12', sdks: 'boto3==1.44.1', tests: aws, min: 12} - - {package: litellm, tier: floor, python: '3.10', sdks: '-e python/frameworks/openai litellm==1.53.1 openai==1.69.0', tests: aws, min: 10} + - {package: litellm, tier: floor, python: '3.10', sdks: '-e python/frameworks/openai litellm==1.53.1 openai==1.69.0', tests: aws, min: 9} - {package: litellm, tier: latest, python: '3.12', sdks: '-e python/frameworks/openai litellm==1.109.1', tests: aws, min: 10} - {package: groq, tier: floor, python: '3.10', sdks: 'groq==0.9.0 httpx<0.28', tests: compat, min: 2} - {package: groq, tier: latest, python: '3.12', sdks: 'groq==1.7.0', tests: compat, min: 2} diff --git a/python/tests/attachments_compat_providers/test_bedrock_litellm_attachments.py b/python/tests/attachments_compat_providers/test_bedrock_litellm_attachments.py index 3ba6084fb..a7608f965 100644 --- a/python/tests/attachments_compat_providers/test_bedrock_litellm_attachments.py +++ b/python/tests/attachments_compat_providers/test_bedrock_litellm_attachments.py @@ -154,7 +154,7 @@ def test_invoke_model_anthropic_body(f01, bedrock): assert sorted(u["purpose"] for u in f01.uploads) == ["computer_screenshot", "model_input"] # As for Converse, a tool_result child carries the tool_result block's index (1), not its own (2). shot = [r for s in exporter.get_finished_spans() for r in refs(s) if r.get("purpose") == "computer_screenshot"] - assert [(r["message_index"], r["part_index"], r["tool_call_id"]) for r in shot] == [(2, 1, "toolu_1")] + assert [(r["message_index"], r["part_index"]) for r in shot] == [(2, 1)] manager.shutdown(5) @@ -286,8 +286,15 @@ def test_litellm_pdf_file_part_uploads_once_under_nested_openai(f01, media_provi exporter = litellm_instrument(manager, TraceConfig(hide_input_images=True), with_openai=True) content = [{"type": "text", "text": "summarize"}, {"type": "file", "file": {"filename": "a.pdf", "file_data": data_uri(PDF, "application/pdf")}}] - litellm.completion(model="openai/gpt-synthetic", messages=[{"role": "user", "content": content}], - api_base=media_provider.url, api_key="sk-synthetic") + try: + litellm.completion(model="openai/gpt-synthetic", messages=[{"role": "user", "content": content}], + api_base=media_provider.url, api_key="sk-synthetic") + except Exception as error: + # Older LiteLLM releases (1.53.x) reject chat `file` parts before any request is sent. + if "invalid content type" not in str(error).lower(): + raise + manager.shutdown(5) + pytest.skip("this LiteLLM release rejects chat file parts") flush(manager) assert f01.payloads == [PDF] owners = [(s.name, r["purpose"]) for s in exporter.get_finished_spans() for r in refs(s)] From 774675b966f0122e8882ca0168ce873c0d0e3be5 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek <8347993+nik13@users.noreply.github.com> Date: Wed, 11 Nov 2026 23:04:24 -0800 Subject: [PATCH 11/15] test(python): match the LiteLLM 1.53 rejection as raised (Invalid user message) Co-Authored-By: Claude Opus 5.5 --- .../test_bedrock_litellm_attachments.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/tests/attachments_compat_providers/test_bedrock_litellm_attachments.py b/python/tests/attachments_compat_providers/test_bedrock_litellm_attachments.py index a7608f965..5eed71e1e 100644 --- a/python/tests/attachments_compat_providers/test_bedrock_litellm_attachments.py +++ b/python/tests/attachments_compat_providers/test_bedrock_litellm_attachments.py @@ -291,7 +291,8 @@ def test_litellm_pdf_file_part_uploads_once_under_nested_openai(f01, media_provi api_base=media_provider.url, api_key="sk-synthetic") except Exception as error: # Older LiteLLM releases (1.53.x) reject chat `file` parts before any request is sent. - if "invalid content type" not in str(error).lower(): + text = str(error).lower() + if "invalid content type" not in text and "invalid user message" not in text: raise manager.shutdown(5) pytest.skip("this LiteLLM release rejects chat file parts") From e4155bf26a4303b3f654a3596b6070f54ffbb5fc Mon Sep 17 00:00:00 2001 From: Nikhil Pareek <8347993+nik13@users.noreply.github.com> Date: Thu, 12 Nov 2026 05:58:54 -0800 Subject: [PATCH 12/15] fix(python): gate xAI/MiniMax deferral on attachments and keep their local deny traceai_xai and traceai_minimax now hand the whole chat call to an enabled traceai_openai only while attachments are enabled for them. With attachments off they trace exactly as before (binding span plus the OpenAI span). A deferred call runs under provider_media.deferred(config): the binding's local hide/PII flags are folded into a restriction-only CapturePolicy and merged (deny wins) with the per-call policy under CAPTURE_POLICY_KEY, so the OpenAI instrumentor cannot capture more than the binding allows, whatever its own TraceConfig. hide_input_images denies every input part of the deferred call (a CapturePolicy cannot single out images). Co-Authored-By: Claude Opus 5.5 --- .../attachments/provider_media.py | 35 +++++++ .../minimax/traceai_minimax/_wrappers.py | 47 ++++++++-- .../frameworks/xai/traceai_xai/_wrappers.py | 48 ++++++++-- .../test_compat_attachments.py | 93 +++++++++++++++++++ .../test_provider_media.py | 48 ++++++++++ 5 files changed, 254 insertions(+), 17 deletions(-) diff --git a/python/fi_instrumentation/attachments/provider_media.py b/python/fi_instrumentation/attachments/provider_media.py index 8ee0ae2fa..b3fff768d 100644 --- a/python/fi_instrumentation/attachments/provider_media.py +++ b/python/fi_instrumentation/attachments/provider_media.py @@ -35,6 +35,7 @@ from opentelemetry import context as otel_context from . import media_capture as mc +from .policy import CapturePolicy from .types import Bytes KiB = 1024 @@ -117,6 +118,40 @@ def claim(context=None): return otel_context.set_value(mc.OWNER_KEY, True, context) +def deferral_policy(trace_config: Any) -> CapturePolicy: + """An instrumentor's local ``TraceConfig`` as a restriction-only ``CapturePolicy``. + + ``pii_redaction`` denies all capture; an input hide flag or ``hide_input_images`` denies + every input part; an output hide flag denies every output part. The receiving layer + cannot tell an image from another input part through a ``CapturePolicy``, so + ``hide_input_images`` also denies input files and audio on the deferred call (deny wins). + An unreadable flag counts as set. + """ + def restricted(names: Tuple[str, ...]) -> bool: + return any(mc._restricted(trace_config, name) for name in names) + + return CapturePolicy( + capture_content=not restricted(("pii_redaction",)), + record_inputs=not restricted(mc._INPUT_HIDE + ("hide_input_images",)), + record_outputs=not restricted(mc._OUTPUT_HIDE), + ) + + +def deferred(trace_config: Any, context=None): + """The context for a call an instrumentor hands WHOLE to another one (xAI/MiniMax to traceai_openai). + + The deferring instrumentor's local deny is merged (deny wins) into the per-call policy + (``media_capture.CAPTURE_POLICY_KEY``), so the receiving instrumentor cannot capture more + than either configuration allows, whatever its own ``TraceConfig``. A policy already in the + context and the owner mark are kept. Never raises: on any error the call is denied. + """ + try: + policy = mc.merge_capture_policies(mc.context_capture_policy(context), deferral_policy(trace_config)) + except Exception: + policy = CapturePolicy(capture_content=False, requested=False) + return otel_context.set_value(mc.CAPTURE_POLICY_KEY, policy, context) + + # ------------------------------------------------------------------ type checks def sniff(head: Any) -> Optional[str]: """The F01 type named by leading magic bytes the caller already holds; None when unknown.""" diff --git a/python/frameworks/minimax/traceai_minimax/_wrappers.py b/python/frameworks/minimax/traceai_minimax/_wrappers.py index 50f64427a..67f0dec75 100644 --- a/python/frameworks/minimax/traceai_minimax/_wrappers.py +++ b/python/frameworks/minimax/traceai_minimax/_wrappers.py @@ -23,13 +23,16 @@ logger = logging.getLogger(__name__) logger.addHandler(logging.NullHandler()) +# Per-call capture policy key, shared by name (fi_instrumentation.attachments.media_capture). +_CAPTURE_POLICY_KEY = "traceai.auto.attachment_policy.v1" def _shared_instrumentor_enabled() -> bool: - """True while traceai_openai's OpenAIInstrumentor is enabled (it then owns the call's media). + """True while traceai_openai's OpenAIInstrumentor is enabled. - Do not import the optional package; inspect live state on each call, so either - installation order works. + With attachments on, the request-level OpenAI instrumentor then owns the call (span + and media), whatever the setup order. Do not import the optional package; inspect + live state on each call. """ module = sys.modules.get("traceai_openai") instrumentor = getattr(module, "OpenAIInstrumentor", None) @@ -49,16 +52,10 @@ def __init__( self._config = config def _capture_chat_media(self, span: _WithSpan, messages: Any) -> None: - """Typed chat media (data-URI images) through the shared manager; never raises. - - The OpenAI instrumentor wraps the same SDK method; while it is enabled it captures this - call's media, so this binding uploads nothing (one upload per part, either order). - """ + """Typed chat media (data-URI images) through the shared manager; never raises.""" if self._attachments is None: return try: - if _shared_instrumentor_enabled(): - return from fi_instrumentation.attachments.provider_media import ProviderMedia ProviderMedia(self._attachments, span._span, "minimax", trace_config=self._config).compatible_messages( @@ -66,6 +63,28 @@ def _capture_chat_media(self, span: _WithSpan, messages: Any) -> None: except Exception: logger.debug("Attachment capture unavailable", exc_info=True) + def _defers(self) -> bool: + """Hand the whole call to an enabled traceai_openai, only while attachments are on. + + With attachments off this binding traces exactly as before (its own span too). + """ + return self._attachments is not None and _shared_instrumentor_enabled() + + @contextmanager + def _deferred(self) -> Iterator[None]: + """Run the handed-over call under this binding's local deny (deny wins downstream).""" + try: + from fi_instrumentation.attachments.provider_media import deferred + + token = context_api.attach(deferred(self._config)) + except Exception: + # An unreadable restriction is a restriction: deny capture for this call. + token = context_api.attach(context_api.set_value(_CAPTURE_POLICY_KEY, False)) + try: + yield + finally: + context_api.detach(token) + @contextmanager def _start_as_current_span( self, @@ -122,6 +141,10 @@ def __call__( if not is_minimax_client(client) and not is_minimax_client(instance): return wrapped(*args, **kwargs) + # One request must have one owner, regardless of setup order. + if self._defers(): + with self._deferred(): + return wrapped(*args, **kwargs) request_parameters = dict(kwargs) is_streaming = request_parameters.get("stream", False) @@ -230,6 +253,10 @@ async def __call__( if not is_minimax_client(client) and not is_minimax_client(instance): return await wrapped(*args, **kwargs) + # One request must have one owner, regardless of setup order. + if self._defers(): + with self._deferred(): + return await wrapped(*args, **kwargs) request_parameters = dict(kwargs) is_streaming = request_parameters.get("stream", False) diff --git a/python/frameworks/xai/traceai_xai/_wrappers.py b/python/frameworks/xai/traceai_xai/_wrappers.py index a4f335aa7..e727cd1e5 100644 --- a/python/frameworks/xai/traceai_xai/_wrappers.py +++ b/python/frameworks/xai/traceai_xai/_wrappers.py @@ -8,6 +8,7 @@ from fi_instrumentation import safe_json_dumps from fi_instrumentation.fi_types import FiSpanKindValues, SpanAttributes, MessageAttributes +from opentelemetry import context as context_api from opentelemetry import trace as trace_api from opentelemetry.trace import INVALID_SPAN, Span, Status, StatusCode from opentelemetry.util.types import AttributeValue @@ -17,6 +18,8 @@ # xAI-specific base URL for detection XAI_BASE_URL = "api.x.ai" +# Per-call capture policy key, shared by name (fi_instrumentation.attachments.media_capture). +_CAPTURE_POLICY_KEY = "traceai.auto.attachment_policy.v1" class _WithTracer(ABC): @@ -51,6 +54,28 @@ def _capture_chat_media(self, span: Any, messages: Any) -> None: except Exception: logger.debug("Attachment capture unavailable", exc_info=True) + def _defers(self) -> bool: + """Hand the whole call to an enabled traceai_openai, only while attachments are on. + + With attachments off this binding traces exactly as before (its own span too). + """ + return self._attachments is not None and _shared_instrumentor_enabled() + + @contextmanager + def _deferred(self) -> Iterator[None]: + """Run the handed-over call under this binding's local deny (deny wins downstream).""" + try: + from fi_instrumentation.attachments.provider_media import deferred + + token = context_api.attach(deferred(self._config)) + except Exception: + # An unreadable restriction is a restriction: deny capture for this call. + token = context_api.attach(context_api.set_value(_CAPTURE_POLICY_KEY, False)) + try: + yield + finally: + context_api.detach(token) + @contextmanager def _start_span( self, @@ -86,10 +111,11 @@ def _is_xai_client(instance: Any) -> bool: def _shared_instrumentor_enabled() -> bool: - """Let the request-level OpenAI instrumentor own the call when both are enabled. + """True while traceai_openai's OpenAIInstrumentor is enabled. - Same rule as the Fireworks route: one request has one owner (span and media), - whatever the setup order. Do not import the optional package; inspect live state. + With attachments on, the request-level OpenAI instrumentor then owns the call (span + and media), whatever the setup order. Do not import the optional package; inspect + live state on each call. """ module = sys.modules.get("traceai_openai") instrumentor = getattr(module, "OpenAIInstrumentor", None) @@ -240,9 +266,13 @@ def __call__( args: Tuple[Any, ...], kwargs: Dict[str, Any], ) -> Any: - # One request must have one owner, regardless of setup order. - if _shared_instrumentor_enabled() or not _is_xai_client(instance): + # Check if this is an xAI client + if not _is_xai_client(instance): return wrapped(*args, **kwargs) + # One request must have one owner, regardless of setup order. + if self._defers(): + with self._deferred(): + return wrapped(*args, **kwargs) # Extract parameters model = kwargs.get("model", args[0] if args else "unknown") @@ -312,9 +342,13 @@ async def __call__( args: Tuple[Any, ...], kwargs: Dict[str, Any], ) -> Any: - # One request must have one owner, regardless of setup order. - if _shared_instrumentor_enabled() or not _is_xai_client(instance): + # Check if this is an xAI client + if not _is_xai_client(instance): return await wrapped(*args, **kwargs) + # One request must have one owner, regardless of setup order. + if self._defers(): + with self._deferred(): + return await wrapped(*args, **kwargs) # Extract parameters model = kwargs.get("model", args[0] if args else "unknown") diff --git a/python/tests/attachments_compat_providers/test_compat_attachments.py b/python/tests/attachments_compat_providers/test_compat_attachments.py index 27231425b..a6a879d76 100644 --- a/python/tests/attachments_compat_providers/test_compat_attachments.py +++ b/python/tests/attachments_compat_providers/test_compat_attachments.py @@ -152,6 +152,99 @@ def test_binding_with_openai_instrumentor_uploads_once(f01, media_provider, inst manager.shutdown(5) +def _compat_binding(package, monkeypatch): + """The binding instrumentor (client check forced on for the loopback URL) and its own span name.""" + if package == "xai": + from traceai_xai import XAIInstrumentor + from traceai_xai import _wrappers + monkeypatch.setattr(_wrappers, "_is_xai_client", lambda instance: True) + return XAIInstrumentor(), "xAI Chat Completions" + from traceai_minimax import MiniMaxInstrumentor + from traceai_minimax import _wrappers + monkeypatch.setattr(_wrappers, "is_minimax_client", lambda instance: True) + return MiniMaxInstrumentor(), "minimax.chat.completions" + + +def _openai_clients(openai, media_provider): + options = dict(api_key="sk-synthetic", base_url=media_provider.url + "/v1", max_retries=0) + return openai.OpenAI(**options), openai.AsyncOpenAI(**options) + + +@pytest.mark.parametrize("order", ["openai_first", "binding_first"]) +@pytest.mark.parametrize("package", ["xai", "minimax"]) +def test_binding_without_attachments_keeps_its_span(media_provider, instrumented, monkeypatch, package, order): + """Attachments off: tracing is unchanged, so the binding span and the OpenAI span both stay.""" + openai = pytest.importorskip("openai") + pytest.importorskip("traceai_openai") + pytest.importorskip("traceai_" + package) + from traceai_openai import OpenAIInstrumentor + + binding, span_name = _compat_binding(package, monkeypatch) + pair = [OpenAIInstrumentor(), binding] + if order == "binding_first": + pair.reverse() + for instrumentor in pair: + exporter = instrumented(instrumentor, None) + client, async_client = _openai_clients(openai, media_provider) + client.chat.completions.create(model="m", messages=[{"role": "user", "content": "hi"}]) + asyncio.run(async_client.chat.completions.create(model="m", messages=[{"role": "user", "content": "hi"}])) + names = sorted(span.name for span in exporter.get_finished_spans()) + assert names == sorted([span_name, "ChatCompletion"] * 2), names + + +@pytest.mark.parametrize("flag", [None, "hide_input_images", "hide_inputs", "pii_redaction"]) +@pytest.mark.parametrize("package", ["xai", "minimax"]) +def test_deferred_call_keeps_binding_local_deny(f01, media_provider, instrumented, monkeypatch, package, flag): + """Attachments on: the whole call goes to traceai_openai (default config), under the binding's deny.""" + openai = pytest.importorskip("openai") + pytest.importorskip("traceai_openai") + pytest.importorskip("traceai_" + package) + from traceai_openai import OpenAIInstrumentor + + manager = f01.manager() + exporter = instrumented(OpenAIInstrumentor(), manager) + binding, _ = _compat_binding(package, monkeypatch) + instrumented(binding, manager, TraceConfig(**({flag: True} if flag else {}))) + client, async_client = _openai_clients(openai, media_provider) + client.chat.completions.create(model="m", messages=image_messages()) + asyncio.run(async_client.chat.completions.create(model="m", messages=image_messages())) + flush(manager) + assert f01.payloads == ([] if flag else [IMAGE, IMAGE]) + assert [span.name for span in exporter.get_finished_spans()] == ["ChatCompletion"] * 2 + manager.shutdown(5) + + +@pytest.mark.parametrize("package", ["xai", "minimax"]) +def test_deferred_call_keeps_context_policy_and_owner(f01, media_provider, instrumented, monkeypatch, package): + """A per-call deny and an outer owner mark already in the context survive the deferral.""" + openai = pytest.importorskip("openai") + pytest.importorskip("traceai_openai") + pytest.importorskip("traceai_" + package) + from opentelemetry import context as otel_context + from fi_instrumentation.attachments import CapturePolicy + from fi_instrumentation.attachments import media_capture + from traceai_openai import OpenAIInstrumentor + + manager = f01.manager() + binding, _ = _compat_binding(package, monkeypatch) + instrumented(binding, manager) + instrumented(OpenAIInstrumentor(), manager) + client, _ = _openai_clients(openai, media_provider) + for key, value in ((media_capture.CAPTURE_POLICY_KEY, CapturePolicy(record_inputs=False)), + (media_capture.OWNER_KEY, True)): + token = otel_context.attach(otel_context.set_value(key, value)) + try: + client.chat.completions.create(model="m", messages=image_messages()) + finally: + otel_context.detach(token) + flush(manager) + assert f01.payloads == [] + client.chat.completions.create(model="m", messages=image_messages()) # control + flush(manager) + assert f01.payloads == [IMAGE] + manager.shutdown(5) + + def test_minimax_chat_images(f01, media_provider, instrumented, monkeypatch): openai = pytest.importorskip("openai") pytest.importorskip("traceai_minimax") diff --git a/python/tests/attachments_compat_providers/test_provider_media.py b/python/tests/attachments_compat_providers/test_provider_media.py index 13874951c..0a818f8af 100644 --- a/python/tests/attachments_compat_providers/test_provider_media.py +++ b/python/tests/attachments_compat_providers/test_provider_media.py @@ -450,3 +450,51 @@ def test_dataclass_policy_unchanged_by_helper(): policy = media_capture.local_capture_policy(None, direction="input", image=True) assert dataclasses.asdict(policy) == dataclasses.asdict( dataclasses.replace(CapturePolicy(), model_input=True)) + + +class _Unreadable: + """A TraceConfig stand-in whose flags cannot be read.""" + + def __getattr__(self, name): + raise RuntimeError(name) + + +@pytest.mark.parametrize("flags,expected", [ + ({}, {}), + ({"hide_input_images": True}, {"record_inputs": False}), + ({"hide_inputs": True}, {"record_inputs": False}), + ({"hide_input_messages": True}, {"record_inputs": False}), + ({"hide_input_text": True}, {"record_inputs": False}), + ({"hide_outputs": True}, {"record_outputs": False}), + ({"hide_output_text": True}, {"record_outputs": False}), + ({"pii_redaction": True}, {"capture_content": False}), +]) +def test_deferral_policy_folds_local_deny(flags, expected): + policy = provider_media.deferral_policy(TraceConfig(**flags)) + assert dataclasses.asdict(policy) == dataclasses.asdict(dataclasses.replace(CapturePolicy(), **expected)) + + +def test_deferred_context_merges_and_fails_closed(): + """A deferred call keeps the context policy and owner mark; local deny is added, never relaxed.""" + key = media_capture.CAPTURE_POLICY_KEY + assert provider_media.deferral_policy(None) == CapturePolicy() + unreadable = provider_media.deferral_policy(_Unreadable()) + assert not (unreadable.capture_content or unreadable.record_inputs or unreadable.record_outputs) + + outer = otel_context.set_value(media_capture.OWNER_KEY, True, + otel_context.set_value(key, CapturePolicy(record_outputs=False))) + merged = provider_media.deferred(TraceConfig(hide_input_images=True), outer) + policy = otel_context.get_value(key, merged) + assert (policy.record_inputs, policy.record_outputs, policy.capture_content) == (False, False, True) + assert media_capture.owned_elsewhere(merged) + + garbage = provider_media.deferred(TraceConfig(), otel_context.set_value(key, "not-a-policy")) + denied = otel_context.get_value(key, garbage) + assert denied.capture_content is False and denied.requested is False + + token = otel_context.attach(provider_media.deferred(TraceConfig(pii_redaction=True))) + try: + assert media_capture.context_capture_policy().capture_content is False + finally: + otel_context.detach(token) + assert media_capture.context_capture_policy() is None From d058417bbe36ac14146ab893d0580e1363b53103 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek <8347993+nik13@users.noreply.github.com> Date: Thu, 12 Nov 2026 06:29:51 -0800 Subject: [PATCH 13/15] ci(python): require the new deferral rows (core 54, xai 12, minimax 13) Co-Authored-By: Claude Opus 5.5 --- .../attachments-python-compat-providers-tests.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/attachments-python-compat-providers-tests.yml b/.github/workflows/attachments-python-compat-providers-tests.yml index c537e29d4..297fedf53 100644 --- a/.github/workflows/attachments-python-compat-providers-tests.yml +++ b/.github/workflows/attachments-python-compat-providers-tests.yml @@ -66,7 +66,7 @@ jobs: fail-fast: false matrix: include: - - {package: core, tier: latest, python: '3.12', sdks: '', tests: core, min: 45} + - {package: core, tier: latest, python: '3.12', sdks: '', tests: core, min: 54} - {package: google-genai, tier: floor, python: '3.10', sdks: 'google-genai==1.8.0', tests: google, min: 9} - {package: google-genai, tier: latest, python: '3.12', sdks: 'google-genai==3.2.0', tests: google, min: 9} - {package: vertexai, tier: floor, python: '3.10', sdks: 'google-cloud-aiplatform==1.61.0', tests: google, min: 5} @@ -81,12 +81,12 @@ jobs: - {package: together, tier: latest, python: '3.12', sdks: 'together==3.1.1', tests: compat, min: 3} - {package: fireworks, tier: floor, python: '3.10', sdks: 'openai==1.69.0 httpx<0.28', tests: compat, min: 3} - {package: fireworks, tier: latest, python: '3.12', sdks: 'openai==3.49.0', tests: compat, min: 3} - - {package: xai, tier: floor, python: '3.10', sdks: '-e python/frameworks/openai openai==1.69.0 httpx<0.28', tests: compat, min: 5} - - {package: xai, tier: latest, python: '3.12', sdks: '-e python/frameworks/openai openai==3.49.0', tests: compat, min: 5} + - {package: xai, tier: floor, python: '3.10', sdks: '-e python/frameworks/openai openai==1.69.0 httpx<0.28', tests: compat, min: 12} + - {package: xai, tier: latest, python: '3.12', sdks: '-e python/frameworks/openai openai==3.49.0', tests: compat, min: 12} - {package: vllm, tier: floor, python: '3.10', sdks: '-e python/frameworks/openai openai==1.69.0 httpx<0.28', tests: compat, min: 4} - {package: vllm, tier: latest, python: '3.12', sdks: '-e python/frameworks/openai openai==3.49.0', tests: compat, min: 4} - - {package: minimax, tier: floor, python: '3.10', sdks: '-e python/frameworks/openai openai==1.69.0 httpx<0.28', tests: compat, min: 6} - - {package: minimax, tier: latest, python: '3.12', sdks: '-e python/frameworks/openai openai==3.49.0', tests: compat, min: 6} + - {package: minimax, tier: floor, python: '3.10', sdks: '-e python/frameworks/openai openai==1.69.0 httpx<0.28', tests: compat, min: 13} + - {package: minimax, tier: latest, python: '3.12', sdks: '-e python/frameworks/openai openai==3.49.0', tests: compat, min: 13} - {package: deepseek, tier: floor, python: '3.10', sdks: 'openai==1.69.0 httpx<0.28', tests: compat, min: 3} - {package: deepseek, tier: latest, python: '3.12', sdks: 'openai==3.49.0', tests: compat, min: 3} - {package: cerebras, tier: floor, python: '3.10', sdks: 'cerebras-cloud-sdk==1.0.0 httpx<0.28', tests: compat, min: 3} From e9d56111c55bdd44c1f58c78f9941591685e4bf1 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek <8347993+nik13@users.noreply.github.com> Date: Thu, 12 Nov 2026 12:47:28 -0800 Subject: [PATCH 14/15] fix(python): defer xAI/MiniMax only to an OpenAI instrumentor that captures traceai_xai and traceai_minimax handed the whole call to traceai_openai whenever it was enabled. In a mixed setup (binding with an attachment manager, OpenAIInstrumentor enabled without one) that dropped the binding span and its media: nothing was uploaded. OpenAIInstrumentor gains a small public accessor, captures_attachments(), true only while it is instrumented with an active shared manager. The bindings now defer only when both sides capture; otherwise they keep the call, trace it and capture its media themselves. An older traceai_openai without the accessor is treated as not capturing. Tests: mixed setup in both orders (sync and async) uploads each image once, owned by the binding span, with both spans kept; legacy-accessor stub never takes the call. CI minimums: xai 15, minimax 16. Co-Authored-By: Claude Opus 5.5 --- ...achments-python-compat-providers-tests.yml | 8 +-- .../minimax/traceai_minimax/_wrappers.py | 24 +++++---- .../openai/traceai_openai/__init__.py | 10 ++++ .../frameworks/xai/traceai_xai/_wrappers.py | 24 +++++---- .../test_compat_attachments.py | 54 ++++++++++++++++++- 5 files changed, 96 insertions(+), 24 deletions(-) diff --git a/.github/workflows/attachments-python-compat-providers-tests.yml b/.github/workflows/attachments-python-compat-providers-tests.yml index 297fedf53..383046b2f 100644 --- a/.github/workflows/attachments-python-compat-providers-tests.yml +++ b/.github/workflows/attachments-python-compat-providers-tests.yml @@ -81,12 +81,12 @@ jobs: - {package: together, tier: latest, python: '3.12', sdks: 'together==3.1.1', tests: compat, min: 3} - {package: fireworks, tier: floor, python: '3.10', sdks: 'openai==1.69.0 httpx<0.28', tests: compat, min: 3} - {package: fireworks, tier: latest, python: '3.12', sdks: 'openai==3.49.0', tests: compat, min: 3} - - {package: xai, tier: floor, python: '3.10', sdks: '-e python/frameworks/openai openai==1.69.0 httpx<0.28', tests: compat, min: 12} - - {package: xai, tier: latest, python: '3.12', sdks: '-e python/frameworks/openai openai==3.49.0', tests: compat, min: 12} + - {package: xai, tier: floor, python: '3.10', sdks: '-e python/frameworks/openai openai==1.69.0 httpx<0.28', tests: compat, min: 15} + - {package: xai, tier: latest, python: '3.12', sdks: '-e python/frameworks/openai openai==3.49.0', tests: compat, min: 15} - {package: vllm, tier: floor, python: '3.10', sdks: '-e python/frameworks/openai openai==1.69.0 httpx<0.28', tests: compat, min: 4} - {package: vllm, tier: latest, python: '3.12', sdks: '-e python/frameworks/openai openai==3.49.0', tests: compat, min: 4} - - {package: minimax, tier: floor, python: '3.10', sdks: '-e python/frameworks/openai openai==1.69.0 httpx<0.28', tests: compat, min: 13} - - {package: minimax, tier: latest, python: '3.12', sdks: '-e python/frameworks/openai openai==3.49.0', tests: compat, min: 13} + - {package: minimax, tier: floor, python: '3.10', sdks: '-e python/frameworks/openai openai==1.69.0 httpx<0.28', tests: compat, min: 16} + - {package: minimax, tier: latest, python: '3.12', sdks: '-e python/frameworks/openai openai==3.49.0', tests: compat, min: 16} - {package: deepseek, tier: floor, python: '3.10', sdks: 'openai==1.69.0 httpx<0.28', tests: compat, min: 3} - {package: deepseek, tier: latest, python: '3.12', sdks: 'openai==3.49.0', tests: compat, min: 3} - {package: cerebras, tier: floor, python: '3.10', sdks: 'cerebras-cloud-sdk==1.0.0 httpx<0.28', tests: compat, min: 3} diff --git a/python/frameworks/minimax/traceai_minimax/_wrappers.py b/python/frameworks/minimax/traceai_minimax/_wrappers.py index 67f0dec75..13136bf10 100644 --- a/python/frameworks/minimax/traceai_minimax/_wrappers.py +++ b/python/frameworks/minimax/traceai_minimax/_wrappers.py @@ -27,16 +27,20 @@ _CAPTURE_POLICY_KEY = "traceai.auto.attachment_policy.v1" -def _shared_instrumentor_enabled() -> bool: - """True while traceai_openai's OpenAIInstrumentor is enabled. +def _shared_instrumentor_captures() -> bool: + """True while traceai_openai's OpenAIInstrumentor is enabled WITH an active attachment manager. - With attachments on, the request-level OpenAI instrumentor then owns the call (span - and media), whatever the setup order. Do not import the optional package; inspect - live state on each call. + The request-level OpenAI instrumentor then owns the call (span and media), whatever + the setup order. Enabled without a manager, or an older traceai_openai without the + accessor, means it would capture nothing, so this binding keeps the call. Do not + import the optional package; inspect live state on each call. """ module = sys.modules.get("traceai_openai") instrumentor = getattr(module, "OpenAIInstrumentor", None) - return bool(instrumentor and instrumentor().is_instrumented_by_opentelemetry) + if instrumentor is None: + return False + captures = getattr(instrumentor(), "captures_attachments", None) + return bool(callable(captures) and captures()) class _WithTracer(ABC): @@ -64,11 +68,13 @@ def _capture_chat_media(self, span: _WithSpan, messages: Any) -> None: logger.debug("Attachment capture unavailable", exc_info=True) def _defers(self) -> bool: - """Hand the whole call to an enabled traceai_openai, only while attachments are on. + """Hand the whole call to traceai_openai only while both sides capture attachments. - With attachments off this binding traces exactly as before (its own span too). + With attachments off here this binding traces exactly as before (its own span too). + With traceai_openai enabled but without a manager, this binding keeps the call and + captures its media itself. """ - return self._attachments is not None and _shared_instrumentor_enabled() + return self._attachments is not None and _shared_instrumentor_captures() @contextmanager def _deferred(self) -> Iterator[None]: diff --git a/python/frameworks/openai/traceai_openai/__init__.py b/python/frameworks/openai/traceai_openai/__init__.py index 55d01cc34..07c6522fc 100644 --- a/python/frameworks/openai/traceai_openai/__init__.py +++ b/python/frameworks/openai/traceai_openai/__init__.py @@ -77,6 +77,7 @@ def _instrument(self, **kwargs: Any) -> None: # The provider-owned shared manager (register(...).attachment_manager). Auto passes # it as attachment_manager; nothing is captured unless an enabled manager is supplied. attachments = _attachment_manager(kwargs) + self._attachments = attachments openai = import_module(_MODULE) self._original_request = openai.OpenAI.request self._original_async_request = openai.AsyncOpenAI.request @@ -110,7 +111,16 @@ def _instrument(self, **kwargs: Any) -> None: else: self._original_protect = None + def captures_attachments(self) -> bool: + """True while this instrumentor is enabled with an active shared attachment manager. + + Bindings that ride on the OpenAI SDK (xAI, MiniMax) hand a call over to this + instrumentor only then; otherwise they keep tracing and capturing it themselves. + """ + return bool(self.is_instrumented_by_opentelemetry and getattr(self, "_attachments", None) is not None) + def _uninstrument(self, **kwargs: Any) -> None: + self._attachments = None openai = import_module(_MODULE) openai.OpenAI.request = self._original_request openai.AsyncOpenAI.request = self._original_async_request diff --git a/python/frameworks/xai/traceai_xai/_wrappers.py b/python/frameworks/xai/traceai_xai/_wrappers.py index e727cd1e5..222c47aa3 100644 --- a/python/frameworks/xai/traceai_xai/_wrappers.py +++ b/python/frameworks/xai/traceai_xai/_wrappers.py @@ -55,11 +55,13 @@ def _capture_chat_media(self, span: Any, messages: Any) -> None: logger.debug("Attachment capture unavailable", exc_info=True) def _defers(self) -> bool: - """Hand the whole call to an enabled traceai_openai, only while attachments are on. + """Hand the whole call to traceai_openai only while both sides capture attachments. - With attachments off this binding traces exactly as before (its own span too). + With attachments off here this binding traces exactly as before (its own span too). + With traceai_openai enabled but without a manager, this binding keeps the call and + captures its media itself. """ - return self._attachments is not None and _shared_instrumentor_enabled() + return self._attachments is not None and _shared_instrumentor_captures() @contextmanager def _deferred(self) -> Iterator[None]: @@ -110,16 +112,20 @@ def _is_xai_client(instance: Any) -> bool: return False -def _shared_instrumentor_enabled() -> bool: - """True while traceai_openai's OpenAIInstrumentor is enabled. +def _shared_instrumentor_captures() -> bool: + """True while traceai_openai's OpenAIInstrumentor is enabled WITH an active attachment manager. - With attachments on, the request-level OpenAI instrumentor then owns the call (span - and media), whatever the setup order. Do not import the optional package; inspect - live state on each call. + The request-level OpenAI instrumentor then owns the call (span and media), whatever + the setup order. Enabled without a manager, or an older traceai_openai without the + accessor, means it would capture nothing, so this binding keeps the call. Do not + import the optional package; inspect live state on each call. """ module = sys.modules.get("traceai_openai") instrumentor = getattr(module, "OpenAIInstrumentor", None) - return bool(instrumentor and instrumentor().is_instrumented_by_opentelemetry) + if instrumentor is None: + return False + captures = getattr(instrumentor(), "captures_attachments", None) + return bool(callable(captures) and captures()) def _get_input_attributes( diff --git a/python/tests/attachments_compat_providers/test_compat_attachments.py b/python/tests/attachments_compat_providers/test_compat_attachments.py index a6a879d76..1e5d23356 100644 --- a/python/tests/attachments_compat_providers/test_compat_attachments.py +++ b/python/tests/attachments_compat_providers/test_compat_attachments.py @@ -100,8 +100,7 @@ def test_openai_sdk_routes_capture_images(f01, media_provider, instrumented, mon from traceai_xai import XAIInstrumentor as Instrumentor from traceai_xai import _wrappers monkeypatch.setattr(_wrappers, "_is_xai_client", lambda instance: True) - if hasattr(_wrappers, "_shared_instrumentor_enabled"): - monkeypatch.setattr(_wrappers, "_shared_instrumentor_enabled", lambda: False) + monkeypatch.setattr(_wrappers, "_shared_instrumentor_captures", lambda: False) manager = f01.manager() exporter = instrumented(Instrumentor(), manager) @@ -192,6 +191,57 @@ def test_binding_without_attachments_keeps_its_span(media_provider, instrumented assert names == sorted([span_name, "ChatCompletion"] * 2), names +@pytest.mark.parametrize("order", ["openai_first", "binding_first"]) +@pytest.mark.parametrize("package", ["xai", "minimax"]) +def test_binding_keeps_call_when_openai_has_no_manager(f01, media_provider, instrumented, monkeypatch, package, + order): + """Mixed setup: binding WITH a manager, traceai_openai enabled WITHOUT one. The binding keeps the call.""" + openai = pytest.importorskip("openai") + pytest.importorskip("traceai_openai") + pytest.importorskip("traceai_" + package) + from traceai_openai import OpenAIInstrumentor + + manager = f01.manager() + binding, span_name = _compat_binding(package, monkeypatch) + pair = [(OpenAIInstrumentor(), None), (binding, manager)] + if order == "binding_first": + pair.reverse() + for instrumentor, attachments in pair: + exporter = instrumented(instrumentor, attachments) + assert not OpenAIInstrumentor().captures_attachments() + client, async_client = _openai_clients(openai, media_provider) + client.chat.completions.create(model="m", messages=image_messages()) + asyncio.run(async_client.chat.completions.create(model="m", messages=image_messages())) + flush(manager) + assert f01.payloads == [IMAGE, IMAGE] + assert media_owners(exporter) == [(span_name, "model_input")] * 2 + names = sorted(span.name for span in exporter.get_finished_spans()) + assert names == sorted([span_name, "ChatCompletion"] * 2), names + manager.shutdown(5) + + +@pytest.mark.parametrize("package", ["xai", "minimax"]) +def test_older_openai_without_accessor_never_takes_the_call(monkeypatch, package): + """An enabled traceai_openai that predates captures_attachments() cannot capture: do not defer to it.""" + import sys + import types + + pytest.importorskip("traceai_" + package) + if package == "xai": + from traceai_xai import _wrappers + else: + from traceai_minimax import _wrappers + + class LegacyOpenAIInstrumentor: + is_instrumented_by_opentelemetry = True + + monkeypatch.setitem(sys.modules, "traceai_openai", + types.SimpleNamespace(OpenAIInstrumentor=LegacyOpenAIInstrumentor)) + assert _wrappers._shared_instrumentor_captures() is False + monkeypatch.delitem(sys.modules, "traceai_openai") + assert _wrappers._shared_instrumentor_captures() is False + + @pytest.mark.parametrize("flag", [None, "hide_input_images", "hide_inputs", "pii_redaction"]) @pytest.mark.parametrize("package", ["xai", "minimax"]) def test_deferred_call_keeps_binding_local_deny(f01, media_provider, instrumented, monkeypatch, package, flag): From 11e6b916f3e53720f220d5e6560665956cbb7a56 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek <8347993+nik13@users.noreply.github.com> Date: Thu, 12 Nov 2026 12:48:33 -0800 Subject: [PATCH 15/15] test(python): cover the reverse mixed setup (OpenAI captures, binding has no manager) With traceai_openai capturing and xAI/MiniMax instrumented without a manager, the binding keeps its own span (tracing unchanged) and the image is uploaded once, owned by the OpenAI span. CI minimums: xai 17, minimax 18. Co-Authored-By: Claude Opus 5.5 --- ...achments-python-compat-providers-tests.yml | 8 +++--- .../test_compat_attachments.py | 28 +++++++++++++++++++ 2 files changed, 32 insertions(+), 4 deletions(-) diff --git a/.github/workflows/attachments-python-compat-providers-tests.yml b/.github/workflows/attachments-python-compat-providers-tests.yml index 383046b2f..4bf7e5743 100644 --- a/.github/workflows/attachments-python-compat-providers-tests.yml +++ b/.github/workflows/attachments-python-compat-providers-tests.yml @@ -81,12 +81,12 @@ jobs: - {package: together, tier: latest, python: '3.12', sdks: 'together==3.1.1', tests: compat, min: 3} - {package: fireworks, tier: floor, python: '3.10', sdks: 'openai==1.69.0 httpx<0.28', tests: compat, min: 3} - {package: fireworks, tier: latest, python: '3.12', sdks: 'openai==3.49.0', tests: compat, min: 3} - - {package: xai, tier: floor, python: '3.10', sdks: '-e python/frameworks/openai openai==1.69.0 httpx<0.28', tests: compat, min: 15} - - {package: xai, tier: latest, python: '3.12', sdks: '-e python/frameworks/openai openai==3.49.0', tests: compat, min: 15} + - {package: xai, tier: floor, python: '3.10', sdks: '-e python/frameworks/openai openai==1.69.0 httpx<0.28', tests: compat, min: 17} + - {package: xai, tier: latest, python: '3.12', sdks: '-e python/frameworks/openai openai==3.49.0', tests: compat, min: 17} - {package: vllm, tier: floor, python: '3.10', sdks: '-e python/frameworks/openai openai==1.69.0 httpx<0.28', tests: compat, min: 4} - {package: vllm, tier: latest, python: '3.12', sdks: '-e python/frameworks/openai openai==3.49.0', tests: compat, min: 4} - - {package: minimax, tier: floor, python: '3.10', sdks: '-e python/frameworks/openai openai==1.69.0 httpx<0.28', tests: compat, min: 16} - - {package: minimax, tier: latest, python: '3.12', sdks: '-e python/frameworks/openai openai==3.49.0', tests: compat, min: 16} + - {package: minimax, tier: floor, python: '3.10', sdks: '-e python/frameworks/openai openai==1.69.0 httpx<0.28', tests: compat, min: 18} + - {package: minimax, tier: latest, python: '3.12', sdks: '-e python/frameworks/openai openai==3.49.0', tests: compat, min: 18} - {package: deepseek, tier: floor, python: '3.10', sdks: 'openai==1.69.0 httpx<0.28', tests: compat, min: 3} - {package: deepseek, tier: latest, python: '3.12', sdks: 'openai==3.49.0', tests: compat, min: 3} - {package: cerebras, tier: floor, python: '3.10', sdks: 'cerebras-cloud-sdk==1.0.0 httpx<0.28', tests: compat, min: 3} diff --git a/python/tests/attachments_compat_providers/test_compat_attachments.py b/python/tests/attachments_compat_providers/test_compat_attachments.py index 1e5d23356..6ad1c0726 100644 --- a/python/tests/attachments_compat_providers/test_compat_attachments.py +++ b/python/tests/attachments_compat_providers/test_compat_attachments.py @@ -220,6 +220,34 @@ def test_binding_keeps_call_when_openai_has_no_manager(f01, media_provider, inst manager.shutdown(5) +@pytest.mark.parametrize("order", ["openai_first", "binding_first"]) +@pytest.mark.parametrize("package", ["xai", "minimax"]) +def test_binding_without_manager_keeps_its_span_next_to_capturing_openai(f01, media_provider, instrumented, + monkeypatch, package, order): + """Mixed setup: traceai_openai WITH a manager, binding WITHOUT one. Binding tracing is unchanged.""" + openai = pytest.importorskip("openai") + pytest.importorskip("traceai_openai") + pytest.importorskip("traceai_" + package) + from traceai_openai import OpenAIInstrumentor + + manager = f01.manager() + binding, span_name = _compat_binding(package, monkeypatch) + pair = [(OpenAIInstrumentor(), manager), (binding, None)] + if order == "binding_first": + pair.reverse() + for instrumentor, attachments in pair: + exporter = instrumented(instrumentor, attachments) + assert OpenAIInstrumentor().captures_attachments() + client, _ = _openai_clients(openai, media_provider) + client.chat.completions.create(model="m", messages=image_messages()) + flush(manager) + assert f01.payloads == [IMAGE] + assert media_owners(exporter) == [("ChatCompletion", "model_input")] + names = sorted(span.name for span in exporter.get_finished_spans()) + assert names == sorted([span_name, "ChatCompletion"]), names + manager.shutdown(5) + + @pytest.mark.parametrize("package", ["xai", "minimax"]) def test_older_openai_without_accessor_never_takes_the_call(monkeypatch, package): """An enabled traceai_openai that predates captures_attachments() cannot capture: do not defer to it."""