diff --git a/.github/workflows/attachments-python-compat-providers-tests.yml b/.github/workflows/attachments-python-compat-providers-tests.yml new file mode 100644 index 000000000..4bf7e5743 --- /dev/null +++ b/.github/workflows/attachments-python-compat-providers-tests.yml @@ -0,0 +1,151 @@ +name: Python provider media (Gemini, Vertex, Bedrock, LiteLLM, compatible chat) + +# Real provider SDKs (declared floor and latest) through the Python +# instrumentors and fi_instrumentation.attachments.provider_media to a TLS +# loopback F01 attachment server. Outbound network is disabled while the tests run. + +on: + pull_request: + paths: + - 'python/fi_instrumentation/attachments/**' + - 'python/frameworks/google-genai/**' + - 'python/frameworks/vertexai/**' + - 'python/frameworks/bedrock/**' + - 'python/frameworks/litellm/**' + - 'python/frameworks/groq/**' + - 'python/frameworks/together/**' + - 'python/frameworks/fireworks/**' + - 'python/frameworks/xai/**' + - 'python/frameworks/deepseek/**' + - 'python/frameworks/cerebras/**' + - 'python/frameworks/mistralai/**' + - 'python/frameworks/openrouter/**' + - 'python/frameworks/portkey/**' + - 'python/frameworks/ollama/**' + - 'python/frameworks/vllm/**' + - 'python/frameworks/minimax/**' + - 'python/frameworks/openai/**' + - 'python/tests/attachments_providers/conftest.py' + - 'python/tests/attachments_compat_providers/**' + - '.github/workflows/attachments-python-compat-providers-tests.yml' + push: + branches: [dev] + paths: + - 'python/fi_instrumentation/attachments/**' + - 'python/frameworks/google-genai/**' + - 'python/frameworks/vertexai/**' + - 'python/frameworks/bedrock/**' + - 'python/frameworks/litellm/**' + - 'python/frameworks/groq/**' + - 'python/frameworks/together/**' + - 'python/frameworks/fireworks/**' + - 'python/frameworks/xai/**' + - 'python/frameworks/deepseek/**' + - 'python/frameworks/cerebras/**' + - 'python/frameworks/mistralai/**' + - 'python/frameworks/openrouter/**' + - 'python/frameworks/portkey/**' + - 'python/frameworks/ollama/**' + - 'python/frameworks/vllm/**' + - 'python/frameworks/minimax/**' + - 'python/frameworks/openai/**' + - 'python/tests/attachments_providers/conftest.py' + - 'python/tests/attachments_compat_providers/**' + - '.github/workflows/attachments-python-compat-providers-tests.yml' + workflow_dispatch: + +permissions: + contents: read + +jobs: + media: + name: Provider media (${{ matrix.package }} ${{ matrix.tier }}) + runs-on: ubuntu-24.04 + timeout-minutes: 25 + strategy: + fail-fast: false + matrix: + include: + - {package: core, tier: latest, python: '3.12', sdks: '', tests: core, min: 54} + - {package: google-genai, tier: floor, python: '3.10', sdks: 'google-genai==1.8.0', tests: google, min: 9} + - {package: google-genai, tier: latest, python: '3.12', sdks: 'google-genai==3.2.0', tests: google, min: 9} + - {package: vertexai, tier: floor, python: '3.10', sdks: 'google-cloud-aiplatform==1.61.0', tests: google, min: 5} + - {package: vertexai, tier: latest, python: '3.12', sdks: 'google-cloud-aiplatform==1.101.0', tests: google, min: 5} + - {package: bedrock, tier: floor, python: '3.10', sdks: 'boto3==1.38.17', tests: aws, min: 12} + - {package: bedrock, tier: latest, python: '3.12', sdks: 'boto3==1.44.1', tests: aws, min: 12} + - {package: litellm, tier: floor, python: '3.10', sdks: '-e python/frameworks/openai litellm==1.53.1 openai==1.69.0', tests: aws, min: 9} + - {package: litellm, tier: latest, python: '3.12', sdks: '-e python/frameworks/openai litellm==1.109.1', tests: aws, min: 10} + - {package: groq, tier: floor, python: '3.10', sdks: 'groq==0.9.0 httpx<0.28', tests: compat, min: 2} + - {package: groq, tier: latest, python: '3.12', sdks: 'groq==1.7.0', tests: compat, min: 2} + - {package: together, tier: floor, python: '3.10', sdks: 'together==1.3.0', tests: compat, min: 3} + - {package: together, tier: latest, python: '3.12', sdks: 'together==3.1.1', tests: compat, min: 3} + - {package: fireworks, tier: floor, python: '3.10', sdks: 'openai==1.69.0 httpx<0.28', tests: compat, min: 3} + - {package: fireworks, tier: latest, python: '3.12', sdks: 'openai==3.49.0', tests: compat, min: 3} + - {package: xai, tier: floor, python: '3.10', sdks: '-e python/frameworks/openai openai==1.69.0 httpx<0.28', tests: compat, min: 17} + - {package: xai, tier: latest, python: '3.12', sdks: '-e python/frameworks/openai openai==3.49.0', tests: compat, min: 17} + - {package: vllm, tier: floor, python: '3.10', sdks: '-e python/frameworks/openai openai==1.69.0 httpx<0.28', tests: compat, min: 4} + - {package: vllm, tier: latest, python: '3.12', sdks: '-e python/frameworks/openai openai==3.49.0', tests: compat, min: 4} + - {package: minimax, tier: floor, python: '3.10', sdks: '-e python/frameworks/openai openai==1.69.0 httpx<0.28', tests: compat, min: 18} + - {package: minimax, tier: latest, python: '3.12', sdks: '-e python/frameworks/openai openai==3.49.0', tests: compat, min: 18} + - {package: deepseek, tier: floor, python: '3.10', sdks: 'openai==1.69.0 httpx<0.28', tests: compat, min: 3} + - {package: deepseek, tier: latest, python: '3.12', sdks: 'openai==3.49.0', tests: compat, min: 3} + - {package: cerebras, tier: floor, python: '3.10', sdks: 'cerebras-cloud-sdk==1.0.0 httpx<0.28', tests: compat, min: 3} + - {package: cerebras, tier: latest, python: '3.12', sdks: 'cerebras-cloud-sdk==1.91.0', tests: compat, min: 3} + - {package: mistralai, tier: floor, python: '3.10', sdks: 'mistralai==1.1.0', tests: compat, min: 5} + - {package: mistralai, tier: latest, python: '3.12', sdks: 'mistralai==3.7.0', tests: compat, min: 5} + - {package: openrouter, tier: floor, python: '3.10', sdks: 'openrouter==1.3.0', tests: compat, min: 3} + - {package: openrouter, tier: latest, python: '3.12', sdks: 'openrouter==1.3.107', tests: compat, min: 3} + - {package: portkey, tier: floor, python: '3.10', sdks: 'portkey-ai==1.9.0 httpx<0.28', tests: compat, min: 3} + - {package: portkey, tier: latest, python: '3.12', sdks: 'portkey-ai==2.3.4', tests: compat, min: 3} + - {package: ollama, tier: floor, python: '3.10', sdks: 'ollama==0.3.0', tests: compat, min: 4} + - {package: ollama, tier: latest, python: '3.12', sdks: 'ollama==0.6.3', tests: compat, min: 4} + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: false + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: ${{ matrix.python }} + - name: Install the core, the integration and the selected SDKs + env: + PACKAGE: ${{ matrix.package }} + SDKS: ${{ matrix.sdks }} + run: | + python -m venv .venv + integration="" + if [ "$PACKAGE" != core ]; then integration="-e python/frameworks/$PACKAGE"; fi + # shellcheck disable=SC2086 + .venv/bin/pip install -e python $integration opentelemetry-sdk 'pytest>=8,<9' 'pytest-asyncio<0.25' $SDKS + .venv/bin/pip freeze | grep -iE '^(google-genai|google-cloud-aiplatform|boto3|litellm|openai|groq|together|cerebras-cloud-sdk|mistralai|openrouter|portkey-ai|ollama|httpx)==' || true + - name: Run with outbound network disabled + # bash -eo pipefail: a failing pytest is not hidden by tee. + shell: bash + env: + SUITE: ${{ matrix.tests }} + MIN: ${{ matrix.min }} + run: | + case "$SUITE" in + core) tests="tests/attachments_compat_providers/test_provider_media.py" ;; + google) tests="tests/attachments_compat_providers/test_google_attachments.py" ;; + aws) tests="tests/attachments_compat_providers/test_bedrock_litellm_attachments.py" ;; + compat) tests="tests/attachments_compat_providers/test_compat_attachments.py" ;; + esac + # A file and a node ID inside the same file would collapse to the node ID only, + # so the descriptor rows are added only where the compat file is not already selected. + case "$SUITE" in + google|aws) tests="$tests tests/attachments_compat_providers/test_compat_attachments.py::test_package_descriptor" ;; + esac + if [ "$SUITE" != core ]; then + tests="$tests tests/attachments_compat_providers/test_provider_media.py::test_disabled_path_never_imports_attachment_modules" + fi + sudo unshare --net -- bash -e -c ' + ip link set lo up + cd "$2/python" + export FI_BASE_URL=http://127.0.0.1:9 FI_GRPC_URL=http://127.0.0.1:9 + export LITELLM_LOCAL_MODEL_COST_MAP=True AWS_EC2_METADATA_DISABLED=true + # shellcheck disable=SC2086 + PYTHONPATH=. "$1" -m pytest $3 -v -rs -p no:cacheprovider -o addopts="" -o faulthandler_timeout=180 + ' _ "$PWD/.venv/bin/python" "$PWD" "$tests" | tee pytest.log + # A job whose SDK rows silently skipped must not pass: require the rows this package proves. + passed=$(grep -Eo '[0-9]+ passed' pytest.log | tail -1 | cut -d' ' -f1) + if [ "${passed:-0}" -lt "$MIN" ]; then echo "expected at least $MIN passed, got ${passed:-0}"; exit 1; fi diff --git a/python/fi_instrumentation/attachments/media_capture.py b/python/fi_instrumentation/attachments/media_capture.py index a2fd637c4..1673fd5f9 100644 --- a/python/fi_instrumentation/attachments/media_capture.py +++ b/python/fi_instrumentation/attachments/media_capture.py @@ -13,6 +13,11 @@ - One registry per manager remembers admitted (span, position) pairs, so a second observer of the same span and position gets the FIRST occurrence back and uploads nothing. +- A per-call ``CapturePolicy`` set in the OpenTelemetry context under + ``CAPTURE_POLICY_KEY`` (``traceai.with_attachment_capture_policy``) can only + add restrictions. A context marked under ``OWNER_KEY`` + (``traceai.with_attachment_ownership``) belongs to an adapter that captures + the media itself, so nothing is captured here for that call. Every public function is non-throwing and returns ``None``/empty on denial. """ @@ -26,6 +31,8 @@ from collections import OrderedDict from typing import Any, Callable, Iterable, Mapping, NamedTuple, Optional, Tuple +from opentelemetry import context as otel_context + from .policy import CapturePolicy from .stream import StreamOptions from .types import Bytes, CorrelationHandle @@ -50,6 +57,13 @@ # mulaw have no F01 MIME, so they are never captured. SPEECH_FORMATS = {"mp3": "audio/mpeg", "opus": "audio/ogg", "ogg": "audio/ogg", "flac": "audio/flac", "wav": "audio/wav"} +# Context keys shared BY NAME with the auto module (``traceai``) and any adapter, like the +# TypeScript ``Symbol.for`` keys: plain strings, so no package has to import another. +CAPTURE_POLICY_KEY = "traceai.auto.attachment_policy.v1" +OWNER_KEY = "traceai.auto.attachment_owner.v1" +# CapturePolicy facts that grant capture; every other fact only restricts. +_PERMISSIONS = frozenset(("capture_content", "record_inputs", "record_outputs", "requested")) +_DENY_ALL = CapturePolicy(capture_content=False, requested=False) class Occurrence(NamedTuple): @@ -93,6 +107,40 @@ def _restricted(config, name: str) -> bool: return True +def merge_capture_policies(*policies: Any) -> CapturePolicy: + """Deny-wins intersection: a grant needs every policy, a restriction needs any one. + + ``None`` adds nothing. Any other value that is not a ``CapturePolicy`` denies. + """ + present = [policy for policy in policies if policy is not None] + if any(not isinstance(policy, CapturePolicy) for policy in present): + return _DENY_ALL + if not present: + return CapturePolicy() + values = {} + for item in dataclasses.fields(CapturePolicy): + facts = [getattr(policy, item.name) for policy in present] + values[item.name] = all(facts) if item.name in _PERMISSIONS else any(facts) + return CapturePolicy(**values) + + +def context_capture_policy(context=None) -> Any: + """The per-call policy in the (current) OpenTelemetry context; None when unset.""" + try: + return otel_context.get_value(CAPTURE_POLICY_KEY, context) + except Exception: + # An unreadable restriction is a restriction, never permission. + return _DENY_ALL + + +def owned_elsewhere(context=None) -> bool: + """True when another adapter marked this call's media as its own.""" + try: + return otel_context.get_value(OWNER_KEY, context) is not None + except Exception: + return True + + def local_capture_policy(trace_config=None, *, direction: str, image: bool = False, error_detail: bool = False, base: Optional[CapturePolicy] = None) -> CapturePolicy: """Fold the instrumentor's local TraceConfig into a deny-only CapturePolicy. @@ -103,6 +151,7 @@ def local_capture_policy(trace_config=None, *, direction: str, image: bool = Fal ancestry, so a tool screenshot is still treated as model input. """ base = base if isinstance(base, CapturePolicy) else CapturePolicy() + base = merge_capture_policies(base, context_capture_policy()) names = _INPUT_HIDE if direction == "input" else _OUTPUT_HIDE hidden = any(_restricted(trace_config, name) for name in names) or _restricted(trace_config, "pii_redaction") if image and direction == "input" and _restricted(trace_config, "hide_input_images"): @@ -116,6 +165,8 @@ def local_capture_policy(trace_config=None, *, direction: str, image: bool = Fal changes["model_input"] = True if error_detail: changes["error_detail"] = True + if owned_elsewhere(): + changes["requested"] = False return dataclasses.replace(base, **changes) @@ -262,7 +313,7 @@ class Observation: def __init__(self, manager, span, namespace: str): self.manager, self.span, self.namespace = manager, span, namespace - self.key = span_key(span) if active(manager) else None + self.key = span_key(span) if active(manager) and not owned_elsewhere() else None self.captured: "OrderedDict[str, Occurrence]" = OrderedDict() self._visited = 0 diff --git a/python/fi_instrumentation/attachments/provider_media.py b/python/fi_instrumentation/attachments/provider_media.py new file mode 100644 index 000000000..b3fff768d --- /dev/null +++ b/python/fi_instrumentation/attachments/provider_media.py @@ -0,0 +1,764 @@ +"""Typed provider media extraction shared by the Python provider instrumentors. + +Mirrors ``@traceai/provider-media`` (TypeScript): one helper, thin per-package +bindings, and a per-package capability table that is also the support matrix. + +Rules (same as ``media_capture``, which this builds on): + +- Typed shapes only: Gemini ``inline_data`` Parts, OpenAI-compatible + ``image_url`` data URIs and ``input_audio``, Bedrock Converse/Nova + ``image``/``document``/``video`` byte blocks, Anthropic-on-Bedrock base64 + blocks, Titan/Nova Canvas image fields, Ollama ``images`` and generated-image + ``b64_json``. No recursive walk of unknown bodies, no URL fetch, no provider + file or ``gs://``/S3 lookup, no filesystem access. +- ``manager.preflight`` runs BEFORE a source is read, sized beyond its shape, + decoded or sniffed. A preflight denied only for size is retried once at the + exact size; a source whose exact type differs from the gate is re-admitted + under that type. Unused tickets are released at once. +- After admission the leading bytes must match the declared type (a PNG + declared as ``audio/wav`` is not uploaded): a hide flag that applies to the + real modality can never be bypassed through a wrong label. +- Local ``TraceConfig`` hide/PII flags, the per-call context policy + (``media_capture.CAPTURE_POLICY_KEY``) and the owner mark + (``media_capture.OWNER_KEY``) apply through ``media_capture`` (deny wins). +- Every public function is non-throwing and returns an empty mapping on denial. + +Not in ``attachments.__all__``: an adapter-facing module, like ``media_capture``. +""" + +import re +from collections.abc import Sequence +from itertools import islice +from types import MappingProxyType +from typing import Any, Callable, Mapping, Optional, Tuple + +from opentelemetry import context as otel_context + +from . import media_capture as mc +from .policy import CapturePolicy +from .types import Bytes + +KiB = 1024 +MiB = 1024 * KiB +#: Bound on synchronous base64 decoding, independent of the manager queue (as in TypeScript). +#: Raw bytes the SDK already holds are bounded by the manager alone (``too_large``/``queue_full``). +MAX_DECODED_BYTES = 1 * MiB +MAX_PARTS = 64 +DATA_URI_TYPES = ("image/png", "image/jpeg", "image/webp") +IMAGE_TYPES = DATA_URI_TYPES +DOCUMENT_TYPES = ("application/pdf",) +AUDIO_TYPES = ("audio/wav", "audio/mpeg", "audio/ogg", "audio/flac", "audio/webm") +VIDEO_TYPES = ("video/mp4", "video/webm") +# Gemini mime_type spellings that name an F01 type. Anything else (audio/L16 PCM, text/*, +# image/heic, ...) is not captured. +GEMINI_TYPES = { + **{mime: mime for mime in IMAGE_TYPES + DOCUMENT_TYPES + AUDIO_TYPES + VIDEO_TYPES}, + "audio/mp3": "audio/mpeg", "audio/x-wav": "audio/wav", "audio/wave": "audio/wav", + "audio/x-flac": "audio/flac", "image/jpg": "image/jpeg", +} +_COMPAT_AUDIO = {"wav": "audio/wav", "mp3": "audio/mpeg"} +_BEDROCK_IMAGE = {"png": "image/png", "jpeg": "image/jpeg", "jpg": "image/jpeg", "webp": "image/webp"} +_BEDROCK_VIDEO = {"mp4": "video/mp4", "webm": "video/webm"} +# Converse / Nova model families whose image and document blocks are qualified. Unknown model +# bodies are never scraped; a model-family name alone is not coverage. +_CONVERSE_MODELS = re.compile( + r"(?:arn:aws[\w-]*:bedrock:[\w-]+:\d*:(?:inference-profile|foundation-model)/)?" + r"(?:(?:us|eu|apac|global)\.)?(?:anthropic\.claude-(?:3|sonnet-4|opus-4|haiku-4)|amazon\.nova-(?:lite|pro|premier))", + re.IGNORECASE) +_NOVA_VIDEO_MODELS = re.compile( + r"(?:arn:aws[\w-]*:bedrock:[\w-]+:\d*:(?:inference-profile|foundation-model)/)?" + r"(?:(?:us|eu|apac|global)\.)?amazon\.nova-(?:lite|pro|premier)", re.IGNORECASE) +_ANTHROPIC_MODELS = re.compile( + r"(?:arn:aws[\w-]*:bedrock:[\w-]+:\d*:(?:inference-profile|foundation-model)/)?" + r"(?:(?:us|eu|apac|global)\.)?anthropic\.claude-", re.IGNORECASE) +_IMAGE_GENERATION_MODELS = re.compile( + r"(?:arn:aws[\w-]*:bedrock:[\w-]+:\d*:(?:inference-profile|foundation-model)/)?" + r"(?:(?:us|eu|apac|global)\.)?amazon\.(?:titan-image-generator-v[12]|nova-canvas-v1)", re.IGNORECASE) +# ISO-BMFF major brands (``ftyp`` bytes 8-12) that name an MP4 video file. HEIF/HEIC and AVIF still +# images, QuickTime, 3GPP and audio-only M4A share the box layout but are not video/mp4, so they are +# not admitted under that label (fail closed: an unlisted brand is unknown). +_MP4_BRANDS = frozenset((b"isom", b"iso2", b"iso3", b"iso4", b"iso5", b"iso6", b"mp41", b"mp42", b"avc1", + b"dash", b"M4V ", b"mmp4")) +_COMPUTER_TOOL = re.compile(r"computer_\d{8}") +# Vertex publisher image-generation models (Imagen). Other prediction endpoints are never read. +_IMAGEN_ENDPOINT = re.compile(r"/models/(?:imagen-|imagegeneration)") +# Titan Image Generator / Nova Canvas request fields that carry one base64 image each. +_IMAGE_PARAMS = ( + ("inPaintingParams", "image"), ("outPaintingParams", "image"), + ("colorGuidedGenerationParams", "referenceImage"), ("backgroundRemovalParams", "image"), + ("textToImageParams", "conditionImage"), +) + +EMPTY: Mapping[str, mc.Occurrence] = mc.EMPTY + + +# ------------------------------------------------------------------ manager and owner +def attachment_manager(manager: Any) -> Any: + """``manager`` when it can admit, else None. Disabled, missing or foreign objects are inert.""" + return manager if mc.active(manager) else None + + +def manager_from_kwargs(kwargs: Mapping[str, Any]) -> Any: + """The shared manager passed as ``attachments`` (or auto's ``attachment_manager``), if it can admit.""" + try: + manager = kwargs.get("attachments") + if manager is None: + manager = kwargs.get("attachment_manager") + return attachment_manager(manager) + except Exception: + return None + + +def claim(context=None): + """The context with this call's media marked as owned (``media_capture.OWNER_KEY``). + + An adapter that captures a call's media itself runs the downstream SDK call in this + context, so nested provider instrumentors (for example OpenAI under LiteLLM) skip it. + """ + return otel_context.set_value(mc.OWNER_KEY, True, context) + + +def deferral_policy(trace_config: Any) -> CapturePolicy: + """An instrumentor's local ``TraceConfig`` as a restriction-only ``CapturePolicy``. + + ``pii_redaction`` denies all capture; an input hide flag or ``hide_input_images`` denies + every input part; an output hide flag denies every output part. The receiving layer + cannot tell an image from another input part through a ``CapturePolicy``, so + ``hide_input_images`` also denies input files and audio on the deferred call (deny wins). + An unreadable flag counts as set. + """ + def restricted(names: Tuple[str, ...]) -> bool: + return any(mc._restricted(trace_config, name) for name in names) + + return CapturePolicy( + capture_content=not restricted(("pii_redaction",)), + record_inputs=not restricted(mc._INPUT_HIDE + ("hide_input_images",)), + record_outputs=not restricted(mc._OUTPUT_HIDE), + ) + + +def deferred(trace_config: Any, context=None): + """The context for a call an instrumentor hands WHOLE to another one (xAI/MiniMax to traceai_openai). + + The deferring instrumentor's local deny is merged (deny wins) into the per-call policy + (``media_capture.CAPTURE_POLICY_KEY``), so the receiving instrumentor cannot capture more + than either configuration allows, whatever its own ``TraceConfig``. A policy already in the + context and the owner mark are kept. Never raises: on any error the call is denied. + """ + try: + policy = mc.merge_capture_policies(mc.context_capture_policy(context), deferral_policy(trace_config)) + except Exception: + policy = CapturePolicy(capture_content=False, requested=False) + return otel_context.set_value(mc.CAPTURE_POLICY_KEY, policy, context) + + +# ------------------------------------------------------------------ type checks +def sniff(head: Any) -> Optional[str]: + """The F01 type named by leading magic bytes the caller already holds; None when unknown.""" + if not isinstance(head, (bytes, bytearray, memoryview)): + return None + head = bytes(head[:16]) + if head[:8] == b"\x89PNG\r\n\x1a\n": + return "image/png" + if head[:3] == b"\xff\xd8\xff": + return "image/jpeg" + if len(head) >= 12 and head[:4] == b"RIFF" and head[8:12] == b"WEBP": + return "image/webp" + if head[:5] == b"%PDF-": + return "application/pdf" + if len(head) >= 12 and head[4:8] == b"ftyp": + return "video/mp4" if head[8:12] in _MP4_BRANDS else None + return mc.sniff_audio(head) + + +def consistent(mime: str, data: Any) -> bool: + """True when the leading bytes are the declared type (EBML may be audio or video WebM).""" + found = sniff(data) + if found is None: + return False + if found == "audio/webm": + return mime in ("audio/webm", "video/webm") + return found == mime + + +def _modality(mime: str) -> str: + return mime.split("/", 1)[0] + + +# ------------------------------------------------------------------ sources +class _Source: + """A source described from its shape: exact type, exact size, and a read that runs after admission.""" + + __slots__ = ("mime", "size", "take") + + def __init__(self, mime: str, size: int, take: Callable[[int], Optional[bytes]]): + self.mime, self.size, self.take = mime, size, take + + +def _bytes_source(data: Any, mime: Optional[str]) -> Optional[_Source]: + if not isinstance(data, (bytes, bytearray, memoryview)): + return None + size = data.nbytes if isinstance(data, memoryview) else len(data) + if size <= 0: + return None + if mime is None: + mime = sniff(data[:16]) + if mime is None: + return None + + def take(max_bytes: int) -> Optional[bytes]: + return bytes(data) if size <= max_bytes else None + + return _Source(mime, size, take) + + +def _base64_source(encoded: Any, mime: Optional[str]) -> Optional[_Source]: + size = mc.encoded_length(encoded, MAX_DECODED_BYTES) + if not size: + return None + decoded = {} + + def decode(max_bytes: int) -> Optional[bytes]: + if "data" not in decoded: + decoded["data"] = mc.strict_decode(encoded, max_bytes, MAX_DECODED_BYTES) + return decoded["data"] + + if mime is None: + # Untyped base64 (Ollama, Titan): the exact type comes from the decoded magic bytes. + # Only described after an exact-size image gate was admitted, so this is post-admission. + data = decode(MAX_DECODED_BYTES) + mime = sniff(data[:16]) if data else None + if mime is None: + return None + return _Source(mime, size, decode) + + +def _data_uri_source(value: Any, types: Tuple[str, ...] = DATA_URI_TYPES) -> Optional[_Source]: + # split_inline bounds the length before it slices, so an oversized string is never copied. + found = mc.split_inline(value, types, False, MAX_DECODED_BYTES) + if found is None: + return None + mime, encoded, _ = found + return _base64_source(encoded, mime) + + +# ------------------------------------------------------------------ capture +class ProviderMedia: + """One adapter observation of one span. Typed extraction only; never raises. + + ``span`` is the real OpenTelemetry span the instrumentor emitted (unwrap any + ``_WithSpan`` helper first). ``namespace`` keeps positions of different + adapters on the same span apart. + """ + + def __init__(self, manager, span, namespace: str, *, trace_config=None): + self.manager, self.span, self.trace_config = attachment_manager(manager), span, trace_config + self.observation = mc.Observation(self.manager, span, namespace) if self.manager is not None else None + self._attempted = 0 + + @property + def usable(self) -> bool: + return self.observation is not None and self.observation.usable + + @property + def captured(self) -> Mapping[str, mc.Occurrence]: + if not self.usable or not self.observation.captured: + return EMPTY + return MappingProxyType(dict(self.observation.captured)) + + # -------------------------------------------------------------- admission + def _admit(self, position: str, gate_mime: str, gate_bytes: int, describe: Callable[[], Optional[_Source]], *, + direction: str, purpose: str, role: Optional[str] = None, message_index: Optional[int] = None, + part_index: Optional[int] = None, tool_call_id: Optional[str] = None, + error_detail: bool = False, exact: bool = True) -> None: + """Gate, describe, (re-)admit at the exact type and size, read, check the magic bytes, attach. + + ``exact`` means ``gate_bytes`` is already the source's exact size (bytes length, or the + decoded length from the base64 shape), so a size denial is final and nothing is described. + """ + if not self.usable or self._attempted >= MAX_PARTS or gate_bytes <= 0: + return + self._attempted += 1 + manager, span = self.manager, self.span + + def policy(mime): + return mc.local_capture_policy(self.trace_config, direction=direction, image=_modality(mime) == "image", + error_detail=error_detail) + + def capture() -> Optional[mc.Occurrence]: + taken = [] + attached = False + + def preflight(mime, size): + ticket = manager.preflight(span, mime_type=mime, direction=direction, max_bytes=size, + purpose=purpose, capture_policy=policy(mime)) + taken.append(ticket) + return ticket + + try: + ticket = preflight(gate_mime, gate_bytes) + # too_large / queue_full come after every privacy, correlation and sampling step, + # so describing the source to size it exactly is still safe. + if not ticket.admitted and (exact or ticket.reason not in ("too_large", "queue_full")): + return None + source = describe() + if source is None or source.size <= 0: + return None + if not ticket.admitted or source.mime != gate_mime or source.size > ticket.max_bytes: + ticket.release() + ticket = preflight(source.mime, source.size) + if not ticket.admitted: + return None + data = source.take(ticket.max_bytes) + if data is None or not consistent(source.mime, data): + return None + attached = True + handle = ticket.attach(Bytes(data), **mc._fields(message_index, part_index, role, tool_call_id)) + return mc.Occurrence(handle.occurrence_id, source.mime) if handle.admission == "queued" else None + except Exception: + return None + finally: + if not attached: + for ticket in taken: + try: + ticket.release() + except Exception: + pass + + try: + self.observation.visit(position, capture) + except Exception: + pass + + def _typed(self, position: str, data: Any, mime: str, **fields) -> None: + """Bytes or raw base64 of a declared type.""" + if isinstance(data, (bytes, bytearray, memoryview)): + size = data.nbytes if isinstance(data, memoryview) else len(data) + self._admit(position, mime, size, lambda: _bytes_source(data, mime), **fields) + elif isinstance(data, str): + size = mc.encoded_length(data, MAX_DECODED_BYTES) + if size: + self._admit(position, mime, size, lambda: _base64_source(data, mime), **fields) + + def _untyped_image(self, position: str, data: Any, **fields) -> None: + """Bytes or raw base64 with no declared type: one image gate, then the magic bytes decide.""" + if isinstance(data, (bytes, bytearray, memoryview)): + size = data.nbytes if isinstance(data, memoryview) else len(data) + self._admit(position, "image/png", size, lambda: _image_only(_bytes_source(data, None)), **fields) + elif isinstance(data, str): + size = mc.encoded_length(data, MAX_DECODED_BYTES) + if size: + self._admit(position, "image/png", size, lambda: _image_only(_base64_source(data, None)), **fields) + + def _data_uri(self, position: str, read: Callable[[], Any], types: Tuple[str, ...] = DATA_URI_TYPES, + **fields) -> None: + # Every candidate type shares one modality, so one gate decides the part before the + # string is parsed. A remote or signed URL is never kept as fallback metadata. + self._admit(position, types[0], MAX_DECODED_BYTES, lambda: _data_uri_source(read(), types), exact=False, + **fields) + + # -------------------------------------------------------------- Gemini + def gemini_contents(self, contents: Any, direction: str = "input") -> "ProviderMedia": + """Gemini ``Content``/``Part`` values: ``inline_data`` only (``file_data`` URIs are never resolved).""" + if not self.usable: + return self + try: + values = _items(contents) if isinstance(contents, Sequence) and not isinstance(contents, str) else [contents] + for m, content in enumerate(values): + parts = mc.get(content, "parts") + if parts is None and _inline(content) is not None: + parts = [content] # A bare Part (or list of Parts) is one single-part message. + role = mc.get(content, "role") + role = "assistant" if role == "model" or direction == "output" else "user" + for p, part in enumerate(_items(parts)[:MAX_PARTS]): + inline = _inline(part) + if inline is None: + continue + mime = GEMINI_TYPES.get(_lower(mc.get(inline, "mime_type") or mc.get(inline, "mimeType"))) + if mime is None: + continue + self._typed("gemini/%s/%d/%d" % (direction, m, p), mc.get(inline, "data"), mime, + direction=direction, purpose=_purpose(direction, mime), role=role, + message_index=m, part_index=p) + except Exception: + pass + return self + + def gemini_response(self, response: Any) -> "ProviderMedia": + """Completed ``GenerateContentResponse``: candidate ``inline_data`` (generated images, audio, files).""" + if not self.usable: + return self + try: + candidates = _items(mc.get(response, "candidates"))[:MAX_PARTS] + self.gemini_contents([mc.get(candidate, "content") for candidate in candidates], "output") + except Exception: + pass + return self + + # -------------------------------------------------------------- OpenAI-compatible chat + def compatible_messages(self, messages: Any, dialect: str = "openai", *, + documents: bool = False) -> "ProviderMedia": + """Chat ``image_url`` data URIs, plus ``input_audio`` (wav/mp3) for OpenAI-dialect routes. + + ``groq``, ``mistral`` and ``image`` dialects are image only; ``mistral`` also accepts + ``image_url`` as a plain string. ``documents=True`` (OpenAI dialect only, for a route whose + PDF parts are qualified) also captures PDF ``file`` (``file.file_data``) and ``input_file`` + (``file_data``) data URIs. Provider file IDs and URLs are never resolved. + """ + if not self.usable: + return self + try: + for m, message in enumerate(_items(messages)[:MAX_PARTS]): + role = _role(mc.get(message, "role")) + for p, part in enumerate(_items(mc.get(message, "content"))[:MAX_PARTS]): + kind = mc.get(part, "type") + fields = dict(direction="input", purpose="model_input", role=role, message_index=m, part_index=p) + if kind == "image_url": + def read(part=part): + image = mc.get(part, "image_url") + if image is None and dialect == "mistral": + image = mc.get(part, "imageUrl") + return image if isinstance(image, str) else mc.get(image, "url") + + self._data_uri("chat/%d/%d" % (m, p), read, **fields) + elif kind == "input_audio" and dialect == "openai": + audio = mc.get(part, "input_audio") + mime = _COMPAT_AUDIO.get(mc.get(audio, "format")) + if mime is not None: + self._typed("chat/%d/%d" % (m, p), mc.get(audio, "data"), mime, **fields) + elif kind in ("file", "input_file") and documents and dialect == "openai": + def read_file(part=part, kind=kind): + return mc.get(mc.get(part, "file") if kind == "file" else part, "file_data") + + fields["purpose"] = "document" + self._data_uri("chat/%d/%d" % (m, p), read_file, DOCUMENT_TYPES, **fields) + except Exception: + pass + return self + + def ollama_images(self, images: Any, *, message_index: Optional[int] = None, role: str = "user") -> "ProviderMedia": + """Ollama ``images``: bytes or base64 only. Path strings and ``Path`` objects are never opened.""" + if not self.usable: + return self + try: + for p, image in enumerate(_items(images)[:MAX_PARTS]): + value = mc.get(image, "value") if not isinstance(image, (str, bytes, bytearray)) else image + position = "ollama/%s/%d" % ("-" if message_index is None else message_index, p) + self._untyped_image(position, value, direction="input", purpose="model_input", role=role, + message_index=message_index, part_index=p) + except Exception: + pass + return self + + def ollama_messages(self, messages: Any) -> "ProviderMedia": + if not self.usable: + return self + try: + for m, message in enumerate(_items(messages)[:MAX_PARTS]): + self.ollama_images(mc.get(message, "images"), message_index=m, role=_role(mc.get(message, "role"))) + except Exception: + pass + return self + + def generated_images(self, data: Any, *, field: str = "b64_json") -> "ProviderMedia": + """Generated-image results that carry base64 (``b64_json``); URL results are never fetched.""" + if not self.usable: + return self + try: + for index, item in enumerate(_items(data)[:MAX_PARTS]): + self._untyped_image("generated/%d" % index, mc.get(item, field), direction="output", + purpose="model_output", role="assistant", part_index=index) + except Exception: + pass + return self + + # -------------------------------------------------------------- Bedrock + def bedrock_converse(self, model_id: Any, messages: Any, additional_fields: Any = None) -> "ProviderMedia": + """Converse (and Nova InvokeModel) ``image``/``document``/``video`` blocks of qualified model families.""" + if not self.usable or not isinstance(model_id, str) or not _CONVERSE_MODELS.match(model_id): + return self + try: + video = bool(_NOVA_VIDEO_MODELS.match(model_id)) + messages = _items(messages)[:MAX_PARTS] + computer = _computer_calls(messages, mc.get(additional_fields, "tools"), "toolUse", "toolUseId", + nested="toolUse") + for m, message in enumerate(messages): + role = _role(mc.get(message, "role")) + for p, block in enumerate(_items(mc.get(message, "content"))[:MAX_PARTS]): + self._bedrock_block("bedrock/%d/%d" % (m, p), block, video, role=role, message_index=m, + part_index=p) + result = mc.get(block, "toolResult") + if result is None: + continue + tool_id = mc.get(result, "toolUseId") + if not isinstance(tool_id, str) or not mc.TOOL_ID.fullmatch(tool_id): + continue # Invalid tool IDs fail closed instead of losing causal identity. + for q, child in enumerate(_items(mc.get(result, "content"))[:MAX_PARTS]): + self._bedrock_block("bedrock/%d/%d/%d" % (m, p, q), child, video, role="tool", + message_index=m, part_index=p, tool_call_id=tool_id, + screenshot=tool_id in computer, + error_detail=mc.get(result, "status") == "error") + except Exception: + pass + return self + + def _bedrock_block(self, position: str, block: Any, video: bool, *, tool_call_id: Optional[str] = None, + screenshot: bool = False, error_detail: bool = False, **fields) -> None: + for kind in ("image", "document", "video"): + media = mc.get(block, kind) + if media is None: + continue + fmt = _lower(mc.get(media, "format")) + if kind == "image": + mime = _BEDROCK_IMAGE.get(fmt) + elif kind == "document": + mime = "application/pdf" if fmt == "pdf" else None + else: + mime = _BEDROCK_VIDEO.get(fmt) if video else None + if mime is None: + return + # Only inline bytes (raw for Converse, base64 for Nova InvokeModel); S3 locations stay unsupported. + data = mc.get(mc.get(media, "source"), "bytes") + if tool_call_id is not None: + purpose = "computer_screenshot" if screenshot and kind == "image" else "tool_result" + else: + purpose = "document" if kind == "document" else "model_input" + self._typed(position, data, mime, direction="input", purpose=purpose, tool_call_id=tool_call_id, + error_detail=error_detail, **fields) + return + + def anthropic_messages(self, messages: Any, tools: Any = None) -> "ProviderMedia": + """Anthropic Messages bodies (Bedrock InvokeModel): base64 image/PDF blocks and one tool_result level.""" + if not self.usable: + return self + try: + messages = _items(messages)[:MAX_PARTS] + computer = _computer_calls(messages, tools, "tool_use", "id") + for m, message in enumerate(messages): + role = _role(mc.get(message, "role")) + for p, block in enumerate(_items(mc.get(message, "content"))[:MAX_PARTS]): + if mc.get(block, "type") == "tool_result": + tool_id = mc.get(block, "tool_use_id") + if not isinstance(tool_id, str) or not mc.TOOL_ID.fullmatch(tool_id): + continue + for q, child in enumerate(_items(mc.get(block, "content"))[:MAX_PARTS]): + # part_index is the tool_result block, as for Converse; q only keeps positions apart. + self._anthropic_block("anthropic/%d/%d/%d" % (m, p, q), child, role="tool", + message_index=m, part_index=p, tool_call_id=tool_id, + screenshot=tool_id in computer, + error_detail=mc.get(block, "is_error") is True) + else: + self._anthropic_block("anthropic/%d/%d" % (m, p), block, role=role, message_index=m, + part_index=p) + except Exception: + pass + return self + + def _anthropic_block(self, position: str, block: Any, *, tool_call_id: Optional[str] = None, + screenshot: bool = False, error_detail: bool = False, **fields) -> None: + kind = mc.get(block, "type") + source = mc.get(block, "source") + if kind not in ("image", "document") or mc.get(source, "type") != "base64": + return + mime = mc.get(source, "media_type") + if mime not in (IMAGE_TYPES if kind == "image" else DOCUMENT_TYPES): + return + if kind == "document": + purpose = "document" + elif tool_call_id is not None: + purpose = "computer_screenshot" if screenshot else "tool_result" + else: + purpose = "model_input" + self._typed(position, mc.get(source, "data"), mime, direction="input", purpose=purpose, + tool_call_id=tool_call_id, error_detail=error_detail, **fields) + + def imagen_predictions(self, endpoint: Any, predictions: Any) -> "ProviderMedia": + """Vertex Imagen ``PredictResponse`` predictions: ``bytesBase64Encoded`` with a declared image type.""" + if not self.usable or not isinstance(endpoint, str) or not _IMAGEN_ENDPOINT.search(endpoint): + return self + try: + for index, prediction in enumerate(_items(predictions)): + mime = mc.get(prediction, "mimeType") + if mime in IMAGE_TYPES: + self._typed("imagen/output/%d" % index, mc.get(prediction, "bytesBase64Encoded"), mime, + direction="output", purpose="model_output", role="assistant", part_index=index) + except Exception: + pass + return self + + def bedrock_invoke_request(self, model_id: Any, body: Any) -> "ProviderMedia": + """InvokeModel request bodies of named families: Anthropic Messages, Nova, Titan/Nova Canvas images.""" + if not self.usable or not isinstance(model_id, str) or not isinstance(body, Mapping): + return self + try: + if _ANTHROPIC_MODELS.match(model_id) and "anthropic_version" in body: + self.anthropic_messages(body.get("messages"), body.get("tools")) + elif _CONVERSE_MODELS.match(model_id): + self.bedrock_converse(model_id, body.get("messages")) + elif _IMAGE_GENERATION_MODELS.match(model_id): + index = 0 + variation = body.get("imageVariationParams") + for image in _items(mc.get(variation, "images"))[:MAX_PARTS]: + self._untyped_image("titan/input/%d" % index, image, direction="input", purpose="model_input", + role="user", part_index=index) + index += 1 + for params, field in _IMAGE_PARAMS: + image = mc.get(body.get(params), field) + if image is not None: + self._untyped_image("titan/input/%s" % params, image, direction="input", + purpose="model_input", role="user", part_index=index) + index += 1 + except Exception: + pass + return self + + def bedrock_invoke_response(self, model_id: Any, body: Any) -> "ProviderMedia": + """InvokeModel response bodies the instrumentor already buffered: Titan/Nova Canvas ``images``.""" + if not self.usable or not isinstance(model_id, str) or not isinstance(body, Mapping): + return self + try: + if _IMAGE_GENERATION_MODELS.match(model_id): + for index, image in enumerate(_items(body.get("images"))[:MAX_PARTS]): + self._untyped_image("titan/output/%d" % index, image, direction="output", + purpose="model_output", role="assistant", part_index=index) + except Exception: + pass + return self + + +def _items(value: Any) -> list: + """A bounded list of a list, tuple or other sequence (proto repeated fields); never a str or bytes.""" + if isinstance(value, (list, tuple)): + return list(value[:MAX_PARTS]) + if isinstance(value, (str, bytes, bytearray, memoryview, Mapping)) or not isinstance(value, Sequence): + return [] + try: + # Proto repeated fields do not all support slicing; bound the walk instead. + return list(islice(value, MAX_PARTS)) + except Exception: + return [] + + +def _image_only(source: Optional[_Source]) -> Optional[_Source]: + return source if source is not None and source.mime in IMAGE_TYPES else None + + +def _inline(part: Any) -> Any: + inline = mc.get(part, "inline_data") + return inline if inline is not None else mc.get(part, "inlineData") + + +def _lower(value: Any) -> str: + return value.lower().split(";", 1)[0].strip() if isinstance(value, str) else "" + + +def _role(value: Any) -> str: + return value if value in ("user", "assistant", "tool", "system") else "unknown" + + +def _purpose(direction: str, mime: str) -> str: + if direction == "output": + return "model_output" + return "document" if mime in DOCUMENT_TYPES else "model_input" + + +def _computer_calls(messages, tools, use_type: str, id_field: str, nested: Optional[str] = None) -> frozenset: + """Tool-call IDs of assistant calls to a declared typed computer tool (``computer_YYYYMMDD``).""" + names = set() + for tool in _items(tools): + kind, name = mc.get(tool, "type"), mc.get(tool, "name") + if isinstance(kind, str) and _COMPUTER_TOOL.fullmatch(kind) and isinstance(name, str): + names.add(name) + if not names: + return frozenset() + calls = set() + for message in messages: + if mc.get(message, "role") != "assistant": + continue + for block in _items(mc.get(message, "content")): + use = mc.get(block, nested) if nested else (block if mc.get(block, "type") == use_type else None) + call_id = mc.get(use, id_field) + if use is not None and isinstance(call_id, str) and mc.get(use, "name") in names: + calls.add(call_id) + return frozenset(calls) + + +# ------------------------------------------------------------------ capability matrix +def _descriptor(operations: Tuple[str, ...], sources: Tuple[str, ...], unsupported: Tuple[str, ...], + note: Optional[str] = None) -> Mapping[str, Any]: + value = {"schema_version": 1, "operations": operations, "sources": sources, "unsupported": unsupported, + "max_inline_bytes": MAX_DECODED_BYTES} + if note: + value["note"] = note + return MappingProxyType(value) + + +_COMPAT_UNSUPPORTED = ("remote_url", "provider_file", "streamed_output", "output_media") + +#: Typed per-package capability descriptors; this IS the Python support matrix. Anything not listed +#: (remote URLs, provider file IDs, gs://, S3, PIL images, async/long-running operations, unknown model +#: bodies, native speech/OCR/image routes) is unsupported and never captured. A package with no +#: operations has no qualified media surface: it keeps tracing and captures nothing. +PROVIDER_MEDIA_CAPABILITIES: Mapping[str, Mapping[str, Any]] = MappingProxyType({ + "traceai_google_genai": _descriptor( + ("models.generate_content.input", "models.generate_content.output", "models.generate_content_stream.input"), + ("sdk_bytes", "inline_base64"), + ("file_data_uri", "gs_uri", "provider_file", "pil_image", "streamed_output", "models.generate_images", + "models.generate_videos", "live", "audio/L16")), + # Proven: the sync gapic PredictionServiceClient (REST and gRPC share the wrapped method). The + # high-level vertexai GenerativeModel / ImageGenerationModel routes run through the same client + # but have no fixture yet, so they are not claimed. + "traceai_vertexai": _descriptor( + ("PredictionServiceClient.generate_content.input", "PredictionServiceClient.generate_content.output", + "PredictionServiceClient.predict.imagen.output"), + ("sdk_bytes", "inline_base64"), + ("file_data_uri", "gs_uri", "streamed_output", "PredictionServiceAsyncClient", + "PredictionServiceClient.stream_generate_content", "PredictionServiceClient.predict.other_endpoints", + "GenerativeModel.generate_content", "ImageGenerationModel.generate_images", + "ImageGenerationModel.edit_image", "audio/L16"), + "GenerativeModel and ImageGenerationModel are not qualified yet; only the PredictionServiceClient " + "paths above are proven."), + "traceai_bedrock": _descriptor( + ("converse.input", "invoke_model.anthropic.input", "invoke_model.nova.input", + "invoke_model.image_generation.input", "invoke_model.image_generation.output"), + ("sdk_bytes", "inline_base64"), + ("s3_location", "converse_stream", "invoke_model_with_response_stream", "unknown_model_body", + "video.anthropic", "converse.output")), + "traceai_litellm": _descriptor( + ("completion.input", "acompletion.input", "image_generation.output", "aimage_generation.output"), + ("data_uri", "inline_base64"), + ("remote_url", "provider_file", "file_id", "streamed_output", "image_generation.url", "image_edit", + "responses", "speech", "transcription"), + "Chat input covers image_url, input_audio and PDF file/input_file data URIs. LiteLLM claims the call, " + "so a nested OpenAI instrumentor uploads nothing more; application/octet-stream files stay uncaptured."), + "traceai_groq": _descriptor(("chat.completions.create.input",), ("data_uri",), _COMPAT_UNSUPPORTED), + # The binding is wired, but traceai_together installs no chat wrapper today (its install check looks + # for `chat` on the Together class, where it is an instance attribute), so nothing is claimed. + "traceai_together": _descriptor((), (), _COMPAT_UNSUPPORTED + ("chat.completions.create", "images.generate"), + "traceai_together does not trace chat.completions.create yet; no media claim."), + "traceai_fireworks": _descriptor(("chat.completions.create.input",), ("data_uri",), _COMPAT_UNSUPPORTED), + "traceai_xai": _descriptor(("chat.completions.create.input",), ("data_uri",), _COMPAT_UNSUPPORTED), + "traceai_mistralai": _descriptor(("chat.complete.input", "chat.stream.input"), ("data_uri",), + _COMPAT_UNSUPPORTED + ("ocr", "document_url", "input_audio")), + "traceai_openrouter": _descriptor(("chat.send.input",), ("data_uri", "inline_base64"), _COMPAT_UNSUPPORTED), + "traceai_ollama": _descriptor(("chat.input", "generate.input"), ("sdk_bytes", "inline_base64"), + ("image_path", "streamed_output", "output_media")), + "traceai_portkey": _descriptor(("chat.completions.create.input",), ("data_uri", "inline_base64"), + _COMPAT_UNSUPPORTED), + # Legacy VLLMInstrumentor (OpenAI-dialect chat: image_url and input_audio). It already defers to + # traceai_openai while that is enabled, which then owns the call's media. + "traceai_vllm": _descriptor(("chat.completions.create.input",), ("data_uri", "inline_base64"), + _COMPAT_UNSUPPORTED + ("video_url",)), + # MiniMax chat takes image_url (M3 / M3.1-Flash); no audio input. Native speech/image/video is separate. + "traceai_minimax": _descriptor(("chat.completions.create.input",), ("data_uri",), + _COMPAT_UNSUPPORTED + ("video_url", "input_audio", "native_media")), + "traceai_deepseek": _descriptor((), (), ("chat.completions.create",), + "DeepSeek chat has no qualified media input or output; tracing only."), + "traceai_cerebras": _descriptor((), (), ("chat.completions.create",), + "Cerebras chat has no qualified media input or output; tracing only."), +}) + + +def capabilities(package: str) -> Mapping[str, Any]: + """The descriptor of one package; an empty-operations descriptor for unknown names.""" + return PROVIDER_MEDIA_CAPABILITIES.get(package) or _descriptor((), (), (), "not qualified") diff --git a/python/frameworks/bedrock/traceai_bedrock/__init__.py b/python/frameworks/bedrock/traceai_bedrock/__init__.py index ee49cf294..16a20e276 100644 --- a/python/frameworks/bedrock/traceai_bedrock/__init__.py +++ b/python/frameworks/bedrock/traceai_bedrock/__init__.py @@ -128,8 +128,21 @@ def reset(self) -> None: self._buffer.seek(0) +def _bedrock_media(attachments: Any, config: Any, span: Any) -> Any: + """A typed media capture for this span, or None (no manager, owned elsewhere, unsampled).""" + if attachments is None: + return None + try: + from fi_instrumentation.attachments.provider_media import ProviderMedia + + media = ProviderMedia(attachments, span, "bedrock", trace_config=config) + return media if media.usable else None + except Exception: + return None + + def _client_creation_wrapper( - tracer: Tracer, module_version: str + tracer: Tracer, module_version: str, attachments: Any = None, config: Any = None ) -> Callable[[ClientCreator], ClientCreator]: def _client_wrapper( wrapped: ClientCreator, @@ -167,14 +180,14 @@ def _client_wrapper( client = cast(InstrumentedClient, client) client._unwrapped_invoke_model = client.invoke_model - client.invoke_model = _model_invocation_wrapper(tracer)(client) + client.invoke_model = _model_invocation_wrapper(tracer, attachments, config)(client) client.invoke_model_with_response_stream = _InvokeModelWithResponseStream(tracer)( client.invoke_model_with_response_stream ) if module_version >= _MINIMUM_CONVERSE_BOTOCORE_VERSION: client._unwrapped_converse = client.converse - client.converse = _model_converse_wrapper(tracer)(client) + client.converse = _model_converse_wrapper(tracer, attachments, config)(client) # ConverseStream is an independent operation, not a call to converse. # Feature-detect it so older botocore service models remain supported. if hasattr(client, "converse_stream"): @@ -185,7 +198,9 @@ def _client_wrapper( return _client_wrapper # type: ignore -def _model_invocation_wrapper(tracer: Tracer) -> Callable[[InstrumentedClient], Callable[..., Any]]: +def _model_invocation_wrapper( + tracer: Tracer, attachments: Any = None, config: Any = None +) -> Callable[[InstrumentedClient], Callable[..., Any]]: def _invocation_wrapper(wrapped_client: InstrumentedClient) -> Callable[..., Any]: """Instruments a bedrock client's `invoke_model` or `converse` method.""" @@ -211,6 +226,11 @@ def instrumented_response(*args: Any, **kwargs: Any) -> Dict[str, Any]: model_id ) + # Typed media of named model families only (Anthropic, Nova, Titan/Nova Canvas). + media = _bedrock_media(attachments, config, span) + if media is not None: + media.bedrock_invoke_request(model_id, request_body) + # Set input attributes only when the request can be captured. if isinstance(request_body, dict): if is_claude_message_api: @@ -232,6 +252,9 @@ def instrumented_response(*args: Any, **kwargs: Any) -> Dict[str, Any]: ) response_body = json.loads(response.get("body").read()) response["body"].reset() + if media is not None: + # Generated images from the body this wrapper already buffered; no extra read. + media.bedrock_invoke_response(model_id, response_body) # Set response attributes based on model type if is_claude_message_api: @@ -249,7 +272,9 @@ def instrumented_response(*args: Any, **kwargs: Any) -> Dict[str, Any]: return _invocation_wrapper -def _model_converse_wrapper(tracer: Tracer) -> Callable[[InstrumentedClient], Callable[..., Any]]: +def _model_converse_wrapper( + tracer: Tracer, attachments: Any = None, config: Any = None +) -> Callable[[InstrumentedClient], Callable[..., Any]]: def _converse_wrapper(wrapped_client: InstrumentedClient) -> Callable[..., Any]: """Instruments a bedrock client's `converse` method.""" @@ -275,6 +300,14 @@ def instrumented_response(*args: Any, **kwargs: Any) -> Dict[str, Any]: span, SpanAttributes.GEN_AI_REQUEST_PARAMETERS, invocation_parameters ) + media = _bedrock_media(attachments, config, span) + if media is not None: + media.bedrock_converse( + kwargs.get("modelId"), + kwargs.get("messages"), + kwargs.get("additionalModelRequestFields"), + ) + aggregated_messages = [] if system_prompts := kwargs.get("system"): aggregated_messages.append( @@ -357,6 +390,20 @@ def instrumented_response(*args: Any, **kwargs: Any) -> Dict[str, Any]: return _converse_wrapper +def _attachment_manager(kwargs: Any) -> Any: + """The shared manager passed as ``attachments`` (or auto's ``attachment_manager``), if it can admit. + + A missing, disabled or pre-attachments core leaves tracing exactly as before. + """ + if kwargs.get("attachments") is None and kwargs.get("attachment_manager") is None: + return None + try: + from fi_instrumentation.attachments.provider_media import manager_from_kwargs + except ImportError: + return None + return manager_from_kwargs(kwargs) + + class BedrockInstrumentor(BaseInstrumentor): # type: ignore __slots__ = ( "_tracer", @@ -386,7 +433,10 @@ def _instrument(self, **kwargs: Any) -> None: module=_MODULE, name="ClientCreator.create_client", wrapper=_client_creation_wrapper( - tracer=self._tracer, module_version=botocore.__version__ + tracer=self._tracer, + module_version=botocore.__version__, + attachments=_attachment_manager(kwargs), + config=config, ), ) @@ -467,3 +517,12 @@ def _get_attributes_from_image( def is_iterable_of(lst: Iterable[object], tp: T) -> bool: return isinstance(lst, Iterable) and all(isinstance(x, tp) for x in lst) + + +def __getattr__(name: str) -> Any: + # Lazy: the descriptor needs a fi-instrumentation-otel release with attachments. + if name == "bedrock_attachment_capabilities": + from fi_instrumentation.attachments.provider_media import capabilities + + return capabilities("traceai_bedrock") + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/python/frameworks/cerebras/traceai_cerebras/__init__.py b/python/frameworks/cerebras/traceai_cerebras/__init__.py index 1062d0797..c7c7cce1c 100644 --- a/python/frameworks/cerebras/traceai_cerebras/__init__.py +++ b/python/frameworks/cerebras/traceai_cerebras/__init__.py @@ -117,3 +117,12 @@ def _uninstrument(self, **kwargs: Any) -> None: cerebras_module.AsyncCompletionsResource.create = self._original_async_completions_create except ImportError: pass + + +def __getattr__(name: str) -> Any: + # Lazy: the descriptor needs a fi-instrumentation-otel release with attachments. + if name == "cerebras_attachment_capabilities": + from fi_instrumentation.attachments.provider_media import capabilities + + return capabilities("traceai_cerebras") + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/python/frameworks/deepseek/traceai_deepseek/__init__.py b/python/frameworks/deepseek/traceai_deepseek/__init__.py index 569460d6d..36829275b 100644 --- a/python/frameworks/deepseek/traceai_deepseek/__init__.py +++ b/python/frameworks/deepseek/traceai_deepseek/__init__.py @@ -131,3 +131,12 @@ def _uninstrument(self, **kwargs: Any) -> None: completions.AsyncCompletions.create = self._original_async_chat_create except Exception as e: logger.warning(f"Failed to uninstrument: {e}") + + +def __getattr__(name: str) -> Any: + # Lazy: the descriptor needs a fi-instrumentation-otel release with attachments. + if name == "deepseek_attachment_capabilities": + from fi_instrumentation.attachments.provider_media import capabilities + + return capabilities("traceai_deepseek") + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/python/frameworks/fireworks/traceai_fireworks/__init__.py b/python/frameworks/fireworks/traceai_fireworks/__init__.py index 00259fba4..11f283eaf 100644 --- a/python/frameworks/fireworks/traceai_fireworks/__init__.py +++ b/python/frameworks/fireworks/traceai_fireworks/__init__.py @@ -59,6 +59,20 @@ ] +def _attachment_manager(kwargs: Any) -> Any: + """The shared manager passed as ``attachments`` (or auto's ``attachment_manager``), if it can admit. + + A missing, disabled or pre-attachments core leaves tracing exactly as before. + """ + if kwargs.get("attachments") is None and kwargs.get("attachment_manager") is None: + return None + try: + from fi_instrumentation.attachments.provider_media import manager_from_kwargs + except ImportError: + return None + return manager_from_kwargs(kwargs) + + class FireworksInstrumentor(BaseInstrumentor): """An instrumentor for Fireworks AI (OpenAI-compatible API).""" @@ -107,7 +121,7 @@ def _instrument(self, **kwargs: Any) -> None: wrap_function_wrapper( module="openai.resources.chat", name="Completions.create", - wrapper=_CompletionsWrapper(tracer=self._tracer, config=config), + wrapper=_CompletionsWrapper(tracer=self._tracer, config=config, attachments=_attachment_manager(kwargs)), ) # Wrap asynchronous completions @@ -115,7 +129,9 @@ def _instrument(self, **kwargs: Any) -> None: wrap_function_wrapper( module="openai.resources.chat", name="AsyncCompletions.create", - wrapper=_AsyncCompletionsWrapper(tracer=self._tracer, config=config), + wrapper=_AsyncCompletionsWrapper( + tracer=self._tracer, config=config, attachments=_attachment_manager(kwargs) + ), ) def _uninstrument(self, **kwargs: Any) -> None: @@ -127,3 +143,12 @@ def _uninstrument(self, **kwargs: Any) -> None: openai_module.AsyncCompletions.create = self._original_async_completions_create except ImportError: pass + + +def __getattr__(name: str) -> Any: + # Lazy: the descriptor needs a fi-instrumentation-otel release with attachments. + if name == "fireworks_attachment_capabilities": + from fi_instrumentation.attachments.provider_media import capabilities + + return capabilities("traceai_fireworks") + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/python/frameworks/fireworks/traceai_fireworks/_wrappers.py b/python/frameworks/fireworks/traceai_fireworks/_wrappers.py index 96ebbf8b1..f649c2c51 100644 --- a/python/frameworks/fireworks/traceai_fireworks/_wrappers.py +++ b/python/frameworks/fireworks/traceai_fireworks/_wrappers.py @@ -49,13 +49,30 @@ class _WithTracer(ABC): """Base class for wrappers that need a tracer.""" + _media_dialect = "image" + def __init__( self, tracer: trace_api.Tracer, *args: Any, - config: Optional[TraceConfig] = None, **kwargs: Any, + config: Optional[TraceConfig] = None, attachments: Any = None, **kwargs: Any, ) -> None: super().__init__(*args, **kwargs) self._tracer = tracer self._config = config if config is not None else TraceConfig() + # Provider-owned shared attachment manager, or None (no capture, no imports). + self._attachments = attachments + + def _capture_chat_media(self, span: Any, messages: Any) -> None: + """Typed chat media (data-URI images) through the shared attachment manager; never raises.""" + if self._attachments is None: + return + try: + from fi_instrumentation.attachments.provider_media import ProviderMedia + + ProviderMedia( + self._attachments, span, "fireworks", trace_config=self._config + ).compatible_messages(messages, self._media_dialect) + except Exception: + logger.debug("Attachment capture unavailable", exc_info=True) @contextmanager def _start_span( @@ -408,6 +425,7 @@ def __call__( input_attrs = _get_input_attributes(model, messages, config=self._config, **extra_kwargs) with self._start_span("Fireworks Chat Completions", input_attrs) as span: + self._capture_chat_media(span, messages) try: response = wrapped(*args, **kwargs) @@ -472,6 +490,7 @@ async def __call__( input_attrs = _get_input_attributes(model, messages, config=self._config, **extra_kwargs) with self._start_span("Fireworks Chat Completions", input_attrs) as span: + self._capture_chat_media(span, messages) try: response = await wrapped(*args, **kwargs) diff --git a/python/frameworks/google-genai/traceai_google_genai/__init__.py b/python/frameworks/google-genai/traceai_google_genai/__init__.py index 3b9305009..cbc62c6ec 100644 --- a/python/frameworks/google-genai/traceai_google_genai/__init__.py +++ b/python/frameworks/google-genai/traceai_google_genai/__init__.py @@ -13,6 +13,20 @@ logger.addHandler(logging.NullHandler()) +def _attachment_manager(kwargs: Any) -> Any: + """The shared manager passed as ``attachments`` (or auto's ``attachment_manager``), if it can admit. + + A missing, disabled or pre-attachments core leaves tracing exactly as before. + """ + if kwargs.get("attachments") is None and kwargs.get("attachment_manager") is None: + return None + try: + from fi_instrumentation.attachments.provider_media import manager_from_kwargs + except ImportError: + return None + return manager_from_kwargs(kwargs) + + class GoogleGenAIInstrumentor(BaseInstrumentor): # type: ignore """ An instrumentor for `google-genai` @@ -55,32 +69,34 @@ def _instrument(self, **kwargs: Any) -> None: _SyncGenerateContentStream, ) + media = dict(attachments=_attachment_manager(kwargs), config=config) + self._original_generate_content = Models.generate_content wrap_function_wrapper( module="google.genai.models", name="Models.generate_content", - wrapper=_SyncGenerateContent(tracer=self._tracer), + wrapper=_SyncGenerateContent(tracer=self._tracer, **media), ) self._original_async_generate_content = AsyncModels.generate_content wrap_function_wrapper( module="google.genai.models", name="AsyncModels.generate_content", - wrapper=_AsyncGenerateContentWrapper(tracer=self._tracer), + wrapper=_AsyncGenerateContentWrapper(tracer=self._tracer, **media), ) self._original_generate_content_stream = Models.generate_content_stream wrap_function_wrapper( module="google.genai.models", name="Models.generate_content_stream", - wrapper=_SyncGenerateContentStream(tracer=self._tracer), + wrapper=_SyncGenerateContentStream(tracer=self._tracer, **media), ) self._original_async_generate_content_stream = AsyncModels.generate_content_stream wrap_function_wrapper( module="google.genai.models", name="AsyncModels.generate_content_stream", - wrapper=_AsyncGenerateContentStream(tracer=self._tracer), + wrapper=_AsyncGenerateContentStream(tracer=self._tracer, **media), ) def _uninstrument(self, **kwargs: Any) -> None: @@ -99,3 +115,12 @@ def _uninstrument(self, **kwargs: Any) -> None: setattr( AsyncModels, "generate_content_stream", self._original_async_generate_content_stream ) + + +def __getattr__(name: str) -> Any: + # Lazy: the descriptor needs a fi-instrumentation-otel release with attachments. + if name == "google_genai_attachment_capabilities": + from fi_instrumentation.attachments.provider_media import capabilities + + return capabilities("traceai_google_genai") + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/python/frameworks/google-genai/traceai_google_genai/_wrappers.py b/python/frameworks/google-genai/traceai_google_genai/_wrappers.py index 39b0ecf18..e0e3cb8d7 100644 --- a/python/frameworks/google-genai/traceai_google_genai/_wrappers.py +++ b/python/frameworks/google-genai/traceai_google_genai/_wrappers.py @@ -54,9 +54,34 @@ class _WithTracer(ABC): Base class for wrappers that need a tracer. """ - def __init__(self, tracer: trace_api.Tracer, *args: Any, **kwargs: Any) -> None: + def __init__( + self, + tracer: trace_api.Tracer, + *args: Any, + attachments: Any = None, + config: Any = None, + **kwargs: Any, + ) -> None: super().__init__(*args, **kwargs) self._tracer = tracer + # Provider-owned shared attachment manager, or None (no capture, no imports). + self._attachments = attachments + self._config = config + + def _capture_input(self, span: _WithSpan, wrapped: Callable[..., Any], args: Any, kwargs: Any) -> Any: + """Typed inline_data Parts of the request; returns the capture for the response, or None.""" + if self._attachments is None: + return None + try: + from fi_instrumentation.attachments.provider_media import ProviderMedia + + bound = signature(wrapped).bind(*args, **kwargs).arguments + return ProviderMedia( + self._attachments, span._span, "google_genai", trace_config=self._config + ).gemini_contents(bound.get("contents")) + except Exception: + logger.debug("Attachment capture unavailable", exc_info=True) + return None @contextmanager def _start_as_current_span( @@ -148,6 +173,7 @@ def __call__( request_parameters ), ) as span: + media = self._capture_input(span, wrapped, args, kwargs) try: response = wrapped(*args, **kwargs) except Exception as exception: @@ -158,6 +184,9 @@ def __call__( ) span.finish_tracing(status=status) raise + if media is not None: + # Completed response only (generated images, audio, files); streams stay input-only. + media.gemini_response(response) try: _finish_tracing( status=trace_api.Status(status_code=trace_api.StatusCode.OK), @@ -209,6 +238,7 @@ def __call__( request_parameters ), ) as span: + media = self._capture_input(span, wrapped, args, kwargs) try: response = wrapped(*args, **kwargs) except Exception as exception: @@ -268,6 +298,7 @@ async def __call__( request_parameters ), ) as span: + media = self._capture_input(span, wrapped, args, kwargs) try: response = await wrapped(*args, **kwargs) except Exception as exception: @@ -278,6 +309,9 @@ async def __call__( ) span.finish_tracing(status=status) raise + if media is not None: + # Completed response only (generated images, audio, files); streams stay input-only. + media.gemini_response(response) try: _finish_tracing( status=trace_api.Status(status_code=trace_api.StatusCode.OK), @@ -329,6 +363,7 @@ async def __call__( request_parameters ), ) as span: + media = self._capture_input(span, wrapped, args, kwargs) try: response = await wrapped(*args, **kwargs) except Exception as exception: diff --git a/python/frameworks/groq/traceai_groq/__init__.py b/python/frameworks/groq/traceai_groq/__init__.py index 6544393ca..6ea706c57 100644 --- a/python/frameworks/groq/traceai_groq/__init__.py +++ b/python/frameworks/groq/traceai_groq/__init__.py @@ -66,14 +66,18 @@ def _instrument(self, **kwargs: Any) -> None: wrap_function_wrapper( module="groq.resources.chat.completions", name="Completions.create", - wrapper=_CompletionsWrapper(tracer=self._tracer), + wrapper=_CompletionsWrapper( + tracer=self._tracer, attachments=_attachment_manager(kwargs), config=config + ), ) self._original_async_completions_create = AsyncCompletions.create wrap_function_wrapper( module="groq.resources.chat.completions", name="AsyncCompletions.create", - wrapper=_AsyncCompletionsWrapper(tracer=self._tracer), + wrapper=_AsyncCompletionsWrapper( + tracer=self._tracer, attachments=_attachment_manager(kwargs), config=config + ), ) # Audio spans exist only with an enabled attachment manager: plain tracing is unchanged. diff --git a/python/frameworks/groq/traceai_groq/_audio.py b/python/frameworks/groq/traceai_groq/_audio.py index 00c791ecc..e8407742d 100644 --- a/python/frameworks/groq/traceai_groq/_audio.py +++ b/python/frameworks/groq/traceai_groq/_audio.py @@ -32,10 +32,12 @@ groq_attachment_capabilities = { "schema_version": 1, + # Chat image input (data URIs) is the compatible-message capture in fi_instrumentation's provider_media. "operations": ("audio.speech.create.output", "audio.transcriptions.create.input", - "audio.translations.create.input"), - "sources": ("sdk_buffered_bytes", "inline_bytes"), - "unsupported": ("streamed_speech", "file_path", "open_file_object", "chat_media"), + "audio.translations.create.input", "chat.completions.create.input"), + "sources": ("sdk_buffered_bytes", "inline_bytes", "data_uri"), + "unsupported": ("streamed_speech", "file_path", "open_file_object", "chat_remote_url", "chat_input_audio", + "chat_output_media"), } diff --git a/python/frameworks/groq/traceai_groq/_wrappers.py b/python/frameworks/groq/traceai_groq/_wrappers.py index a239eee46..0702398ac 100644 --- a/python/frameworks/groq/traceai_groq/_wrappers.py +++ b/python/frameworks/groq/traceai_groq/_wrappers.py @@ -51,9 +51,34 @@ class _WithTracer(ABC): Base class for wrappers that need a tracer. """ - def __init__(self, tracer: trace_api.Tracer, *args: Any, **kwargs: Any) -> None: + _media_dialect = "groq" + + def __init__( + self, + tracer: trace_api.Tracer, + *args: Any, + attachments: Any = None, + config: Any = None, + **kwargs: Any, + ) -> None: super().__init__(*args, **kwargs) self._tracer = tracer + # Provider-owned shared attachment manager, or None (no capture, no imports). + self._attachments = attachments + self._config = config + + def _capture_chat_media(self, span: Any, messages: Any) -> None: + """Typed chat media (data-URI images) through the shared attachment manager; never raises.""" + if self._attachments is None: + return + try: + from fi_instrumentation.attachments.provider_media import ProviderMedia + + ProviderMedia( + self._attachments, span, "groq", trace_config=self._config + ).compatible_messages(messages, self._media_dialect) + except Exception: + logger.debug("Attachment capture unavailable", exc_info=True) @contextmanager def _start_as_current_span( @@ -297,6 +322,7 @@ def __call__( request_parameters ), ) as span: + self._capture_chat_media(span._span, kwargs.get("messages")) try: response = wrapped(*args, **kwargs) streaming = request_parameters.get("stream", False) @@ -370,6 +396,7 @@ async def __call__( request_parameters ), ) as span: + self._capture_chat_media(span._span, kwargs.get("messages")) try: response = await wrapped(*args, **kwargs) streaming = request_parameters.get("stream", False) diff --git a/python/frameworks/litellm/traceai_litellm/__init__.py b/python/frameworks/litellm/traceai_litellm/__init__.py index 483cedf88..5a4704592 100644 --- a/python/frameworks/litellm/traceai_litellm/__init__.py +++ b/python/frameworks/litellm/traceai_litellm/__init__.py @@ -1,5 +1,6 @@ import json from enum import Enum +from contextlib import contextmanager from functools import wraps from inspect import Parameter, signature from typing import ( @@ -11,6 +12,7 @@ Iterable, Iterator, Mapping, + Optional, Tuple, TypeVar, ) @@ -335,6 +337,68 @@ def _finalize_span(span: trace_api.Span, result: Any) -> None: break +def _media_capture(instrumentor: Any, span: trace_api.Span, arguments: Optional[Mapping[str, Any]] = None) -> Any: + """Capture typed chat media of this call; returns the capture (also for output), or None.""" + attachments = getattr(instrumentor, "_attachments", None) + if attachments is None: + return None + try: + from fi_instrumentation.attachments.provider_media import ProviderMedia + + media = ProviderMedia(attachments, span, "litellm", trace_config=getattr(instrumentor, "_media_config", None)) + if not media.usable: + return None + if arguments is not None: + # PDF file parts too: LiteLLM claims the call, so the nested OpenAI instrumentor skips them. + media.compatible_messages(arguments.get("messages"), "openai", documents=True) + return media + except Exception: + logger.debug("Attachment capture unavailable", exc_info=True) + return None + + +@contextmanager +def _media_owned(media: Any) -> Iterator[None]: + """Run LiteLLM's downstream SDK call as the owner of this call's media. + + A nested provider instrumentor (for example OpenAI under LiteLLM) then + uploads nothing for the same call. Only set while this adapter captures. + """ + if media is None: + yield + return + from fi_instrumentation.attachments.provider_media import claim + + token = context_api.attach(claim()) + try: + yield + finally: + context_api.detach(token) + + +def _media_generated(media: Any, result: Any) -> None: + if media is None or not isinstance(result, ImageResponse): + return + try: + media.generated_images(result.data) + except Exception: + logger.debug("Attachment capture unavailable", exc_info=True) + + +def _attachment_manager(kwargs: Any) -> Any: + """The shared manager passed as ``attachments`` (or auto's ``attachment_manager``), if it can admit. + + A missing, disabled or pre-attachments core leaves tracing exactly as before. + """ + if kwargs.get("attachments") is None and kwargs.get("attachment_manager") is None: + return None + try: + from fi_instrumentation.attachments.provider_media import manager_from_kwargs + except ImportError: + return None + return manager_from_kwargs(kwargs) + + class LiteLLMInstrumentor(BaseInstrumentor): # type: ignore original_litellm_funcs: Dict[str, Callable[..., Any]] = ( {} @@ -354,6 +418,9 @@ def _instrument(self, **kwargs: Any) -> None: trace_api.get_tracer(__name__, __version__, tracer_provider), config=config, ) + # Provider-owned shared attachment manager, or None (no capture, no imports). + self._attachments = _attachment_manager(kwargs) + self._media_config = config functions_to_instrument = { "completion": self._completion_wrapper, @@ -445,7 +512,9 @@ def _completion_wrapper(self, *args: Any, **kwargs: Any) -> Any: ) as span: try: _instrument_func_type_completion(span, arguments) - result = self.original_litellm_funcs["completion"](*args, **kwargs) + media = _media_capture(self, span, arguments) + with _media_owned(media): + result = self.original_litellm_funcs["completion"](*args, **kwargs) _finalize_span(span, result) span.set_status(trace_api.StatusCode.OK) return result @@ -459,9 +528,11 @@ def _completion_wrapper(self, *args: Any, **kwargs: Any) -> Any: span = span_cm.__enter__() try: _instrument_func_type_completion(span, arguments) - original_iterator = self.original_litellm_funcs["completion"]( - *args, **kwargs - ) + media = _media_capture(self, span, arguments) + with _media_owned(media): + original_iterator = self.original_litellm_funcs["completion"]( + *args, **kwargs + ) return StreamingIteratorWrapper(original_iterator, span, span_cm) except BaseException as e: span_cm.__exit__(type(e), e, e.__traceback__) @@ -479,9 +550,11 @@ async def _acompletion_wrapper(self, *args: Any, **kwargs: Any) -> Any: ) as span: try: _instrument_func_type_completion(span, arguments) - result = await self.original_litellm_funcs["acompletion"]( - *args, **kwargs - ) + media = _media_capture(self, span, arguments) + with _media_owned(media): + result = await self.original_litellm_funcs["acompletion"]( + *args, **kwargs + ) _finalize_span(span, result) span.set_status(trace_api.StatusCode.OK) return result @@ -495,9 +568,11 @@ async def _acompletion_wrapper(self, *args: Any, **kwargs: Any) -> Any: span = span_cm.__enter__() try: _instrument_func_type_completion(span, arguments) - original_async_iterator = await self.original_litellm_funcs[ - "acompletion" - ](*args, **kwargs) + media = _media_capture(self, span, arguments) + with _media_owned(media): + original_async_iterator = await self.original_litellm_funcs[ + "acompletion" + ](*args, **kwargs) return AsyncStreamingIteratorWrapper( original_async_iterator, span, span_cm ) @@ -521,9 +596,11 @@ def _completion_with_retries_wrapper( ) as span: try: _instrument_func_type_completion(span, arguments) - result = self.original_litellm_funcs["completion_with_retries"]( - *args, **kwargs - ) + media = _media_capture(self, span, arguments) + with _media_owned(media): + result = self.original_litellm_funcs["completion_with_retries"]( + *args, **kwargs + ) _finalize_span(span, result) span.set_status(trace_api.StatusCode.OK) return result # type:ignore @@ -546,9 +623,11 @@ async def _acompletion_with_retries_wrapper( ) as span: try: _instrument_func_type_completion(span, arguments) - result = await self.original_litellm_funcs[ - "acompletion_with_retries" - ](*args, **kwargs) + media = _media_capture(self, span, arguments) + with _media_owned(media): + result = await self.original_litellm_funcs[ + "acompletion_with_retries" + ](*args, **kwargs) _finalize_span(span, result) span.set_status(trace_api.StatusCode.OK) return result # type:ignore @@ -610,9 +689,12 @@ def _image_generation_wrapper(self, *args: Any, **kwargs: Any) -> ImageResponse: ) as span: try: _instrument_func_type_image_generation(span, arguments, args) - result = self.original_litellm_funcs["image_generation"]( - *args, **kwargs - ) + media = _media_capture(self, span) + with _media_owned(media): + result = self.original_litellm_funcs["image_generation"]( + *args, **kwargs + ) + _media_generated(media, result) _finalize_span(span, result) span.set_status(trace_api.StatusCode.OK) return result # type:ignore @@ -634,9 +716,12 @@ async def _aimage_generation_wrapper( ) as span: try: _instrument_func_type_image_generation(span, arguments, args) - result = await self.original_litellm_funcs["aimage_generation"]( - *args, **kwargs - ) + media = _media_capture(self, span) + with _media_owned(media): + result = await self.original_litellm_funcs["aimage_generation"]( + *args, **kwargs + ) + _media_generated(media, result) _finalize_span(span, result) span.set_status(trace_api.StatusCode.OK) return result # type:ignore @@ -856,3 +941,12 @@ def _process_messages(messages): "eval_input": None, "query": None, } + + +def __getattr__(name: str) -> Any: + # Lazy: the descriptor needs a fi-instrumentation-otel release with attachments. + if name == "litellm_attachment_capabilities": + from fi_instrumentation.attachments.provider_media import capabilities + + return capabilities("traceai_litellm") + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/python/frameworks/minimax/traceai_minimax/__init__.py b/python/frameworks/minimax/traceai_minimax/__init__.py index f3b1e6fe2..8517357b6 100644 --- a/python/frameworks/minimax/traceai_minimax/__init__.py +++ b/python/frameworks/minimax/traceai_minimax/__init__.py @@ -21,6 +21,20 @@ _instruments = ("openai >= 1.0.0",) +def _attachment_manager(kwargs: Any) -> Any: + """The shared manager passed as ``attachments`` (or auto's ``attachment_manager``), if it can admit. + + A missing, disabled or pre-attachments core leaves tracing exactly as before. + """ + if kwargs.get("attachments") is None and kwargs.get("attachment_manager") is None: + return None + try: + from fi_instrumentation.attachments.provider_media import manager_from_kwargs + except ImportError: + return None + return manager_from_kwargs(kwargs) + + class MiniMaxInstrumentor(BaseInstrumentor): """ OpenTelemetry instrumentor for MiniMax. @@ -78,7 +92,9 @@ def _instrument(self, **kwargs: Any) -> None: wrap_function_wrapper( module="openai.resources.chat.completions", name="Completions.create", - wrapper=_ChatCompletionWrapper(tracer=self._tracer), + wrapper=_ChatCompletionWrapper( + tracer=self._tracer, attachments=_attachment_manager(kwargs), config=config + ), ) # Async client @@ -89,7 +105,9 @@ def _instrument(self, **kwargs: Any) -> None: wrap_function_wrapper( module="openai.resources.chat.completions", name="AsyncCompletions.create", - wrapper=_AsyncChatCompletionWrapper(tracer=self._tracer), + wrapper=_AsyncChatCompletionWrapper( + tracer=self._tracer, attachments=_attachment_manager(kwargs), config=config + ), ) except Exception as e: logger.warning(f"Failed to instrument OpenAI for MiniMax: {e}") @@ -104,3 +122,12 @@ def _uninstrument(self, **kwargs: Any) -> None: completions.AsyncCompletions.create = self._original_async_chat_create except Exception as e: logger.warning(f"Failed to uninstrument: {e}") + + +def __getattr__(name: str) -> Any: + # Lazy: the descriptor needs a fi-instrumentation-otel release with attachments. + if name == "minimax_attachment_capabilities": + from fi_instrumentation.attachments.provider_media import capabilities + + return capabilities("traceai_minimax") + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/python/frameworks/minimax/traceai_minimax/_wrappers.py b/python/frameworks/minimax/traceai_minimax/_wrappers.py index 6e2eea238..13136bf10 100644 --- a/python/frameworks/minimax/traceai_minimax/_wrappers.py +++ b/python/frameworks/minimax/traceai_minimax/_wrappers.py @@ -1,4 +1,5 @@ import logging +import sys from abc import ABC from contextlib import contextmanager from typing import Any, Callable, Dict, Iterable, Iterator, List, Mapping, Tuple @@ -22,14 +23,73 @@ logger = logging.getLogger(__name__) logger.addHandler(logging.NullHandler()) +# Per-call capture policy key, shared by name (fi_instrumentation.attachments.media_capture). +_CAPTURE_POLICY_KEY = "traceai.auto.attachment_policy.v1" + + +def _shared_instrumentor_captures() -> bool: + """True while traceai_openai's OpenAIInstrumentor is enabled WITH an active attachment manager. + + The request-level OpenAI instrumentor then owns the call (span and media), whatever + the setup order. Enabled without a manager, or an older traceai_openai without the + accessor, means it would capture nothing, so this binding keeps the call. Do not + import the optional package; inspect live state on each call. + """ + module = sys.modules.get("traceai_openai") + instrumentor = getattr(module, "OpenAIInstrumentor", None) + if instrumentor is None: + return False + captures = getattr(instrumentor(), "captures_attachments", None) + return bool(callable(captures) and captures()) class _WithTracer(ABC): """Base class for wrappers that need a tracer.""" - def __init__(self, tracer: trace_api.Tracer, *args: Any, **kwargs: Any) -> None: + def __init__( + self, tracer: trace_api.Tracer, *args: Any, attachments: Any = None, config: Any = None, **kwargs: Any + ) -> None: super().__init__(*args, **kwargs) self._tracer = tracer + # Shared attachment manager, or None (no capture, no imports). + self._attachments = attachments + self._config = config + + def _capture_chat_media(self, span: _WithSpan, messages: Any) -> None: + """Typed chat media (data-URI images) through the shared manager; never raises.""" + if self._attachments is None: + return + try: + from fi_instrumentation.attachments.provider_media import ProviderMedia + + ProviderMedia(self._attachments, span._span, "minimax", trace_config=self._config).compatible_messages( + messages, "image") + except Exception: + logger.debug("Attachment capture unavailable", exc_info=True) + + def _defers(self) -> bool: + """Hand the whole call to traceai_openai only while both sides capture attachments. + + With attachments off here this binding traces exactly as before (its own span too). + With traceai_openai enabled but without a manager, this binding keeps the call and + captures its media itself. + """ + return self._attachments is not None and _shared_instrumentor_captures() + + @contextmanager + def _deferred(self) -> Iterator[None]: + """Run the handed-over call under this binding's local deny (deny wins downstream).""" + try: + from fi_instrumentation.attachments.provider_media import deferred + + token = context_api.attach(deferred(self._config)) + except Exception: + # An unreadable restriction is a restriction: deny capture for this call. + token = context_api.attach(context_api.set_value(_CAPTURE_POLICY_KEY, False)) + try: + yield + finally: + context_api.detach(token) @contextmanager def _start_as_current_span( @@ -87,6 +147,10 @@ def __call__( if not is_minimax_client(client) and not is_minimax_client(instance): return wrapped(*args, **kwargs) + # One request must have one owner, regardless of setup order. + if self._defers(): + with self._deferred(): + return wrapped(*args, **kwargs) request_parameters = dict(kwargs) is_streaming = request_parameters.get("stream", False) @@ -99,6 +163,7 @@ def __call__( context_attributes=get_attributes_from_context(), extra_attributes=self._request_extractor.get_extra_attributes_from_request(request_parameters), ) as span: + self._capture_chat_media(span, request_parameters.get("messages")) try: response = wrapped(*args, **kwargs) except Exception as exception: @@ -194,6 +259,10 @@ async def __call__( if not is_minimax_client(client) and not is_minimax_client(instance): return await wrapped(*args, **kwargs) + # One request must have one owner, regardless of setup order. + if self._defers(): + with self._deferred(): + return await wrapped(*args, **kwargs) request_parameters = dict(kwargs) is_streaming = request_parameters.get("stream", False) @@ -206,6 +275,7 @@ async def __call__( context_attributes=get_attributes_from_context(), extra_attributes=self._request_extractor.get_extra_attributes_from_request(request_parameters), ) as span: + self._capture_chat_media(span, request_parameters.get("messages")) try: response = await wrapped(*args, **kwargs) except Exception as exception: diff --git a/python/frameworks/mistralai/traceai_mistralai/__init__.py b/python/frameworks/mistralai/traceai_mistralai/__init__.py index c72b6139f..969748bfc 100644 --- a/python/frameworks/mistralai/traceai_mistralai/__init__.py +++ b/python/frameworks/mistralai/traceai_mistralai/__init__.py @@ -17,6 +17,20 @@ _MODULE = "mistralai" +def _attachment_manager(kwargs: Any) -> Any: + """The shared manager passed as ``attachments`` (or auto's ``attachment_manager``), if it can admit. + + A missing, disabled or pre-attachments core leaves tracing exactly as before. + """ + if kwargs.get("attachments") is None and kwargs.get("attachment_manager") is None: + return None + try: + from fi_instrumentation.attachments.provider_media import manager_from_kwargs + except ImportError: + return None + return manager_from_kwargs(kwargs) + + class MistralAIInstrumentor(BaseInstrumentor): # type: ignore """ An instrumentor for mistralai @@ -61,10 +75,13 @@ def _instrument(self, **kwargs: Any) -> None: "traceAI-mistralai supports mistralai >=1.1,<2 or >=3.6,<4" ) self._native = None + media = dict(attachments=_attachment_manager(kwargs), config=config) if sdk_version >= Version("3.6.0"): from traceai_mistralai._native import _NativeInstrumentation - self._native = _NativeInstrumentation(tracer_provider, config) + self._native = _NativeInstrumentation( + tracer_provider, config, attachments=media["attachments"] + ) return from traceai_mistralai._chat_wrapper import ( @@ -104,20 +121,20 @@ def _instrument(self, **kwargs: Any) -> None: wrap_function_wrapper( module="mistralai.chat", name="Chat.complete", - wrapper=_SyncChatWrapper("MistralClient.chat", self._tracer, mistralai), + wrapper=_SyncChatWrapper("MistralClient.chat", self._tracer, mistralai, **media), ) wrap_function_wrapper( module="mistralai.chat", name="Chat.stream", - wrapper=_SyncChatWrapper("MistralClient.chat", self._tracer, mistralai), + wrapper=_SyncChatWrapper("MistralClient.chat", self._tracer, mistralai, **media), ) wrap_function_wrapper( module="mistralai.chat", name="Chat.complete_async", wrapper=_AsyncChatWrapper( - "MistralAsyncClient.chat", self._tracer, mistralai + "MistralAsyncClient.chat", self._tracer, mistralai, **media ), ) @@ -125,7 +142,7 @@ def _instrument(self, **kwargs: Any) -> None: module="mistralai.chat", name="Chat.stream_async", wrapper=_AsyncStreamChatWrapper( - "MistralAsyncClient.chat", self._tracer, mistralai + "MistralAsyncClient.chat", self._tracer, mistralai, **media ), ) @@ -201,3 +218,12 @@ def _uninstrument(self, **kwargs: Any) -> None: Agents.stream = self._original_sync_stream_agent_method # type: ignore Agents.complete_async = self._original_async_agent_method # type: ignore Agents.stream_async = self._original_async_stream_agent_method # type: ignore + + +def __getattr__(name: str) -> Any: + # Lazy: the descriptor needs a fi-instrumentation-otel release with attachments. + if name == "mistralai_attachment_capabilities": + from fi_instrumentation.attachments.provider_media import capabilities + + return capabilities("traceai_mistralai") + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/python/frameworks/mistralai/traceai_mistralai/_chat_wrapper.py b/python/frameworks/mistralai/traceai_mistralai/_chat_wrapper.py index 940ceaa7d..21398b445 100644 --- a/python/frameworks/mistralai/traceai_mistralai/_chat_wrapper.py +++ b/python/frameworks/mistralai/traceai_mistralai/_chat_wrapper.py @@ -102,9 +102,32 @@ async def _async_parse(wrapped, instance, args, kwargs): class _WithTracer(ABC): - def __init__(self, tracer: trace_api.Tracer, *args: Any, **kwargs: Any) -> None: + def __init__( + self, + tracer: trace_api.Tracer, + *args: Any, + attachments: Any = None, + config: Any = None, + **kwargs: Any, + ) -> None: super().__init__(*args, **kwargs) self._tracer = tracer + # Provider-owned shared attachment manager, or None (no capture, no imports). + self._attachments = attachments + self._config = config + + def _capture_chat_media(self, span: Any, messages: Any) -> None: + """Typed chat media (data-URI images) through the shared attachment manager; never raises.""" + if self._attachments is None: + return + try: + from fi_instrumentation.attachments.provider_media import ProviderMedia + + ProviderMedia( + self._attachments, span, "mistralai", trace_config=self._config + ).compatible_messages(messages, "mistral") + except Exception: + logger.debug("Attachment capture unavailable", exc_info=True) @contextmanager def _start_as_current_span( @@ -304,6 +327,7 @@ def __call__( request_parameters ), ) as with_span: + self._capture_chat_media(with_span._span, kwargs.get("messages")) try: response = wrapped(*args, **kwargs) except Exception as exception: @@ -359,6 +383,7 @@ async def __call__( request_parameters ), ) as with_span: + self._capture_chat_media(with_span._span, kwargs.get("messages")) try: response = await wrapped(*args, **kwargs) except (Exception, asyncio.CancelledError) as exception: @@ -414,6 +439,7 @@ async def __call__( request_parameters ), ) as with_span: + self._capture_chat_media(with_span._span, kwargs.get("messages")) try: response = await wrapped(*args, **kwargs) except (Exception, asyncio.CancelledError) as exception: diff --git a/python/frameworks/mistralai/traceai_mistralai/_native.py b/python/frameworks/mistralai/traceai_mistralai/_native.py index d6d19ee9b..38564d911 100644 --- a/python/frameworks/mistralai/traceai_mistralai/_native.py +++ b/python/frameworks/mistralai/traceai_mistralai/_native.py @@ -152,10 +152,17 @@ class _NativeInstrumentation: "_telemetry_auto_disabled", ) - def __init__(self, provider: trace.TracerProvider, config: TraceConfig) -> None: + def __init__( + self, provider: trace.TracerProvider, config: TraceConfig, attachments: Any = None + ) -> None: from mistralai.client._hooks.tracing import TracingHook self._provider = _NativeProvider(provider, config) + # Typed chat messages of the current call, handed to the SDK-created request span + # in before_request. Only set while an attachment manager is configured. + self._attachments = attachments + self._config = config + self._media_messages = ContextVar("mistral_media_messages", default=None) self._hooks = WeakKeyDictionary() self._async_sends = WeakKeyDictionary() self._lock = RLock() @@ -224,13 +231,54 @@ def _call_scope(self): self._response_scope.reset(token) def _call(self, wrapped, instance, args, kwargs): - with self._call_scope(): + with self._call_scope(), self._media_scope(wrapped, instance, kwargs): return wrapped(*args, **kwargs) async def _call_async(self, wrapped, instance, args, kwargs): - with self._call_scope(): + with self._call_scope(), self._media_scope(wrapped, instance, kwargs): return await wrapped(*args, **kwargs) + @contextmanager + def _media_scope(self, wrapped, instance, kwargs): + """Hold chat.complete/stream messages for the first request span of this call.""" + messages = None + if self._attachments is not None and self._media_messages.get() is None: + try: + from mistralai.client.chat import Chat + + if isinstance(instance, Chat) and getattr(wrapped, "__name__", "").split("_")[0] in ( + "complete", + "stream", + ): + messages = kwargs.get("messages") + except Exception: + messages = None + if messages is None: + yield + return + token = self._media_messages.set([messages]) + try: + yield + finally: + self._media_messages.reset(token) + + def _capture_media(self, instance, request) -> None: + holder = self._media_messages.get() + if not holder or holder[0] is None: + return + messages, holder[0] = holder[0], None # Retries never upload the same call twice. + try: + span = instance._get_request_span(request) + if not isinstance(span, _NativeSpan): + return + from fi_instrumentation.attachments.provider_media import ProviderMedia + + ProviderMedia( + self._attachments, span, "mistralai", trace_config=self._config + ).compatible_messages(messages, "mistral") + except Exception: + logger.debug("Attachment capture unavailable") + def _before_request(self, wrapped, instance, args, kwargs): with self._lock: if ( @@ -244,6 +292,8 @@ def _before_request(self, wrapped, instance, args, kwargs): # Do not let SDK auto-export configuration create a second destination. instance._telemetry_auto_disabled = True request = wrapped(*args, **kwargs) + if self._attachments is not None: + self._capture_media(instance, request) hook_ctx = kwargs.get("hook_ctx", args[0] if args else None) client = getattr(getattr(hook_ctx, "config", None), "async_client", None) if client is not None: diff --git a/python/frameworks/ollama/traceai_ollama/__init__.py b/python/frameworks/ollama/traceai_ollama/__init__.py index 2743dbe08..6d44a132e 100644 --- a/python/frameworks/ollama/traceai_ollama/__init__.py +++ b/python/frameworks/ollama/traceai_ollama/__init__.py @@ -32,6 +32,20 @@ _instruments = ("ollama >= 0.3.0",) +def _attachment_manager(kwargs: Any) -> Any: + """The shared manager passed as ``attachments`` (or auto's ``attachment_manager``), if it can admit. + + A missing, disabled or pre-attachments core leaves tracing exactly as before. + """ + if kwargs.get("attachments") is None and kwargs.get("attachment_manager") is None: + return None + try: + from fi_instrumentation.attachments.provider_media import manager_from_kwargs + except ImportError: + return None + return manager_from_kwargs(kwargs) + + class OllamaInstrumentor(BaseInstrumentor): """OpenTelemetry instrumentor for Ollama.""" @@ -64,20 +78,21 @@ def _instrument(self, **kwargs: Any) -> None: ) ollama_module = import_module("ollama") + attachments = _attachment_manager(kwargs) # Wrap synchronous Client methods self._original_chat = ollama_module.Client.chat wrap_function_wrapper( module="ollama", name="Client.chat", - wrapper=_ChatWrapper(tracer=self._tracer), + wrapper=_ChatWrapper(tracer=self._tracer, attachments=attachments), ) self._original_generate = ollama_module.Client.generate wrap_function_wrapper( module="ollama", name="Client.generate", - wrapper=_GenerateWrapper(tracer=self._tracer), + wrapper=_GenerateWrapper(tracer=self._tracer, attachments=attachments), ) # Ollama uses 'embed' for single and 'embeddings' for batch @@ -102,14 +117,14 @@ def _instrument(self, **kwargs: Any) -> None: wrap_function_wrapper( module="ollama", name="AsyncClient.chat", - wrapper=_AsyncChatWrapper(tracer=self._tracer), + wrapper=_AsyncChatWrapper(tracer=self._tracer, attachments=attachments), ) self._original_async_generate = ollama_module.AsyncClient.generate wrap_function_wrapper( module="ollama", name="AsyncClient.generate", - wrapper=_AsyncGenerateWrapper(tracer=self._tracer), + wrapper=_AsyncGenerateWrapper(tracer=self._tracer, attachments=attachments), ) if hasattr(ollama_module.AsyncClient, "embed"): @@ -133,14 +148,14 @@ def _instrument(self, **kwargs: Any) -> None: wrap_function_wrapper( module="ollama", name="chat", - wrapper=_ChatWrapper(tracer=self._tracer), + wrapper=_ChatWrapper(tracer=self._tracer, attachments=attachments), ) if hasattr(ollama_module, "generate"): wrap_function_wrapper( module="ollama", name="generate", - wrapper=_GenerateWrapper(tracer=self._tracer), + wrapper=_GenerateWrapper(tracer=self._tracer, attachments=attachments), ) if hasattr(ollama_module, "embed"): @@ -187,3 +202,12 @@ def _uninstrument(self, **kwargs: Any) -> None: ollama_module.AsyncClient.embed = self._original_async_embed if hasattr(self, "_original_async_embeddings"): ollama_module.AsyncClient.embeddings = self._original_async_embeddings + + +def __getattr__(name: str) -> Any: + # Lazy: the descriptor needs a fi-instrumentation-otel release with attachments. + if name == "ollama_attachment_capabilities": + from fi_instrumentation.attachments.provider_media import capabilities + + return capabilities("traceai_ollama") + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/python/frameworks/ollama/traceai_ollama/_wrappers.py b/python/frameworks/ollama/traceai_ollama/_wrappers.py index b5f33c22e..dcbd43c88 100644 --- a/python/frameworks/ollama/traceai_ollama/_wrappers.py +++ b/python/frameworks/ollama/traceai_ollama/_wrappers.py @@ -23,12 +23,29 @@ class _WithTracer(ABC): """Base class for wrappers that need a tracer.""" - def __init__(self, tracer: trace_api.Tracer, *args: Any, **kwargs: Any) -> None: + def __init__(self, tracer: trace_api.Tracer, *args: Any, attachments: Any = None, **kwargs: Any) -> None: super().__init__(*args, **kwargs) self._tracer = tracer # Use the exact FITracer config: explicit options override environment values. config = getattr(tracer, "_self_config", None) self._config = config if isinstance(config, TraceConfig) else TraceConfig() + # Provider-owned shared attachment manager, or None (no capture, no imports). + self._attachments = attachments + + def _capture_images(self, span: Any, request: Mapping[str, Any]) -> None: + """Ollama ``images`` (bytes or base64; never a path) through the shared manager; never raises.""" + if self._attachments is None: + return + try: + from fi_instrumentation.attachments.provider_media import ProviderMedia + + media = ProviderMedia(self._attachments, span, "ollama", trace_config=self._config) + if "messages" in request: + media.ollama_messages(request.get("messages")) + else: + media.ollama_images(request.get("images")) + except Exception: + logger.debug("Attachment capture unavailable", exc_info=True) @contextmanager def _start_as_current_span( @@ -91,6 +108,7 @@ def __call__( context_attributes=get_attributes_from_context(), extra_attributes=self._request_extractor.get_extra_attributes_from_request(request_parameters), ) as span: + self._capture_images(span._span, request_parameters) try: response = wrapped(*args, **kwargs) except Exception as exception: @@ -222,6 +240,7 @@ async def __call__( context_attributes=get_attributes_from_context(), extra_attributes=self._request_extractor.get_extra_attributes_from_request(request_parameters), ) as span: + self._capture_images(span._span, request_parameters) try: response = await wrapped(*args, **kwargs) except Exception as exception: @@ -350,6 +369,7 @@ def __call__( context_attributes=get_attributes_from_context(), extra_attributes=self._request_extractor.get_extra_attributes_from_request(request_parameters), ) as span: + self._capture_images(span._span, request_parameters) try: response = wrapped(*args, **kwargs) except Exception as exception: @@ -475,6 +495,7 @@ async def __call__( context_attributes=get_attributes_from_context(), extra_attributes=self._request_extractor.get_extra_attributes_from_request(request_parameters), ) as span: + self._capture_images(span._span, request_parameters) try: response = await wrapped(*args, **kwargs) except Exception as exception: diff --git a/python/frameworks/openai/traceai_openai/__init__.py b/python/frameworks/openai/traceai_openai/__init__.py index 55d01cc34..07c6522fc 100644 --- a/python/frameworks/openai/traceai_openai/__init__.py +++ b/python/frameworks/openai/traceai_openai/__init__.py @@ -77,6 +77,7 @@ def _instrument(self, **kwargs: Any) -> None: # The provider-owned shared manager (register(...).attachment_manager). Auto passes # it as attachment_manager; nothing is captured unless an enabled manager is supplied. attachments = _attachment_manager(kwargs) + self._attachments = attachments openai = import_module(_MODULE) self._original_request = openai.OpenAI.request self._original_async_request = openai.AsyncOpenAI.request @@ -110,7 +111,16 @@ def _instrument(self, **kwargs: Any) -> None: else: self._original_protect = None + def captures_attachments(self) -> bool: + """True while this instrumentor is enabled with an active shared attachment manager. + + Bindings that ride on the OpenAI SDK (xAI, MiniMax) hand a call over to this + instrumentor only then; otherwise they keep tracing and capturing it themselves. + """ + return bool(self.is_instrumented_by_opentelemetry and getattr(self, "_attachments", None) is not None) + def _uninstrument(self, **kwargs: Any) -> None: + self._attachments = None openai = import_module(_MODULE) openai.OpenAI.request = self._original_request openai.AsyncOpenAI.request = self._original_async_request diff --git a/python/frameworks/openrouter/traceai_openrouter/__init__.py b/python/frameworks/openrouter/traceai_openrouter/__init__.py index ef653184a..a48a3b65b 100644 --- a/python/frameworks/openrouter/traceai_openrouter/__init__.py +++ b/python/frameworks/openrouter/traceai_openrouter/__init__.py @@ -238,7 +238,7 @@ async def close(self): self._finish(complete=False) -def _start(tracer: Any, operation: str, kwargs: dict[str, Any], config: TraceConfig): +def _start(tracer: Any, operation: str, kwargs: dict[str, Any], config: TraceConfig, attachments: Any = None): kind = FiSpanKindValues.EMBEDDING if operation == "embeddings" else FiSpanKindValues.LLM model = kwargs.get("model") attrs = {A.GEN_AI_PROVIDER_NAME: "openrouter", A.GEN_AI_OPERATION_NAME: operation} @@ -251,10 +251,22 @@ def _start(tracer: Any, operation: str, kwargs: dict[str, Any], config: TraceCon config.hide_inputs or config.hide_input_text or config.hide_input_messages or config.hide_input_images ) _input(span, operation, kwargs, capture_input) + if attachments is not None and operation == "chat": + _capture_chat_media(attachments, span, kwargs.get("messages"), config) return span -def _wrap(tracer: Any, operation: str, config: TraceConfig, is_async: bool): +def _capture_chat_media(attachments: Any, span: Any, messages: Any, config: TraceConfig) -> None: + """Chat ``image_url`` data URIs and ``input_audio`` through the shared manager; never raises.""" + try: + from fi_instrumentation.attachments.provider_media import ProviderMedia + + ProviderMedia(attachments, span, "openrouter", trace_config=config).compatible_messages(messages, "openai") + except Exception: + pass + + +def _wrap(tracer: Any, operation: str, config: TraceConfig, is_async: bool, attachments: Any = None): def complete(span, result): capture_output = not (config.hide_outputs or config.hide_output_text or config.hide_output_messages) if hasattr(result, "__anext__"): @@ -267,7 +279,7 @@ def complete(span, result): if is_async: async def wrapped(wrapped, instance, args, kwargs): - span = _start(tracer, operation, kwargs, config) + span = _start(tracer, operation, kwargs, config, attachments) try: with trace.use_span(span, end_on_exit=False, record_exception=False, set_status_on_exception=False): result = await wrapped(*args, **kwargs) @@ -277,7 +289,7 @@ async def wrapped(wrapped, instance, args, kwargs): raise else: def wrapped(wrapped, instance, args, kwargs): - span = _start(tracer, operation, kwargs, config) + span = _start(tracer, operation, kwargs, config, attachments) try: with trace.use_span(span, end_on_exit=False, record_exception=False, set_status_on_exception=False): result = wrapped(*args, **kwargs) @@ -288,6 +300,20 @@ def wrapped(wrapped, instance, args, kwargs): return wrapped +def _attachment_manager(kwargs: Any) -> Any: + """The shared manager passed as ``attachments`` (or auto's ``attachment_manager``), if it can admit. + + A missing, disabled or pre-attachments core leaves tracing exactly as before. + """ + if kwargs.get("attachments") is None and kwargs.get("attachment_manager") is None: + return None + try: + from fi_instrumentation.attachments.provider_media import manager_from_kwargs + except ImportError: + return None + return manager_from_kwargs(kwargs) + + class OpenRouterInstrumentor(BaseInstrumentor): # type: ignore[misc] """Instrument dedicated OpenRouter chat, embeddings and Responses calls.""" @@ -299,12 +325,13 @@ def _instrument(self, **kwargs: Any) -> None: if not isinstance(config, TraceConfig): raise TypeError("config must be a TraceConfig") tracer = FITracer(trace.get_tracer(__name__, __version__, kwargs.get("tracer_provider")), config) + attachments = _attachment_manager(kwargs) self._originals = [] for module, cls_name, operation, method in _ENDPOINTS: cls = getattr(import_module(module), cls_name) for name, is_async in ((method, False), (method + "_async", True)): original = cls.__dict__[name] - wrap_function_wrapper(module, cls_name + "." + name, _wrap(tracer, operation, config, is_async)) + wrap_function_wrapper(module, cls_name + "." + name, _wrap(tracer, operation, config, is_async, attachments)) self._originals.append((cls, name, original)) def _uninstrument(self, **kwargs: Any) -> None: @@ -314,3 +341,11 @@ def _uninstrument(self, **kwargs: Any) -> None: __all__ = ["OpenRouterInstrumentor", "__version__"] + +def __getattr__(name: str) -> Any: + # Lazy: the descriptor needs a fi-instrumentation-otel release with attachments. + if name == "openrouter_attachment_capabilities": + from fi_instrumentation.attachments.provider_media import capabilities + + return capabilities("traceai_openrouter") + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/python/frameworks/portkey/traceai_portkey/__init__.py b/python/frameworks/portkey/traceai_portkey/__init__.py index 6f0aa70fc..4b7594d9c 100644 --- a/python/frameworks/portkey/traceai_portkey/__init__.py +++ b/python/frameworks/portkey/traceai_portkey/__init__.py @@ -17,6 +17,20 @@ _instruments = ("portkey_ai >= 0.1.0",) +def _attachment_manager(kwargs: Any) -> Any: + """The shared manager passed as ``attachments`` (or auto's ``attachment_manager``), if it can admit. + + A missing, disabled or pre-attachments core leaves tracing exactly as before. + """ + if kwargs.get("attachments") is None and kwargs.get("attachment_manager") is None: + return None + try: + from fi_instrumentation.attachments.provider_media import manager_from_kwargs + except ImportError: + return None + return manager_from_kwargs(kwargs) + + class PortkeyInstrumentor(BaseInstrumentor): # type: ignore[misc] """An instrumentor for the Portkey AI framework.""" @@ -41,7 +55,10 @@ def _instrument(self, **kwargs: Any) -> None: wrap_function_wrapper( module="portkey_ai.api_resources.apis.chat_complete", name="Completions.create", - wrapper=_CompletionsWrapper(tracer=self._tracer), + # Chat only: the generation API carries no typed media. + wrapper=_CompletionsWrapper( + tracer=self._tracer, attachments=_attachment_manager(kwargs), config=config + ), ) wrap_function_wrapper( module="portkey_ai.api_resources.apis.generation", @@ -52,4 +69,12 @@ def _instrument(self, **kwargs: Any) -> None: def _uninstrument(self, **kwargs: Any) -> None: portkey_module = import_module("portkey_ai.api_resources.apis.chat_complete") if self._original_completions_create is not None: - portkey_module.Completions.create = self._original_completions_create \ No newline at end of file + portkey_module.Completions.create = self._original_completions_create + +def __getattr__(name: str) -> Any: + # Lazy: the descriptor needs a fi-instrumentation-otel release with attachments. + if name == "portkey_attachment_capabilities": + from fi_instrumentation.attachments.provider_media import capabilities + + return capabilities("traceai_portkey") + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/python/frameworks/portkey/traceai_portkey/_wrappers.py b/python/frameworks/portkey/traceai_portkey/_wrappers.py index 941c02814..d608bfa8d 100644 --- a/python/frameworks/portkey/traceai_portkey/_wrappers.py +++ b/python/frameworks/portkey/traceai_portkey/_wrappers.py @@ -44,10 +44,35 @@ def _flatten(mapping: Mapping[str, Any]) -> Iterator[Tuple[str, AttributeValue]] class _WithTracer(ABC): """Base class for wrappers that use a tracer.""" - def __init__(self, tracer: trace_api.Tracer, *args: Any, **kwargs: Any) -> None: + _media_dialect = "openai" + + def __init__( + self, + tracer: trace_api.Tracer, + *args: Any, + attachments: Any = None, + config: Any = None, + **kwargs: Any, + ) -> None: self._tracer = tracer + # Provider-owned shared attachment manager, or None (no capture, no imports). + self._attachments = attachments + self._config = config super().__init__(*args, **kwargs) + def _capture_chat_media(self, span: Any, messages: Any) -> None: + """Typed chat media (data-URI images) through the shared attachment manager; never raises.""" + if self._attachments is None: + return + try: + from fi_instrumentation.attachments.provider_media import ProviderMedia + + ProviderMedia( + self._attachments, span, "portkey", trace_config=self._config + ).compatible_messages(messages, self._media_dialect) + except Exception: + logger.debug("Attachment capture unavailable", exc_info=True) + @contextmanager def _start_as_current_span( self, @@ -120,6 +145,7 @@ def __call__( request_parameters ), ) as span: + self._capture_chat_media(span._span, kwargs.get("messages")) try: response = wrapped(*args, **kwargs) except Exception as exception: diff --git a/python/frameworks/together/traceai_together/__init__.py b/python/frameworks/together/traceai_together/__init__.py index 7fca7fb8f..c2a2b284f 100644 --- a/python/frameworks/together/traceai_together/__init__.py +++ b/python/frameworks/together/traceai_together/__init__.py @@ -32,6 +32,20 @@ _instruments = ("together >= 1.0.0",) +def _attachment_manager(kwargs: Any) -> Any: + """The shared manager passed as ``attachments`` (or auto's ``attachment_manager``), if it can admit. + + A missing, disabled or pre-attachments core leaves tracing exactly as before. + """ + if kwargs.get("attachments") is None and kwargs.get("attachment_manager") is None: + return None + try: + from fi_instrumentation.attachments.provider_media import manager_from_kwargs + except ImportError: + return None + return manager_from_kwargs(kwargs) + + class TogetherInstrumentor(BaseInstrumentor): """OpenTelemetry instrumentor for Together AI.""" @@ -61,6 +75,7 @@ def _instrument(self, **kwargs: Any) -> None: trace_api.get_tracer(__name__, __version__, tracer_provider), config=config, ) + media = dict(attachments=_attachment_manager(kwargs), config=config) # Import together client together_module = import_module("together") @@ -88,7 +103,7 @@ def _instrument(self, **kwargs: Any) -> None: wrap_function_wrapper( module="together.resources.chat", name="ChatCompletions.create", - wrapper=_ChatCompletionsWrapper(tracer=self._tracer), + wrapper=_ChatCompletionsWrapper(tracer=self._tracer, **media), ) except (ImportError, AttributeError) as e: logger.debug(f"Could not wrap chat.completions.create: {e}") @@ -130,7 +145,7 @@ def _instrument(self, **kwargs: Any) -> None: wrap_function_wrapper( module="together.resources.chat", name="AsyncChatCompletions.create", - wrapper=_AsyncChatCompletionsWrapper(tracer=self._tracer), + wrapper=_AsyncChatCompletionsWrapper(tracer=self._tracer, **media), ) except (ImportError, AttributeError) as e: logger.debug(f"Could not wrap async chat.completions.create: {e}") @@ -222,3 +237,12 @@ def _uninstrument(self, **kwargs: Any) -> None: except Exception as e: logger.exception(f"Error uninstrumenting Together client: {e}") + + +def __getattr__(name: str) -> Any: + # Lazy: the descriptor needs a fi-instrumentation-otel release with attachments. + if name == "together_attachment_capabilities": + from fi_instrumentation.attachments.provider_media import capabilities + + return capabilities("traceai_together") + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/python/frameworks/together/traceai_together/_wrappers.py b/python/frameworks/together/traceai_together/_wrappers.py index 6e71c5e8f..660a2ebae 100644 --- a/python/frameworks/together/traceai_together/_wrappers.py +++ b/python/frameworks/together/traceai_together/_wrappers.py @@ -32,9 +32,34 @@ class _WithTracer(ABC): """Base class for wrappers that need a tracer.""" - def __init__(self, tracer: trace_api.Tracer, *args: Any, **kwargs: Any) -> None: + _media_dialect = "image" + + def __init__( + self, + tracer: trace_api.Tracer, + *args: Any, + attachments: Any = None, + config: Any = None, + **kwargs: Any, + ) -> None: super().__init__(*args, **kwargs) self._tracer = tracer + # Provider-owned shared attachment manager, or None (no capture, no imports). + self._attachments = attachments + self._config = config + + def _capture_chat_media(self, span: Any, messages: Any) -> None: + """Typed chat media (data-URI images) through the shared attachment manager; never raises.""" + if self._attachments is None: + return + try: + from fi_instrumentation.attachments.provider_media import ProviderMedia + + ProviderMedia( + self._attachments, span, "together", trace_config=self._config + ).compatible_messages(messages, self._media_dialect) + except Exception: + logger.debug("Attachment capture unavailable", exc_info=True) @contextmanager def _start_as_current_span( @@ -97,6 +122,7 @@ def __call__( context_attributes=get_attributes_from_context(), extra_attributes=self._request_extractor.get_extra_attributes_from_request(request_parameters), ) as span: + self._capture_chat_media(span._span, kwargs.get("messages")) span.set_attribute(SpanAttributes.INPUT_VALUE, safe_json_dumps(request_parameters)) try: @@ -204,6 +230,7 @@ async def __call__( context_attributes=get_attributes_from_context(), extra_attributes=self._request_extractor.get_extra_attributes_from_request(request_parameters), ) as span: + self._capture_chat_media(span._span, kwargs.get("messages")) span.set_attribute(SpanAttributes.INPUT_VALUE, safe_json_dumps(request_parameters)) try: diff --git a/python/frameworks/vertexai/traceai_vertexai/__init__.py b/python/frameworks/vertexai/traceai_vertexai/__init__.py index e35531974..34aabfa1d 100644 --- a/python/frameworks/vertexai/traceai_vertexai/__init__.py +++ b/python/frameworks/vertexai/traceai_vertexai/__init__.py @@ -21,6 +21,20 @@ logger.addHandler(logging.NullHandler()) +def _attachment_manager(kwargs: Any) -> Any: + """The shared manager passed as ``attachments`` (or auto's ``attachment_manager``), if it can admit. + + A missing, disabled or pre-attachments core leaves tracing exactly as before. + """ + if kwargs.get("attachments") is None and kwargs.get("attachment_manager") is None: + return None + try: + from fi_instrumentation.attachments.provider_media import manager_from_kwargs + except ImportError: + return None + return manager_from_kwargs(kwargs) + + class VertexAIInstrumentor(BaseInstrumentor): # type: ignore """ An instrumentor for `google-cloud-aiplatform` or the `vertexai` SDK @@ -46,11 +60,12 @@ def _instrument(self, **kwargs: Any) -> None: import google.api_core.gapic_v1 as gapic from traceai_vertexai._wrapper import _Wrapper + media = dict(attachments=_attachment_manager(kwargs), config=config) for method in (gapic.method.wrap_method, gapic.method_async.wrap_method): wrap_function_wrapper( module=method.__module__, name=method.__name__, - wrapper=lambda f, _, args, kwargs: _Wrapper(tracer)(f(*args, **kwargs)), + wrapper=lambda f, _, args, kwargs: _Wrapper(tracer, **media)(f(*args, **kwargs)), ) if Protect is not None: self._original_protect = Protect.protect @@ -69,3 +84,12 @@ def _uninstrument(self, **kwargs: Any) -> None: for module in (gapic.method, gapic.method_async): if wrapped := getattr(module.wrap_method, "__wrapped__", None): setattr(module, "wrap_method", wrapped) + + +def __getattr__(name: str) -> Any: + # Lazy: the descriptor needs a fi-instrumentation-otel release with attachments. + if name == "vertexai_attachment_capabilities": + from fi_instrumentation.attachments.provider_media import capabilities + + return capabilities("traceai_vertexai") + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/python/frameworks/vertexai/traceai_vertexai/_wrapper.py b/python/frameworks/vertexai/traceai_vertexai/_wrapper.py index c4dfcc062..8c26de71a 100644 --- a/python/frameworks/vertexai/traceai_vertexai/_wrapper.py +++ b/python/frameworks/vertexai/traceai_vertexai/_wrapper.py @@ -112,9 +112,19 @@ class _Wrapper: variable `_instrumentation_status._IS_INSTRUMENTED` to False or True, respectively. """ - def __init__(self, tracer: trace_api.Tracer, *args: Any, **kwargs: Any) -> None: + def __init__( + self, + tracer: trace_api.Tracer, + *args: Any, + attachments: Any = None, + config: Any = None, + **kwargs: Any, + ) -> None: super().__init__(*args, **kwargs) self._tracer = tracer + # Provider-owned shared attachment manager, or None (no capture, no imports). + self._attachments = attachments + self._config = config @wrapt.decorator # type: ignore[misc] def __call__( @@ -160,6 +170,7 @@ def __call__( _update_span(request, span) except BaseException: pass + media = _media_input(self._attachments, self._config, span, request) try: with use_span(span, False, False, False): result = wrapped(*args, **kwargs) @@ -169,13 +180,44 @@ def __call__( span.end() raise if isinstance(result, Awaitable): - return _proxy(result, _CallbackForAwaitable(request, span), _use_span(span)) + return _proxy(result, _CallbackForAwaitable(request, span, media), _use_span(span)) if isinstance(result, (Iterable, AsyncIterable)): return _proxy(result, _CallbackForIterable(request, span), _use_span(span)) + _media_output(media, request, result) _finish(span, result) return result +def _media_input(attachments: Any, config: Any, span: Span, request: Any) -> Any: + """Typed inline_data Parts of a GenerateContentRequest; returns the capture for the response.""" + if attachments is None: + return None + try: + from fi_instrumentation.attachments.provider_media import ProviderMedia + + media = ProviderMedia(attachments, span, "vertexai", trace_config=config) + if not media.usable: + return None + if isinstance(request, (v1.GenerateContentRequest, v1beta1.GenerateContentRequest)): + media.gemini_contents(list(request.contents)) + return media + except Exception: + return None + + +def _media_output(media: Any, request: Any, result: Any) -> None: + """Completed responses only: Gemini candidate inline_data and Imagen predictions.""" + if media is None: + return + try: + if isinstance(result, (v1.GenerateContentResponse, v1beta1.GenerateContentResponse)): + media.gemini_response(result) + elif isinstance(result, v1.PredictResponse) and isinstance(request, v1.PredictRequest): + media.imagen_predictions(request.endpoint, result.predictions) + except Exception: + pass + + class _CallbackForAwaitable: """ Callback function for the proxy object when wrapping an Awaitable. This is how we get notified @@ -183,9 +225,10 @@ class _CallbackForAwaitable: Awaitable or when an exception is raised. """ - def __init__(self, request: proto.Message, span: Span) -> None: + def __init__(self, request: proto.Message, span: Span, media: Any = None) -> None: self._request = request self._span = span + self._media = media def __call__(self, obj: _AnyT) -> _AnyT: request, span = self._request, self._span @@ -194,6 +237,8 @@ def __call__(self, obj: _AnyT) -> _AnyT: _AnyT, _proxy(obj, _CallbackForIterable(request, span), _use_span(span)) ) else: + if not isinstance(obj, BaseException): + _media_output(self._media, request, obj) _finish(span, obj) return obj diff --git a/python/frameworks/vllm/traceai_vllm/__init__.py b/python/frameworks/vllm/traceai_vllm/__init__.py index 94413cd86..04753bf3a 100644 --- a/python/frameworks/vllm/traceai_vllm/__init__.py +++ b/python/frameworks/vllm/traceai_vllm/__init__.py @@ -33,6 +33,20 @@ ] +def _attachment_manager(kwargs: Any) -> Any: + """The shared manager passed as ``attachments`` (or auto's ``attachment_manager``), if it can admit. + + A missing, disabled or pre-attachments core leaves tracing exactly as before. + """ + if kwargs.get("attachments") is None and kwargs.get("attachment_manager") is None: + return None + try: + from fi_instrumentation.attachments.provider_media import manager_from_kwargs + except ImportError: + return None + return manager_from_kwargs(kwargs) + + class VLLMInstrumentor(BaseInstrumentor): """An instrumentor for vLLM (OpenAI-compatible API).""" @@ -93,7 +107,10 @@ def _instrument(self, **kwargs: Any) -> None: wrap_function_wrapper( module="openai.resources.chat", name="Completions.create", - wrapper=_CompletionsWrapper(tracer=self._tracer, vllm_base_urls=self._vllm_base_urls), + wrapper=_CompletionsWrapper( + tracer=self._tracer, vllm_base_urls=self._vllm_base_urls, + attachments=_attachment_manager(kwargs), config=config, + ), ) # Wrap asynchronous completions @@ -101,7 +118,10 @@ def _instrument(self, **kwargs: Any) -> None: wrap_function_wrapper( module="openai.resources.chat", name="AsyncCompletions.create", - wrapper=_AsyncCompletionsWrapper(tracer=self._tracer, vllm_base_urls=self._vllm_base_urls), + wrapper=_AsyncCompletionsWrapper( + tracer=self._tracer, vllm_base_urls=self._vllm_base_urls, + attachments=_attachment_manager(kwargs), config=config, + ), ) def _uninstrument(self, **kwargs: Any) -> None: @@ -113,3 +133,12 @@ def _uninstrument(self, **kwargs: Any) -> None: openai_module.AsyncCompletions.create = self._original_async_completions_create except ImportError: pass + + +def __getattr__(name: str) -> Any: + # Lazy: the descriptor needs a fi-instrumentation-otel release with attachments. + if name == "vllm_attachment_capabilities": + from fi_instrumentation.attachments.provider_media import capabilities + + return capabilities("traceai_vllm") + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/python/frameworks/vllm/traceai_vllm/_wrappers.py b/python/frameworks/vllm/traceai_vllm/_wrappers.py index 29a3ab580..2b3aebacc 100644 --- a/python/frameworks/vllm/traceai_vllm/_wrappers.py +++ b/python/frameworks/vllm/traceai_vllm/_wrappers.py @@ -44,11 +44,28 @@ def __init__( tracer: trace_api.Tracer, vllm_base_urls: List[str], *args: Any, + attachments: Any = None, + config: Any = None, **kwargs: Any, ) -> None: super().__init__(*args, **kwargs) self._tracer = tracer self._vllm_base_urls = vllm_base_urls + # Shared attachment manager, or None (no capture, no imports). + self._attachments = attachments + self._config = config + + def _capture_chat_media(self, span: Any, messages: Any) -> None: + """Typed chat media (image_url and input_audio data) through the shared manager; never raises.""" + if self._attachments is None: + return + try: + from fi_instrumentation.attachments.provider_media import ProviderMedia + + ProviderMedia(self._attachments, span, "vllm", trace_config=self._config).compatible_messages( + messages, "openai") + except Exception: + logger.debug("Attachment capture unavailable", exc_info=True) def _is_vllm_client(self, instance: Any) -> bool: """Check if the client instance is configured for vLLM.""" @@ -268,6 +285,7 @@ def __call__( input_attrs = _get_input_attributes(model, messages, **extra_kwargs) with self._start_span("vLLM Chat Completions", input_attrs) as span: + self._capture_chat_media(span, messages) try: response = wrapped(*args, **kwargs) @@ -312,6 +330,7 @@ async def __call__( input_attrs = _get_input_attributes(model, messages, **extra_kwargs) with self._start_span("vLLM Chat Completions", input_attrs) as span: + self._capture_chat_media(span, messages) try: response = await wrapped(*args, **kwargs) diff --git a/python/frameworks/xai/traceai_xai/__init__.py b/python/frameworks/xai/traceai_xai/__init__.py index 45820a151..952b112ed 100644 --- a/python/frameworks/xai/traceai_xai/__init__.py +++ b/python/frameworks/xai/traceai_xai/__init__.py @@ -58,6 +58,20 @@ ] +def _attachment_manager(kwargs: Any) -> Any: + """The shared manager passed as ``attachments`` (or auto's ``attachment_manager``), if it can admit. + + A missing, disabled or pre-attachments core leaves tracing exactly as before. + """ + if kwargs.get("attachments") is None and kwargs.get("attachment_manager") is None: + return None + try: + from fi_instrumentation.attachments.provider_media import manager_from_kwargs + except ImportError: + return None + return manager_from_kwargs(kwargs) + + class XAIInstrumentor(BaseInstrumentor): """An instrumentor for xAI (Grok) (OpenAI-compatible API).""" @@ -98,7 +112,7 @@ def _instrument(self, **kwargs: Any) -> None: wrap_function_wrapper( module="openai.resources.chat", name="Completions.create", - wrapper=_CompletionsWrapper(tracer=self._tracer), + wrapper=_CompletionsWrapper(tracer=self._tracer, attachments=_attachment_manager(kwargs), config=config), ) # Wrap asynchronous completions @@ -106,7 +120,9 @@ def _instrument(self, **kwargs: Any) -> None: wrap_function_wrapper( module="openai.resources.chat", name="AsyncCompletions.create", - wrapper=_AsyncCompletionsWrapper(tracer=self._tracer), + wrapper=_AsyncCompletionsWrapper( + tracer=self._tracer, attachments=_attachment_manager(kwargs), config=config + ), ) def _uninstrument(self, **kwargs: Any) -> None: @@ -118,3 +134,12 @@ def _uninstrument(self, **kwargs: Any) -> None: openai_module.AsyncCompletions.create = self._original_async_completions_create except ImportError: pass + + +def __getattr__(name: str) -> Any: + # Lazy: the descriptor needs a fi-instrumentation-otel release with attachments. + if name == "xai_attachment_capabilities": + from fi_instrumentation.attachments.provider_media import capabilities + + return capabilities("traceai_xai") + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/python/frameworks/xai/traceai_xai/_wrappers.py b/python/frameworks/xai/traceai_xai/_wrappers.py index 5453706ae..222c47aa3 100644 --- a/python/frameworks/xai/traceai_xai/_wrappers.py +++ b/python/frameworks/xai/traceai_xai/_wrappers.py @@ -1,12 +1,14 @@ """Wrapper classes for xAI (Grok) instrumentation.""" import logging +import sys from abc import ABC from contextlib import contextmanager from typing import Any, Dict, Iterator, Tuple from fi_instrumentation import safe_json_dumps from fi_instrumentation.fi_types import FiSpanKindValues, SpanAttributes, MessageAttributes +from opentelemetry import context as context_api from opentelemetry import trace as trace_api from opentelemetry.trace import INVALID_SPAN, Span, Status, StatusCode from opentelemetry.util.types import AttributeValue @@ -16,14 +18,65 @@ # xAI-specific base URL for detection XAI_BASE_URL = "api.x.ai" +# Per-call capture policy key, shared by name (fi_instrumentation.attachments.media_capture). +_CAPTURE_POLICY_KEY = "traceai.auto.attachment_policy.v1" class _WithTracer(ABC): """Base class for wrappers that need a tracer.""" - def __init__(self, tracer: trace_api.Tracer, *args: Any, **kwargs: Any) -> None: + _media_dialect = "image" + + def __init__( + self, + tracer: trace_api.Tracer, + *args: Any, + attachments: Any = None, + config: Any = None, + **kwargs: Any, + ) -> None: super().__init__(*args, **kwargs) self._tracer = tracer + # Provider-owned shared attachment manager, or None (no capture, no imports). + self._attachments = attachments + self._config = config + + def _capture_chat_media(self, span: Any, messages: Any) -> None: + """Typed chat media (data-URI images) through the shared attachment manager; never raises.""" + if self._attachments is None: + return + try: + from fi_instrumentation.attachments.provider_media import ProviderMedia + + ProviderMedia( + self._attachments, span, "xai", trace_config=self._config + ).compatible_messages(messages, self._media_dialect) + except Exception: + logger.debug("Attachment capture unavailable", exc_info=True) + + def _defers(self) -> bool: + """Hand the whole call to traceai_openai only while both sides capture attachments. + + With attachments off here this binding traces exactly as before (its own span too). + With traceai_openai enabled but without a manager, this binding keeps the call and + captures its media itself. + """ + return self._attachments is not None and _shared_instrumentor_captures() + + @contextmanager + def _deferred(self) -> Iterator[None]: + """Run the handed-over call under this binding's local deny (deny wins downstream).""" + try: + from fi_instrumentation.attachments.provider_media import deferred + + token = context_api.attach(deferred(self._config)) + except Exception: + # An unreadable restriction is a restriction: deny capture for this call. + token = context_api.attach(context_api.set_value(_CAPTURE_POLICY_KEY, False)) + try: + yield + finally: + context_api.detach(token) @contextmanager def _start_span( @@ -59,6 +112,22 @@ def _is_xai_client(instance: Any) -> bool: return False +def _shared_instrumentor_captures() -> bool: + """True while traceai_openai's OpenAIInstrumentor is enabled WITH an active attachment manager. + + The request-level OpenAI instrumentor then owns the call (span and media), whatever + the setup order. Enabled without a manager, or an older traceai_openai without the + accessor, means it would capture nothing, so this binding keeps the call. Do not + import the optional package; inspect live state on each call. + """ + module = sys.modules.get("traceai_openai") + instrumentor = getattr(module, "OpenAIInstrumentor", None) + if instrumentor is None: + return False + captures = getattr(instrumentor(), "captures_attachments", None) + return bool(callable(captures) and captures()) + + def _get_input_attributes( model: str, messages: list, @@ -206,6 +275,10 @@ def __call__( # Check if this is an xAI client if not _is_xai_client(instance): return wrapped(*args, **kwargs) + # One request must have one owner, regardless of setup order. + if self._defers(): + with self._deferred(): + return wrapped(*args, **kwargs) # Extract parameters model = kwargs.get("model", args[0] if args else "unknown") @@ -217,6 +290,7 @@ def __call__( input_attrs = _get_input_attributes(model, messages, **extra_kwargs) with self._start_span("xAI Chat Completions", input_attrs) as span: + self._capture_chat_media(span, messages) try: response = wrapped(*args, **kwargs) @@ -277,6 +351,10 @@ async def __call__( # Check if this is an xAI client if not _is_xai_client(instance): return await wrapped(*args, **kwargs) + # One request must have one owner, regardless of setup order. + if self._defers(): + with self._deferred(): + return await wrapped(*args, **kwargs) # Extract parameters model = kwargs.get("model", args[0] if args else "unknown") @@ -288,6 +366,7 @@ async def __call__( input_attrs = _get_input_attributes(model, messages, **extra_kwargs) with self._start_span("xAI Chat Completions", input_attrs) as span: + self._capture_chat_media(span, messages) try: response = await wrapped(*args, **kwargs) diff --git a/python/tests/attachments_compat_providers/conftest.py b/python/tests/attachments_compat_providers/conftest.py new file mode 100644 index 000000000..67aef6d2e --- /dev/null +++ b/python/tests/attachments_compat_providers/conftest.py @@ -0,0 +1,215 @@ +"""Loopback harness for provider-media tests: synthetic data only. + +Reuses the F01 server, span exporter and media builders of +``tests/attachments_providers/conftest.py`` (loaded by path, so both suites +share one harness), and adds a plain-HTTP provider that answers like Gemini, +Vertex (REST), Bedrock Runtime, Ollama and OpenAI-compatible chat APIs, so the +REAL installed SDKs run end to end with outbound network disabled. +""" + +import base64 +import importlib.util +import json +import os +import sys +import threading +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from urllib.parse import unquote + +import pytest + +# LiteLLM fetches its model cost map at import unless told to use the bundled copy. +os.environ.setdefault("LITELLM_LOCAL_MODEL_COST_MAP", "True") +os.environ.setdefault("AWS_EC2_METADATA_DISABLED", "true") + +pytest.importorskip("fi_instrumentation.attachments.provider_media") + +_spec = importlib.util.spec_from_file_location( + "_attachment_provider_harness", Path(__file__).parent.parent / "attachments_providers" / "conftest.py") +harness = importlib.util.module_from_spec(_spec) +sys.modules["_attachment_provider_harness"] = harness +_spec.loader.exec_module(harness) + +# Fixtures (pytest finds them by name in this module) and helpers. +f01 = harness.f01 +spans = harness.spans +clean_env = harness.clean_env +png, wav, PDF, b64, refs, flush, wait_uploads = (harness.png, harness.wav, harness.PDF, harness.b64, harness.refs, + harness.flush, harness.wait_uploads) + + +def jpeg(seed=0) -> bytes: + """Leading bytes of a JPEG (SOI + APP0); the loopback F01 server does not decode it.""" + return b"\xff\xd8\xff\xe0\x00\x10JFIF\x00\x01\x01\x00\x00\x01\x00\x01\x00\x00" + bytes([seed % 256]) * 32 + b"\xff\xd9" + + +MP4 = b"\x00\x00\x00\x18ftypisom\x00\x00\x02\x00isomiso2" + b"\x00" * 24 + + +def data_uri(data: bytes, mime="image/png") -> str: + return "data:%s;base64,%s" % (mime, b64(data)) + + +def uploads_by_purpose(f01_server): + found = {} + for upload in f01_server.uploads: + found.setdefault(upload.get("purpose"), []).append(upload) + return found + + +def span_named(exporter, *names): + matches = [span for span in exporter.get_finished_spans() if span.name in names] + assert matches, [span.name for span in exporter.get_finished_spans()] + return matches[-1] + + +def all_refs(exporter): + found = [] + for span in exporter.get_finished_spans(): + for ref in refs(span): + found.append((span.name, ref)) + return found + + +class MediaProviderState: + def __init__(self): + self.requests = [] + self.image = png(seed=21) + self.generated = png(seed=22) + + +def _openai_chat(body): + if body.get("stream"): + chunks = [{"id": "c1", "object": "chat.completion.chunk", "created": 1, "model": "m", + "choices": [{"index": 0, "delta": {"role": "assistant", "content": "ok"}, "finish_reason": None}]}, + {"id": "c1", "object": "chat.completion.chunk", "created": 1, "model": "m", + "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}}] + return "text/event-stream", "".join("data: %s\n\n" % json.dumps(c) for c in chunks).encode() + b"data: [DONE]\n\n" + return "application/json", json.dumps({ + "id": "c1", "object": "chat.completion", "created": 1, "model": body.get("model", "m"), + "system_fingerprint": "fp_synthetic", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}}).encode() + + +def _gemini(state, stream): + response = {"candidates": [{"content": {"role": "model", "parts": [ + {"text": "ok"}, {"inlineData": {"mimeType": "image/png", "data": b64(state.generated)}}]}, + "finishReason": "STOP", "index": 0}], + "usageMetadata": {"promptTokenCount": 1, "candidatesTokenCount": 1, "totalTokenCount": 2}, + "modelVersion": "synthetic"} + if stream: + return "text/event-stream", ("data: %s\r\n\r\n" % json.dumps(response)).encode() + return "application/json", json.dumps(response).encode() + + +def _bedrock(state, path, body): + model = unquote(path.split("/model/", 1)[1].rsplit("/", 1)[0]) + if path.endswith("/converse"): + return "application/json", json.dumps({ + "output": {"message": {"role": "assistant", "content": [{"text": "ok"}]}}, "stopReason": "end_turn", + "usage": {"inputTokens": 1, "outputTokens": 1, "totalTokens": 2}, "metrics": {"latencyMs": 1}}).encode() + if "titan-image" in model or "nova-canvas" in model: + return "application/json", json.dumps({"images": [b64(state.generated)], "error": None}).encode() + if "anthropic" in model: + return "application/json", json.dumps({ + "id": "msg_1", "type": "message", "role": "assistant", "model": model, + "content": [{"type": "text", "text": "ok"}], "stop_reason": "end_turn", "stop_sequence": None, + "usage": {"input_tokens": 1, "output_tokens": 1}}).encode() + return "application/json", json.dumps({ + "output": {"message": {"role": "assistant", "content": [{"text": "ok"}]}}, "stopReason": "end_turn", + "usage": {"inputTokens": 1, "outputTokens": 1, "totalTokens": 2}}).encode() + + +def _ollama(path, body): + done = {"model": body.get("model", "m"), "created_at": "2025-01-01T00:00:00Z", "done": True, + "done_reason": "stop", "total_duration": 1, "load_duration": 1, "prompt_eval_count": 1, + "prompt_eval_duration": 1, "eval_count": 1, "eval_duration": 1} + if path.endswith("/api/chat"): + done["message"] = {"role": "assistant", "content": "ok"} + else: + done["response"] = "ok" + if body.get("stream", True): + return "application/x-ndjson", (json.dumps(done) + "\n").encode() + return "application/json", json.dumps(done).encode() + + +@pytest.fixture +def media_provider(): + state = MediaProviderState() + + class Handler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def log_message(self, *args): + pass + + def send(self, content_type, data, status=200): + self.send_response(status) + self.send_header("Content-Type", content_type) + self.send_header("Content-Length", str(len(data))) + self.end_headers() + self.wfile.write(data) + + def body(self): + if self.headers.get("Transfer-Encoding", "").lower() == "chunked": + parts = [] + while True: + size = int(self.rfile.readline().split(b";")[0].strip() or b"0", 16) + if not size: + self.rfile.readline() + return b"".join(parts) + parts.append(self.rfile.read(size)) + self.rfile.readline() + return self.rfile.read(int(self.headers.get("Content-Length") or 0)) + + def do_GET(self): + self.send("application/json", b"{}") + + def do_HEAD(self): + self.send_response(200) + self.send_header("Content-Length", "0") + self.end_headers() + + def do_POST(self): + raw = self.body() + try: + body = json.loads(raw) if raw else {} + except ValueError: + body = {} + state.requests.append((self.path, body, raw)) + path = self.path.split("?")[0] + if path.endswith(":generateContent"): + self.send(*_gemini(state, False)) + elif path.endswith(":streamGenerateContent"): + self.send(*_gemini(state, True)) + elif path.endswith(":predict"): + self.send("application/json", json.dumps({"predictions": [ + {"bytesBase64Encoded": b64(state.generated), "mimeType": "image/png"}], + "deployedModelId": "synthetic"}).encode()) + elif "/model/" in path and path.endswith(("/converse", "/invoke")): + self.send(*_bedrock(state, path, body)) + elif path.endswith(("/api/chat", "/api/generate")): + self.send(*_ollama(path, body)) + elif path.endswith("/chat/completions"): + self.send(*_openai_chat(body)) + elif path.endswith("/images/generations"): + self.send("application/json", json.dumps({"created": 1, "data": [ + {"b64_json": b64(state.generated)}, {"url": "https://images.invalid/x.png"}]}).encode()) + else: + self.send("application/json", b"{}", status=404) + + server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + state.url = "http://127.0.0.1:%d" % server.server_port + yield state + server.shutdown() + server.server_close() + thread.join(2) + + +def decoded(value: str) -> bytes: + return base64.b64decode(value) diff --git a/python/tests/attachments_compat_providers/test_bedrock_litellm_attachments.py b/python/tests/attachments_compat_providers/test_bedrock_litellm_attachments.py new file mode 100644 index 000000000..5eed71e1e --- /dev/null +++ b/python/tests/attachments_compat_providers/test_bedrock_litellm_attachments.py @@ -0,0 +1,338 @@ +"""Real boto3 (Bedrock Runtime) and LiteLLM -> traceAI instrumentors -> loopback F01.""" + +import asyncio +import base64 +import json + +import pytest + +from conftest import MP4, PDF, b64, data_uri, flush, png, refs, span_named, uploads_by_purpose # noqa: E402 +from fi_instrumentation import TraceConfig # noqa: E402 + +IMAGE = png(seed=51) +SHOT_1 = png(seed=52) +SHOT_2 = png(seed=53) +TOOL_IMAGE = png(seed=54) +CLAUDE = "us.anthropic.claude-sonnet-4-20250514-v1:0" +NOVA = "amazon.nova-lite-v1:0" + + +# ------------------------------------------------------------------ Bedrock +@pytest.fixture +def bedrock(spans, media_provider): + pytest.importorskip("boto3") + pytest.importorskip("traceai_bedrock") + from traceai_bedrock import BedrockInstrumentor + + tracer_provider, exporter = spans + state = {} + + def install(manager, config=None): + import boto3 + + BedrockInstrumentor().instrument(tracer_provider=tracer_provider, attachments=manager, + config=config or TraceConfig()) + state["on"] = True + # Created AFTER instrument(): the instrumentor wraps client creation. + client = boto3.client("bedrock-runtime", region_name="us-east-1", endpoint_url=media_provider.url, + aws_access_key_id="AKIDSYNTHETIC", aws_secret_access_key="synthetic") + return client, exporter + + yield install + if state.get("on"): + BedrockInstrumentor().uninstrument() + + +def converse_messages(computer=True): + return [ + {"role": "user", "content": [ + {"text": "open the page"}, + {"image": {"format": "png", "source": {"bytes": IMAGE}}}, + {"document": {"format": "pdf", "name": "brief", "source": {"bytes": PDF}}}, + {"image": {"format": "png", "source": {"s3Location": {"uri": "s3://bucket/never-read.png"}}}}, + ]}, + {"role": "assistant", "content": [ + {"toolUse": {"toolUseId": "tooluse_1", "name": "computer", "input": {"action": "screenshot"}}}, + {"toolUse": {"toolUseId": "tooluse_2", "name": "chart", "input": {}}}, + ]}, + {"role": "user", "content": [ + {"toolResult": {"toolUseId": "tooluse_1", "content": [ + {"image": {"format": "png", "source": {"bytes": SHOT_1}}}]}}, + {"toolResult": {"toolUseId": "tooluse_2", "content": [ + {"text": "chart"}, {"image": {"format": "png", "source": {"bytes": TOOL_IMAGE}}}]}}, + ]}, + ] + + +def computer_fields(): + return {"tools": [{"type": "computer_20250124", "name": "computer", "display_width_px": 1024, + "display_height_px": 768}], "anthropic_beta": ["computer-use-2025-01-24"]} + + +def test_converse_vision_document_and_computer_screenshots(f01, bedrock): + manager = f01.manager() + client, exporter = bedrock(manager) + client.converse(modelId=CLAUDE, messages=converse_messages(), + toolConfig={"tools": [{"toolSpec": {"name": "chart", "inputSchema": {"json": {"type": "object"}}}}]}, + additionalModelRequestFields=computer_fields()) + flush(manager) + assert sorted(f01.payloads) == sorted([IMAGE, PDF, SHOT_1, TOOL_IMAGE]) + by_purpose = uploads_by_purpose(f01) + assert [u["tool_call_id"] for u in by_purpose["computer_screenshot"]] == ["tooluse_1"] + assert [u["tool_call_id"] for u in by_purpose["tool_result"]] == ["tooluse_2"] + assert len(by_purpose["document"]) == 1 and len(by_purpose["model_input"]) == 1 + span = span_named(exporter, "bedrock.converse") + shots = [r for r in refs(span) if r["purpose"] == "computer_screenshot"] + assert [(r["message_index"], r["part_index"], r["role"]) for r in shots] == [(2, 0, "tool")] + manager.shutdown(5) + + +def test_converse_screenshot_needs_declared_computer_tool(f01, bedrock): + manager = f01.manager() + client, _ = bedrock(manager) + client.converse(modelId=CLAUDE, messages=converse_messages()) + flush(manager) + purposes = sorted(u["purpose"] for u in f01.uploads) + assert "computer_screenshot" not in purposes and purposes.count("tool_result") == 2 + manager.shutdown(5) + + +def test_converse_unknown_model_family_is_never_read(f01, bedrock): + manager = f01.manager() + client, _ = bedrock(manager) + client.converse(modelId="meta.llama3-2-90b-instruct-v1:0", messages=converse_messages()) + flush(manager) + assert f01.payloads == [] + manager.shutdown(5) + + +def test_nova_converse_video_block(f01, bedrock): + f01.state["policy"] = {"allowed_modalities": ["audio", "image", "document", "file", "video"]} + manager = f01.manager() + client, _ = bedrock(manager) + client.converse(modelId=NOVA, messages=[{"role": "user", "content": [ + {"video": {"format": "mp4", "source": {"bytes": MP4}}}, {"text": "summarise"}]}]) + client.converse(modelId=CLAUDE, messages=[{"role": "user", "content": [ + {"video": {"format": "mp4", "source": {"bytes": MP4}}}, {"text": "summarise"}]}]) + flush(manager) + assert f01.payloads == [MP4] # video is qualified for Nova only + manager.shutdown(5) + + +def test_converse_hide_input_images_keeps_documents(f01, bedrock): + manager = f01.manager() + client, _ = bedrock(manager, TraceConfig(hide_input_images=True)) + client.converse(modelId=CLAUDE, messages=converse_messages(), additionalModelRequestFields=computer_fields()) + flush(manager) + assert f01.payloads == [PDF] + manager.shutdown(5) + + +def test_invoke_model_anthropic_body(f01, bedrock): + manager = f01.manager() + client, exporter = bedrock(manager) + body = {"anthropic_version": "bedrock-2023-05-31", "max_tokens": 8, + "tools": [{"type": "computer_20250124", "name": "computer", "display_width_px": 800, + "display_height_px": 600}], + "messages": [ + {"role": "user", "content": [ + {"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": b64(IMAGE)}}]}, + {"role": "assistant", "content": [{"type": "tool_use", "id": "toolu_1", "name": "computer", + "input": {"action": "screenshot"}}]}, + {"role": "user", "content": [ + {"type": "text", "text": "result follows"}, + {"type": "tool_result", "tool_use_id": "toolu_1", "content": [ + {"type": "text", "text": "screenshot taken"}, + {"type": "text", "text": "below"}, + {"type": "image", "source": {"type": "base64", "media_type": "image/png", + "data": b64(SHOT_2)}}]}]}, + ]} + response = client.invoke_model(modelId="anthropic.claude-3-5-sonnet-20240620-v1:0", body=json.dumps(body)) + assert json.loads(response["body"].read())["content"][0]["text"] == "ok" + flush(manager) + assert sorted(f01.payloads) == sorted([IMAGE, SHOT_2]) + assert sorted(u["purpose"] for u in f01.uploads) == ["computer_screenshot", "model_input"] + # As for Converse, a tool_result child carries the tool_result block's index (1), not its own (2). + shot = [r for s in exporter.get_finished_spans() for r in refs(s) if r.get("purpose") == "computer_screenshot"] + assert [(r["message_index"], r["part_index"]) for r in shot] == [(2, 1)] + manager.shutdown(5) + + +def test_invoke_model_nova_body(f01, bedrock): + manager = f01.manager() + client, _ = bedrock(manager) + body = {"schemaVersion": "messages-v1", "messages": [{"role": "user", "content": [ + {"image": {"format": "png", "source": {"bytes": b64(IMAGE)}}}, + {"document": {"format": "pdf", "name": "brief", "source": {"bytes": b64(PDF)}}}, + {"text": "describe"}]}]} + client.invoke_model(modelId=NOVA, body=json.dumps(body)) + flush(manager) + assert sorted(f01.payloads) == sorted([IMAGE, PDF]) + manager.shutdown(5) + + +@pytest.mark.parametrize("model", ["amazon.titan-image-generator-v2:0", "amazon.nova-canvas-v1:0"]) +def test_invoke_model_image_generation_both_directions(f01, bedrock, media_provider, model): + manager = f01.manager() + client, exporter = bedrock(manager) + body = {"taskType": "INPAINTING", "inPaintingParams": {"image": b64(IMAGE), "text": "add a hat", + "maskPrompt": "head"}, + "imageGenerationConfig": {"numberOfImages": 1}} + response = client.invoke_model(modelId=model, body=json.dumps(body)) + # The output came from the body the instrumentor already buffered; the caller still reads it whole. + assert json.loads(response["body"].read())["images"] == [b64(media_provider.generated)] + flush(manager) + assert sorted(f01.payloads) == sorted([IMAGE, media_provider.generated]) + directions = sorted((u["direction"], u["purpose"]) for u in f01.uploads) + assert directions == [("input", "model_input"), ("output", "model_output")] + span = span_named(exporter, "bedrock.invoke_model") + assert len(refs(span)) == 2 + manager.shutdown(5) + + +def test_invoke_model_unknown_body_is_never_scraped(f01, bedrock): + manager = f01.manager() + client, _ = bedrock(manager) + body = {"prompt": "x", "images": [b64(IMAGE)], "messages": [{"role": "user", "content": [ + {"image": {"format": "png", "source": {"bytes": b64(IMAGE)}}}]}]} + client.invoke_model(modelId="meta.llama3-2-90b-instruct-v1:0", body=json.dumps(body)) + flush(manager) + assert f01.payloads == [] + manager.shutdown(5) + + +# ------------------------------------------------------------------ LiteLLM +@pytest.fixture +def litellm_instrument(spans): + pytest.importorskip("litellm") + pytest.importorskip("traceai_litellm") + from traceai_litellm import LiteLLMInstrumentor + + tracer_provider, exporter = spans + state = {} + + def install(manager, config=None, with_openai=False): + LiteLLMInstrumentor().instrument(tracer_provider=tracer_provider, attachments=manager, + config=config or TraceConfig()) + state["litellm"] = True + if with_openai: + from traceai_openai import OpenAIInstrumentor + + OpenAIInstrumentor().instrument(tracer_provider=tracer_provider, attachments=manager) + state["openai"] = True + return exporter + + yield install + if state.get("openai"): + from traceai_openai import OpenAIInstrumentor + + OpenAIInstrumentor().uninstrument() + if state.get("litellm"): + LiteLLMInstrumentor().uninstrument() + + +def chat_messages(): + return [{"role": "user", "content": [ + {"type": "text", "text": "what is this"}, + {"type": "image_url", "image_url": {"url": data_uri(IMAGE)}}, + {"type": "image_url", "image_url": {"url": "https://example.invalid/x.png"}}, + ]}] + + +@pytest.mark.parametrize("mode", ["sync", "async", "stream"]) +def test_litellm_completion_input_media(f01, media_provider, litellm_instrument, mode): + import litellm + + manager = f01.manager() + exporter = litellm_instrument(manager) + options = dict(model="openai/gpt-synthetic", messages=chat_messages(), api_base=media_provider.url, + api_key="sk-synthetic") + if mode == "async": + asyncio.run(litellm.acompletion(**options)) + elif mode == "stream": + for _ in litellm.completion(stream=True, **options): + pass + else: + litellm.completion(**options) + flush(manager) + assert f01.payloads == [IMAGE] + span = span_named(exporter, "completion", "acompletion") + assert [r["purpose"] for r in refs(span)] == ["model_input"] + manager.shutdown(5) + + +def test_litellm_owns_media_under_nested_openai_instrumentor(f01, media_provider, litellm_instrument): + """LiteLLM marks the call as owned, so the nested OpenAI instrumentor uploads nothing more.""" + pytest.importorskip("traceai_openai") + import litellm + + manager = f01.manager() + exporter = litellm_instrument(manager, with_openai=True) + litellm.completion(model="openai/gpt-synthetic", messages=chat_messages(), api_base=media_provider.url, + api_key="sk-synthetic") + flush(manager) + assert f01.payloads == [IMAGE] + owners = [name for name, _ in ((s.name, r) for s in exporter.get_finished_spans() for r in refs(s))] + assert owners == ["completion"], owners + manager.shutdown(5) + + +def test_litellm_pdf_file_part_uploads_once_under_nested_openai(f01, media_provider, litellm_instrument): + """LiteLLM claims the call, so it captures PDF ``file`` parts itself: exactly one upload, on its span.""" + pytest.importorskip("traceai_openai") + import litellm + + manager = f01.manager() + exporter = litellm_instrument(manager, TraceConfig(hide_input_images=True), with_openai=True) + content = [{"type": "text", "text": "summarize"}, + {"type": "file", "file": {"filename": "a.pdf", "file_data": data_uri(PDF, "application/pdf")}}] + try: + litellm.completion(model="openai/gpt-synthetic", messages=[{"role": "user", "content": content}], + api_base=media_provider.url, api_key="sk-synthetic") + except Exception as error: + # Older LiteLLM releases (1.53.x) reject chat `file` parts before any request is sent. + text = str(error).lower() + if "invalid content type" not in text and "invalid user message" not in text: + raise + manager.shutdown(5) + pytest.skip("this LiteLLM release rejects chat file parts") + flush(manager) + assert f01.payloads == [PDF] + owners = [(s.name, r["purpose"]) for s in exporter.get_finished_spans() for r in refs(s)] + assert owners == [("completion", "document")], owners + sent = [raw for path, _, raw in media_provider.requests if path.endswith("/chat/completions")] + assert sent and b"file_data" in sent[-1] # the SDK request itself is unchanged + manager.shutdown(5) + + +@pytest.mark.parametrize("async_", [False, True]) +def test_litellm_image_generation_output(f01, media_provider, litellm_instrument, async_): + import litellm + + manager = f01.manager() + exporter = litellm_instrument(manager) + # gpt-image-1 always answers with b64_json; the loopback also returns a URL result, which is never fetched. + options = dict(model="openai/gpt-image-1", prompt="a cat", api_base=media_provider.url, api_key="sk-synthetic") + if async_: + result = asyncio.run(litellm.aimage_generation(**options)) + else: + result = litellm.image_generation(**options) + assert base64.b64decode(result.data[0].b64_json) == media_provider.generated + flush(manager) + assert f01.payloads == [media_provider.generated] # the URL result is never fetched + assert f01.uploads[0]["direction"] == "output" + span = span_named(exporter, "image_generation", "aimage_generation") + assert len(refs(span)) == 1 + manager.shutdown(5) + + +def test_litellm_hide_input_images(f01, media_provider, litellm_instrument): + import litellm + + manager = f01.manager() + litellm_instrument(manager, TraceConfig(hide_input_images=True)) + litellm.completion(model="openai/gpt-synthetic", messages=chat_messages(), api_base=media_provider.url, + api_key="sk-synthetic") + flush(manager) + assert f01.payloads == [] + manager.shutdown(5) diff --git a/python/tests/attachments_compat_providers/test_compat_attachments.py b/python/tests/attachments_compat_providers/test_compat_attachments.py new file mode 100644 index 000000000..6ad1c0726 --- /dev/null +++ b/python/tests/attachments_compat_providers/test_compat_attachments.py @@ -0,0 +1,572 @@ +"""Real compatible-chat SDKs -> thin traceAI bindings -> loopback F01. + +One typed fixture per wrapper plus a capability-negative row; DeepSeek and +Cerebras have no qualified media surface and must upload nothing. +""" + +import asyncio + +import pytest + +from conftest import b64, data_uri, flush, png, refs, wav # noqa: E402 +from fi_instrumentation import TraceConfig # noqa: E402 + +IMAGE = png(seed=61) + + +def image_messages(url_form="object"): + image = data_uri(IMAGE) + return [{"role": "user", "content": [ + {"type": "text", "text": "what is this"}, + {"type": "image_url", "image_url": image if url_form == "string" else {"url": image}}, + {"type": "image_url", "image_url": {"url": "https://example.invalid/x.png"}}, + ]}] + + +def audio_messages(sound): + return [{"role": "user", "content": [ + {"type": "input_audio", "input_audio": {"data": b64(sound), "format": "wav"}}]}] + + +@pytest.fixture +def instrumented(spans): + tracer_provider, exporter = spans + installed = [] + + def install(instrumentor, manager, config=None): + instrumentor.instrument(tracer_provider=tracer_provider, attachments=manager, config=config or TraceConfig()) + installed.append(instrumentor) + return exporter + + yield install + for instrumentor in installed: + instrumentor.uninstrument() + + +def media_refs(exporter): + return [r for s in exporter.get_finished_spans() for r in refs(s)] + + +# ------------------------------------------------------------------ groq (image-only chat) +def test_groq_chat_images(f01, media_provider, instrumented): + groq = pytest.importorskip("groq") + pytest.importorskip("traceai_groq") + from traceai_groq import GroqInstrumentor + + manager = f01.manager() + exporter = instrumented(GroqInstrumentor(), manager) + client = groq.Groq(api_key="gsk-synthetic", base_url=media_provider.url, max_retries=0) + client.chat.completions.create(model="m", messages=image_messages()) + client.chat.completions.create(model="m", messages=audio_messages(wav())) # Groq chat: no audio parts + asyncio.run(groq.AsyncGroq(api_key="gsk-synthetic", base_url=media_provider.url, max_retries=0) + .chat.completions.create(model="m", messages=image_messages())) + flush(manager) + assert f01.payloads == [IMAGE, IMAGE] + assert [r["purpose"] for r in media_refs(exporter)] == ["model_input", "model_input"] + manager.shutdown(5) + + +# ------------------------------------------------------------------ together +def test_together_chat_is_not_claimed(f01, media_provider, instrumented): + """traceai_together does not trace chat today, so it must not claim (or upload) chat media.""" + together = pytest.importorskip("together") + pytest.importorskip("traceai_together") + import traceai_together + from traceai_together import TogetherInstrumentor + + manager = f01.manager() + exporter = instrumented(TogetherInstrumentor(), manager) + client = together.Together(api_key="synthetic", base_url=media_provider.url + "/v1") + client.chat.completions.create(model="m", messages=image_messages()) + flush(manager) + assert f01.payloads == [] and media_refs(exporter) == [] + descriptor = traceai_together.together_attachment_capabilities + assert descriptor["operations"] == () and "chat.completions.create" in descriptor["unsupported"] + manager.shutdown(5) + + +# ------------------------------------------------------------------ OpenAI-SDK hosted routes +@pytest.mark.parametrize("package", ["fireworks", "xai"]) +def test_openai_sdk_routes_capture_images(f01, media_provider, instrumented, monkeypatch, package): + openai = pytest.importorskip("openai") + pytest.importorskip("traceai_" + package) + if package == "fireworks": + from traceai_fireworks import FireworksInstrumentor as Instrumentor + from traceai_fireworks import _wrappers + monkeypatch.setattr(_wrappers, "_is_fireworks_client", lambda instance: True) + if hasattr(_wrappers, "_shared_instrumentor_enabled"): + monkeypatch.setattr(_wrappers, "_shared_instrumentor_enabled", lambda: False) + else: + from traceai_xai import XAIInstrumentor as Instrumentor + from traceai_xai import _wrappers + monkeypatch.setattr(_wrappers, "_is_xai_client", lambda instance: True) + monkeypatch.setattr(_wrappers, "_shared_instrumentor_captures", lambda: False) + + manager = f01.manager() + exporter = instrumented(Instrumentor(), manager) + client = openai.OpenAI(api_key="sk-synthetic", base_url=media_provider.url + "/v1", max_retries=0) + client.chat.completions.create(model="m", messages=image_messages()) + client.chat.completions.create(model="m", messages=audio_messages(wav())) # image-only routes + flush(manager) + assert f01.payloads == [IMAGE] + assert len(media_refs(exporter)) == 1 + manager.shutdown(5) + + +def media_owners(exporter): + return [(s.name, r["purpose"]) for s in exporter.get_finished_spans() for r in refs(s)] + + +@pytest.mark.parametrize("order", ["openai_first", "binding_first"]) +@pytest.mark.parametrize("package", ["xai", "minimax"]) +def test_binding_with_openai_instrumentor_uploads_once(f01, media_provider, instrumented, monkeypatch, package, + order): + """The OpenAI instrumentor wraps the same SDK call; one image is uploaded once, whatever the order.""" + openai = pytest.importorskip("openai") + pytest.importorskip("traceai_openai") + pytest.importorskip("traceai_" + package) + from traceai_openai import OpenAIInstrumentor + + if package == "xai": + from traceai_xai import XAIInstrumentor as Instrumentor + from traceai_xai import _wrappers + monkeypatch.setattr(_wrappers, "_is_xai_client", lambda instance: True) + else: + from traceai_minimax import MiniMaxInstrumentor as Instrumentor + from traceai_minimax import _wrappers + monkeypatch.setattr(_wrappers, "is_minimax_client", lambda instance: True) + + manager = f01.manager() + pair = [OpenAIInstrumentor(), Instrumentor()] + if order == "binding_first": + pair.reverse() + for instrumentor in pair: + exporter = instrumented(instrumentor, manager) + client = openai.OpenAI(api_key="sk-synthetic", base_url=media_provider.url + "/v1", max_retries=0) + client.chat.completions.create(model="m", messages=image_messages()) + flush(manager) + assert f01.payloads == [IMAGE] + owners = media_owners(exporter) + assert len(owners) == 1 and owners[0][1] == "model_input", owners + manager.shutdown(5) + + +def _compat_binding(package, monkeypatch): + """The binding instrumentor (client check forced on for the loopback URL) and its own span name.""" + if package == "xai": + from traceai_xai import XAIInstrumentor + from traceai_xai import _wrappers + monkeypatch.setattr(_wrappers, "_is_xai_client", lambda instance: True) + return XAIInstrumentor(), "xAI Chat Completions" + from traceai_minimax import MiniMaxInstrumentor + from traceai_minimax import _wrappers + monkeypatch.setattr(_wrappers, "is_minimax_client", lambda instance: True) + return MiniMaxInstrumentor(), "minimax.chat.completions" + + +def _openai_clients(openai, media_provider): + options = dict(api_key="sk-synthetic", base_url=media_provider.url + "/v1", max_retries=0) + return openai.OpenAI(**options), openai.AsyncOpenAI(**options) + + +@pytest.mark.parametrize("order", ["openai_first", "binding_first"]) +@pytest.mark.parametrize("package", ["xai", "minimax"]) +def test_binding_without_attachments_keeps_its_span(media_provider, instrumented, monkeypatch, package, order): + """Attachments off: tracing is unchanged, so the binding span and the OpenAI span both stay.""" + openai = pytest.importorskip("openai") + pytest.importorskip("traceai_openai") + pytest.importorskip("traceai_" + package) + from traceai_openai import OpenAIInstrumentor + + binding, span_name = _compat_binding(package, monkeypatch) + pair = [OpenAIInstrumentor(), binding] + if order == "binding_first": + pair.reverse() + for instrumentor in pair: + exporter = instrumented(instrumentor, None) + client, async_client = _openai_clients(openai, media_provider) + client.chat.completions.create(model="m", messages=[{"role": "user", "content": "hi"}]) + asyncio.run(async_client.chat.completions.create(model="m", messages=[{"role": "user", "content": "hi"}])) + names = sorted(span.name for span in exporter.get_finished_spans()) + assert names == sorted([span_name, "ChatCompletion"] * 2), names + + +@pytest.mark.parametrize("order", ["openai_first", "binding_first"]) +@pytest.mark.parametrize("package", ["xai", "minimax"]) +def test_binding_keeps_call_when_openai_has_no_manager(f01, media_provider, instrumented, monkeypatch, package, + order): + """Mixed setup: binding WITH a manager, traceai_openai enabled WITHOUT one. The binding keeps the call.""" + openai = pytest.importorskip("openai") + pytest.importorskip("traceai_openai") + pytest.importorskip("traceai_" + package) + from traceai_openai import OpenAIInstrumentor + + manager = f01.manager() + binding, span_name = _compat_binding(package, monkeypatch) + pair = [(OpenAIInstrumentor(), None), (binding, manager)] + if order == "binding_first": + pair.reverse() + for instrumentor, attachments in pair: + exporter = instrumented(instrumentor, attachments) + assert not OpenAIInstrumentor().captures_attachments() + client, async_client = _openai_clients(openai, media_provider) + client.chat.completions.create(model="m", messages=image_messages()) + asyncio.run(async_client.chat.completions.create(model="m", messages=image_messages())) + flush(manager) + assert f01.payloads == [IMAGE, IMAGE] + assert media_owners(exporter) == [(span_name, "model_input")] * 2 + names = sorted(span.name for span in exporter.get_finished_spans()) + assert names == sorted([span_name, "ChatCompletion"] * 2), names + manager.shutdown(5) + + +@pytest.mark.parametrize("order", ["openai_first", "binding_first"]) +@pytest.mark.parametrize("package", ["xai", "minimax"]) +def test_binding_without_manager_keeps_its_span_next_to_capturing_openai(f01, media_provider, instrumented, + monkeypatch, package, order): + """Mixed setup: traceai_openai WITH a manager, binding WITHOUT one. Binding tracing is unchanged.""" + openai = pytest.importorskip("openai") + pytest.importorskip("traceai_openai") + pytest.importorskip("traceai_" + package) + from traceai_openai import OpenAIInstrumentor + + manager = f01.manager() + binding, span_name = _compat_binding(package, monkeypatch) + pair = [(OpenAIInstrumentor(), manager), (binding, None)] + if order == "binding_first": + pair.reverse() + for instrumentor, attachments in pair: + exporter = instrumented(instrumentor, attachments) + assert OpenAIInstrumentor().captures_attachments() + client, _ = _openai_clients(openai, media_provider) + client.chat.completions.create(model="m", messages=image_messages()) + flush(manager) + assert f01.payloads == [IMAGE] + assert media_owners(exporter) == [("ChatCompletion", "model_input")] + names = sorted(span.name for span in exporter.get_finished_spans()) + assert names == sorted([span_name, "ChatCompletion"]), names + manager.shutdown(5) + + +@pytest.mark.parametrize("package", ["xai", "minimax"]) +def test_older_openai_without_accessor_never_takes_the_call(monkeypatch, package): + """An enabled traceai_openai that predates captures_attachments() cannot capture: do not defer to it.""" + import sys + import types + + pytest.importorskip("traceai_" + package) + if package == "xai": + from traceai_xai import _wrappers + else: + from traceai_minimax import _wrappers + + class LegacyOpenAIInstrumentor: + is_instrumented_by_opentelemetry = True + + monkeypatch.setitem(sys.modules, "traceai_openai", + types.SimpleNamespace(OpenAIInstrumentor=LegacyOpenAIInstrumentor)) + assert _wrappers._shared_instrumentor_captures() is False + monkeypatch.delitem(sys.modules, "traceai_openai") + assert _wrappers._shared_instrumentor_captures() is False + + +@pytest.mark.parametrize("flag", [None, "hide_input_images", "hide_inputs", "pii_redaction"]) +@pytest.mark.parametrize("package", ["xai", "minimax"]) +def test_deferred_call_keeps_binding_local_deny(f01, media_provider, instrumented, monkeypatch, package, flag): + """Attachments on: the whole call goes to traceai_openai (default config), under the binding's deny.""" + openai = pytest.importorskip("openai") + pytest.importorskip("traceai_openai") + pytest.importorskip("traceai_" + package) + from traceai_openai import OpenAIInstrumentor + + manager = f01.manager() + exporter = instrumented(OpenAIInstrumentor(), manager) + binding, _ = _compat_binding(package, monkeypatch) + instrumented(binding, manager, TraceConfig(**({flag: True} if flag else {}))) + client, async_client = _openai_clients(openai, media_provider) + client.chat.completions.create(model="m", messages=image_messages()) + asyncio.run(async_client.chat.completions.create(model="m", messages=image_messages())) + flush(manager) + assert f01.payloads == ([] if flag else [IMAGE, IMAGE]) + assert [span.name for span in exporter.get_finished_spans()] == ["ChatCompletion"] * 2 + manager.shutdown(5) + + +@pytest.mark.parametrize("package", ["xai", "minimax"]) +def test_deferred_call_keeps_context_policy_and_owner(f01, media_provider, instrumented, monkeypatch, package): + """A per-call deny and an outer owner mark already in the context survive the deferral.""" + openai = pytest.importorskip("openai") + pytest.importorskip("traceai_openai") + pytest.importorskip("traceai_" + package) + from opentelemetry import context as otel_context + from fi_instrumentation.attachments import CapturePolicy + from fi_instrumentation.attachments import media_capture + from traceai_openai import OpenAIInstrumentor + + manager = f01.manager() + binding, _ = _compat_binding(package, monkeypatch) + instrumented(binding, manager) + instrumented(OpenAIInstrumentor(), manager) + client, _ = _openai_clients(openai, media_provider) + for key, value in ((media_capture.CAPTURE_POLICY_KEY, CapturePolicy(record_inputs=False)), + (media_capture.OWNER_KEY, True)): + token = otel_context.attach(otel_context.set_value(key, value)) + try: + client.chat.completions.create(model="m", messages=image_messages()) + finally: + otel_context.detach(token) + flush(manager) + assert f01.payloads == [] + client.chat.completions.create(model="m", messages=image_messages()) # control + flush(manager) + assert f01.payloads == [IMAGE] + manager.shutdown(5) + + +def test_minimax_chat_images(f01, media_provider, instrumented, monkeypatch): + openai = pytest.importorskip("openai") + pytest.importorskip("traceai_minimax") + from traceai_minimax import MiniMaxInstrumentor + from traceai_minimax import _wrappers + + monkeypatch.setattr(_wrappers, "is_minimax_client", lambda instance: True) + manager = f01.manager() + exporter = instrumented(MiniMaxInstrumentor(), manager) + client = openai.OpenAI(api_key="sk-synthetic", base_url=media_provider.url + "/v1", max_retries=0) + client.chat.completions.create(model="MiniMax-M3", messages=image_messages()) + client.chat.completions.create(model="MiniMax-M3", messages=audio_messages(wav())) # no audio input + asyncio.run(openai.AsyncOpenAI(api_key="sk-synthetic", base_url=media_provider.url + "/v1", max_retries=0) + .chat.completions.create(model="MiniMax-M3", messages=image_messages())) + flush(manager) + assert f01.payloads == [IMAGE, IMAGE] + assert media_owners(exporter) == [("minimax.chat.completions", "model_input")] * 2 + manager.shutdown(5) + + +def test_minimax_hide_input_images(f01, media_provider, instrumented, monkeypatch): + openai = pytest.importorskip("openai") + pytest.importorskip("traceai_minimax") + from traceai_minimax import MiniMaxInstrumentor + from traceai_minimax import _wrappers + + monkeypatch.setattr(_wrappers, "is_minimax_client", lambda instance: True) + manager = f01.manager() + instrumented(MiniMaxInstrumentor(), manager, TraceConfig(hide_input_images=True)) + client = openai.OpenAI(api_key="sk-synthetic", base_url=media_provider.url + "/v1", max_retries=0) + client.chat.completions.create(model="MiniMax-M3", messages=image_messages()) + flush(manager) + assert f01.payloads == [] + manager.shutdown(5) + + +# ------------------------------------------------------------------ vllm (legacy OpenAI-dialect chat binding) +def test_vllm_chat_images_and_audio(f01, media_provider, instrumented): + openai = pytest.importorskip("openai") + pytest.importorskip("traceai_vllm") + from traceai_vllm import VLLMInstrumentor + + manager = f01.manager() + sound = wav(seed=9) + exporter = instrumented(VLLMInstrumentor(vllm_base_urls=[media_provider.url]), manager) + client = openai.OpenAI(api_key="EMPTY", base_url=media_provider.url + "/v1", max_retries=0) + client.chat.completions.create(model="m", messages=image_messages() + audio_messages(sound)) + asyncio.run(openai.AsyncOpenAI(api_key="EMPTY", base_url=media_provider.url + "/v1", max_retries=0) + .chat.completions.create(model="m", messages=image_messages())) + flush(manager) + assert sorted(f01.payloads) == sorted([IMAGE, sound, IMAGE]) + assert sorted(media_owners(exporter)) == [("vLLM Chat Completions", "model_input")] * 3 + manager.shutdown(5) + + +def test_vllm_defers_to_openai_instrumentor(f01, media_provider, instrumented): + """VLLMInstrumentor already defers the whole call to traceai_openai while it is enabled: one upload.""" + openai = pytest.importorskip("openai") + pytest.importorskip("traceai_openai") + pytest.importorskip("traceai_vllm") + from traceai_openai import OpenAIInstrumentor + from traceai_vllm import VLLMInstrumentor + + manager = f01.manager() + instrumented(VLLMInstrumentor(vllm_base_urls=[media_provider.url]), manager) + exporter = instrumented(OpenAIInstrumentor(), manager) + client = openai.OpenAI(api_key="EMPTY", base_url=media_provider.url + "/v1", max_retries=0) + client.chat.completions.create(model="m", messages=image_messages()) + flush(manager) + assert f01.payloads == [IMAGE] + owners = media_owners(exporter) + assert len(owners) == 1 and owners[0][0] != "vLLM Chat Completions", owners + manager.shutdown(5) + + +@pytest.mark.parametrize("package", ["deepseek", "cerebras"]) +def test_routes_without_media_surface_capture_nothing(f01, media_provider, instrumented, monkeypatch, package): + pytest.importorskip("traceai_" + package) + if package == "deepseek": + openai = pytest.importorskip("openai") + import traceai_deepseek + from traceai_deepseek import DeepSeekInstrumentor as Instrumentor + + for name in ("_is_deepseek_client", "_is_deepseek"): + if hasattr(traceai_deepseek._wrappers, name): + monkeypatch.setattr(traceai_deepseek._wrappers, name, lambda *a, **k: True) + manager = f01.manager() + exporter = instrumented(Instrumentor(), manager) + client = openai.OpenAI(api_key="sk-synthetic", base_url=media_provider.url + "/v1", max_retries=0) + client.chat.completions.create(model="deepseek-chat", messages=image_messages()) + descriptor = traceai_deepseek.deepseek_attachment_capabilities + else: + sdk = pytest.importorskip("cerebras.cloud.sdk") + import traceai_cerebras + from traceai_cerebras import CerebrasInstrumentor as Instrumentor + + manager = f01.manager() + exporter = instrumented(Instrumentor(), manager) + try: + client = sdk.Cerebras(api_key="synthetic", base_url=media_provider.url, max_retries=0, + warm_tcp_connection=False) + except TypeError: # cerebras-cloud-sdk 1.0.0 has no warm_tcp_connection option + client = sdk.Cerebras(api_key="synthetic", base_url=media_provider.url, max_retries=0) + client.chat.completions.create(model="llama", messages=image_messages()) + descriptor = traceai_cerebras.cerebras_attachment_capabilities + flush(manager) + assert f01.payloads == [] and media_refs(exporter) == [] + assert descriptor["operations"] == () and descriptor["note"] + manager.shutdown(5) + + +# ------------------------------------------------------------------ mistralai (legacy 1.x and native 3.6+) +@pytest.mark.parametrize("async_", [False, True]) +def test_mistral_chat_images(f01, media_provider, instrumented, async_): + mistralai = pytest.importorskip("mistralai") + pytest.importorskip("traceai_mistralai") + from traceai_mistralai import MistralAIInstrumentor + + manager = f01.manager() + exporter = instrumented(MistralAIInstrumentor(), manager) + Mistral = getattr(mistralai, "Mistral", None) + if Mistral is None: + from mistralai.client import Mistral + client = Mistral(api_key="synthetic", server_url=media_provider.url) + messages = image_messages("string") + [{"role": "user", "content": [ + {"type": "image_url", "image_url": {"url": data_uri(png(seed=62))}}]}] + if async_: + asyncio.run(client.chat.complete_async(model="pixtral-synthetic", messages=messages)) + else: + client.chat.complete(model="pixtral-synthetic", messages=messages) + flush(manager) + assert sorted(f01.payloads) == sorted([IMAGE, png(seed=62)]) + assert len(media_refs(exporter)) == 2 + manager.shutdown(5) + + +def test_mistral_stream_and_hide_input_images(f01, media_provider, instrumented): + mistralai = pytest.importorskip("mistralai") + pytest.importorskip("traceai_mistralai") + from traceai_mistralai import MistralAIInstrumentor + + manager = f01.manager() + instrumented(MistralAIInstrumentor(), manager, TraceConfig(hide_input_images=True)) + Mistral = getattr(mistralai, "Mistral", None) + if Mistral is None: + from mistralai.client import Mistral + client = Mistral(api_key="synthetic", server_url=media_provider.url) + client.chat.complete(model="pixtral-synthetic", messages=image_messages("string")) + flush(manager) + assert f01.payloads == [] + manager.shutdown(5) + + +# ------------------------------------------------------------------ openrouter +def test_openrouter_chat_images_and_audio(f01, media_provider, instrumented): + openrouter = pytest.importorskip("openrouter") + pytest.importorskip("traceai_openrouter") + from traceai_openrouter import OpenRouterInstrumentor + + manager = f01.manager() + exporter = instrumented(OpenRouterInstrumentor(), manager) + sound = wav(seed=8) + with openrouter.OpenRouter(api_key="sk-or-synthetic", server_url=media_provider.url) as client: + client.chat.send(model="m", messages=image_messages() + audio_messages(sound)) + flush(manager) + assert sorted(f01.payloads) == sorted([IMAGE, sound]) + assert len(media_refs(exporter)) == 2 + manager.shutdown(5) + + +# ------------------------------------------------------------------ portkey +def test_portkey_chat_images(f01, media_provider, instrumented): + portkey_ai = pytest.importorskip("portkey_ai") + pytest.importorskip("traceai_portkey") + from traceai_portkey import PortkeyInstrumentor + + manager = f01.manager() + exporter = instrumented(PortkeyInstrumentor(), manager) + client = portkey_ai.Portkey(api_key="synthetic", base_url=media_provider.url + "/v1", provider="openai", + Authorization="Bearer sk-synthetic") + client.chat.completions.create(model="m", messages=image_messages()) + flush(manager) + assert f01.payloads == [IMAGE] + assert len(media_refs(exporter)) == 1 + manager.shutdown(5) + + +# ------------------------------------------------------------------ ollama +def test_ollama_chat_and_generate_images(f01, media_provider, instrumented, tmp_path): + ollama = pytest.importorskip("ollama") + pytest.importorskip("traceai_ollama") + from traceai_ollama import OllamaInstrumentor + + manager = f01.manager() + exporter = instrumented(OllamaInstrumentor(), manager) + second = png(seed=63) + path = tmp_path / "local.png" + path.write_bytes(png(seed=64)) + client = ollama.Client(host=media_provider.url) + client.chat(model="llava", messages=[{"role": "user", "content": "describe", "images": [IMAGE]}], stream=False) + client.generate(model="llava", prompt="describe", images=[b64(second)], stream=False) + # A local path: the SDK reads it for its own request; the adapter never opens it or uploads it. + client.generate(model="llava", prompt="describe", images=[str(path)], stream=False) + flush(manager) + assert sorted(f01.payloads) == sorted([IMAGE, second]) + assert len(media_refs(exporter)) == 2 + manager.shutdown(5) + + +def test_ollama_async_chat(f01, media_provider, instrumented): + ollama = pytest.importorskip("ollama") + pytest.importorskip("traceai_ollama") + from traceai_ollama import OllamaInstrumentor + + manager = f01.manager() + instrumented(OllamaInstrumentor(), manager) + asyncio.run(ollama.AsyncClient(host=media_provider.url).chat( + model="llava", messages=[{"role": "user", "content": "describe", "images": [IMAGE]}], stream=False)) + flush(manager) + assert f01.payloads == [IMAGE] + manager.shutdown(5) + + +# ------------------------------------------------------------------ descriptors exported per package +@pytest.mark.parametrize("module,name", [ + ("traceai_google_genai", "google_genai_attachment_capabilities"), + ("traceai_vertexai", "vertexai_attachment_capabilities"), + ("traceai_bedrock", "bedrock_attachment_capabilities"), + ("traceai_litellm", "litellm_attachment_capabilities"), + ("traceai_together", "together_attachment_capabilities"), + ("traceai_fireworks", "fireworks_attachment_capabilities"), + ("traceai_xai", "xai_attachment_capabilities"), + ("traceai_mistralai", "mistralai_attachment_capabilities"), + ("traceai_openrouter", "openrouter_attachment_capabilities"), + ("traceai_ollama", "ollama_attachment_capabilities"), + ("traceai_portkey", "portkey_attachment_capabilities"), + ("traceai_deepseek", "deepseek_attachment_capabilities"), + ("traceai_cerebras", "cerebras_attachment_capabilities"), + ("traceai_vllm", "vllm_attachment_capabilities"), + ("traceai_minimax", "minimax_attachment_capabilities"), +]) +def test_package_descriptor(module, name): + package = pytest.importorskip(module) + descriptor = getattr(package, name) + assert descriptor["schema_version"] == 1 and isinstance(descriptor["operations"], tuple) + with pytest.raises(AttributeError): + getattr(package, "no_such_attachment_capabilities") diff --git a/python/tests/attachments_compat_providers/test_google_attachments.py b/python/tests/attachments_compat_providers/test_google_attachments.py new file mode 100644 index 000000000..478ffeba6 --- /dev/null +++ b/python/tests/attachments_compat_providers/test_google_attachments.py @@ -0,0 +1,206 @@ +"""Real google-genai and Vertex (gapic REST) SDKs -> traceAI instrumentors -> loopback F01.""" + +import asyncio +import json + +import pytest + +from conftest import MP4, PDF, b64, flush, png, refs, span_named, uploads_by_purpose, wav # noqa: E402 +from fi_instrumentation import TraceConfig # noqa: E402 + +IMAGE = png(seed=41) + + +# ------------------------------------------------------------------ google-genai +@pytest.fixture +def genai_instrument(spans): + pytest.importorskip("google.genai") + pytest.importorskip("traceai_google_genai") + from traceai_google_genai import GoogleGenAIInstrumentor + + tracer_provider, exporter = spans + state = {} + + def install(manager, config=None): + GoogleGenAIInstrumentor().instrument(tracer_provider=tracer_provider, attachments=manager, + config=config or TraceConfig()) + state["on"] = True + return exporter + + yield install + if state.get("on"): + GoogleGenAIInstrumentor().uninstrument() + + +def genai_client(media_provider): + from google import genai + from google.genai import types + + return genai.Client(api_key="synthetic-key", http_options=types.HttpOptions(base_url=media_provider.url + "/")) + + +def gemini_contents(): + from google.genai import types + + return [types.Content(role="user", parts=[ + types.Part(text="describe"), + types.Part(inline_data=types.Blob(mime_type="image/png", data=IMAGE)), + types.Part(inline_data=types.Blob(mime_type="application/pdf", data=PDF)), + types.Part(file_data=types.FileData(file_uri="gs://bucket/never-read.png", mime_type="image/png")), + types.Part(inline_data=types.Blob(mime_type="audio/L16;codec=pcm;rate=24000", data=b"\x00\x01" * 64)), + ])] + + +@pytest.mark.parametrize("async_", [False, True]) +def test_genai_generate_content_input_and_generated_output(f01, media_provider, genai_instrument, async_): + manager = f01.manager() + exporter = genai_instrument(manager) + client = genai_client(media_provider) + if async_: + response = asyncio.run(client.aio.models.generate_content(model="gemini-synthetic", contents=gemini_contents())) + else: + response = client.models.generate_content(model="gemini-synthetic", contents=gemini_contents()) + assert response.candidates[0].content.parts[1].inline_data.data == media_provider.generated + flush(manager) + assert sorted(f01.payloads) == sorted([IMAGE, PDF, media_provider.generated]) + by_purpose = uploads_by_purpose(f01) + assert sorted(by_purpose) == ["document", "model_input", "model_output"] + assert by_purpose["model_output"][0]["direction"] == "output" + span = span_named(exporter, "GenerateContent", "AsyncGenerateContent") + positions = sorted((r["direction"], r["message_index"], r["part_index"]) for r in refs(span)) + assert positions == [("input", 0, 1), ("input", 0, 2), ("output", 0, 1)] + # The application's request reached the provider unchanged (base64 on the wire, as the SDK sends it). + sent = json.dumps(media_provider.requests[-1][1]) + assert b64(IMAGE) in sent + manager.shutdown(5) + + +def test_genai_stream_captures_input_only(f01, media_provider, genai_instrument): + manager = f01.manager() + genai_instrument(manager) + client = genai_client(media_provider) + for _ in client.models.generate_content_stream(model="gemini-synthetic", contents=gemini_contents()): + pass + flush(manager) + assert sorted(f01.payloads) == sorted([IMAGE, PDF]) + manager.shutdown(5) + + +def test_genai_bare_parts_and_audio(f01, media_provider, genai_instrument): + from google.genai import types + + f01.state["policy"] = {"allowed_modalities": ["audio", "image", "document", "file", "video"]} + manager = f01.manager() + genai_instrument(manager) + sound = wav(seed=5) + client = genai_client(media_provider) + client.models.generate_content(model="gemini-synthetic", contents=[ + types.Part.from_bytes(data=sound, mime_type="audio/wav"), + types.Part.from_bytes(data=MP4, mime_type="video/mp4"), + "transcribe", + ]) + flush(manager) + assert sorted(f01.payloads) == sorted([sound, MP4, media_provider.generated]) + manager.shutdown(5) + + +@pytest.mark.parametrize("config", [TraceConfig(hide_input_images=True), TraceConfig(hide_outputs=True)]) +def test_genai_local_restrictions(f01, media_provider, genai_instrument, config): + manager = f01.manager() + genai_instrument(manager, config) + client = genai_client(media_provider) # keep the client alive: it closes its transport when collected + client.models.generate_content(model="gemini-synthetic", contents=gemini_contents()) + flush(manager) + if config.hide_input_images: + assert sorted(f01.payloads) == sorted([PDF, media_provider.generated]) + else: + assert sorted(f01.payloads) == sorted([IMAGE, PDF]) + manager.shutdown(5) + + +def test_genai_without_manager_is_unchanged(media_provider, genai_instrument): + exporter = genai_instrument(None) + client = genai_client(media_provider) # keep the client alive: it closes its transport when collected + client.models.generate_content(model="gemini-synthetic", contents=gemini_contents()) + span = span_named(exporter, "GenerateContent") + assert "fi.attachment.refs" not in span.attributes + + +# ------------------------------------------------------------------ Vertex (gapic, REST transport) +@pytest.fixture +def vertex_instrument(spans): + pytest.importorskip("google.cloud.aiplatform_v1") + pytest.importorskip("traceai_vertexai") + from traceai_vertexai import VertexAIInstrumentor + + tracer_provider, exporter = spans + state = {} + + def install(manager, config=None): + VertexAIInstrumentor().instrument(tracer_provider=tracer_provider, attachments=manager, + config=config or TraceConfig()) + state["on"] = True + return exporter + + yield install + if state.get("on"): + VertexAIInstrumentor().uninstrument() + + +def vertex_client(media_provider): + from google.auth.credentials import AnonymousCredentials + from google.cloud import aiplatform_v1 + + # Created AFTER instrument(): the instrumentor wraps gapic's wrap_method. + return aiplatform_v1.PredictionServiceClient( + credentials=AnonymousCredentials(), transport="rest", + client_options={"api_endpoint": media_provider.url}) + + +MODEL = "projects/synthetic/locations/us-central1/publishers/google/models/" + + +def test_vertex_generate_content_input_and_output(f01, media_provider, vertex_instrument): + from google.cloud import aiplatform_v1 as v1 + + manager = f01.manager() + exporter = vertex_instrument(manager) + request = v1.GenerateContentRequest(model=MODEL + "gemini-synthetic", contents=[v1.Content(role="user", parts=[ + v1.Part(text="describe"), + v1.Part(inline_data=v1.Blob(mime_type="image/png", data=IMAGE)), + v1.Part(inline_data=v1.Blob(mime_type="application/pdf", data=PDF)), + v1.Part(file_data=v1.FileData(file_uri="gs://bucket/never-read.png", mime_type="image/png")), + ])]) + response = vertex_client(media_provider).generate_content(request=request) + assert response.candidates[0].content.parts[1].inline_data.data == media_provider.generated + flush(manager) + assert sorted(f01.payloads) == sorted([IMAGE, PDF, media_provider.generated]) + assert sorted(uploads_by_purpose(f01)) == ["document", "model_input", "model_output"] + assert len(refs(exporter.get_finished_spans()[-1])) == 3 + manager.shutdown(5) + + +def test_vertex_imagen_predictions_are_generated_images(f01, media_provider, vertex_instrument): + from google.protobuf import json_format, struct_pb2 + + manager = f01.manager() + vertex_instrument(manager) + instance = json_format.ParseDict({"prompt": "a cat"}, struct_pb2.Value()) + vertex_client(media_provider).predict(endpoint=MODEL + "imagen-3.0-generate-002", instances=[instance]) + flush(manager) + assert f01.payloads == [media_provider.generated] + assert f01.uploads[0]["purpose"] == "model_output" and f01.uploads[0]["direction"] == "output" + manager.shutdown(5) + + +def test_vertex_other_prediction_endpoints_are_never_read(f01, media_provider, vertex_instrument): + from google.protobuf import json_format, struct_pb2 + + manager = f01.manager() + vertex_instrument(manager) + instance = json_format.ParseDict({"content": "x"}, struct_pb2.Value()) + vertex_client(media_provider).predict(endpoint="projects/synthetic/locations/us-central1/endpoints/123", + instances=[instance]) + flush(manager) + assert f01.payloads == [] + manager.shutdown(5) diff --git a/python/tests/attachments_compat_providers/test_provider_media.py b/python/tests/attachments_compat_providers/test_provider_media.py new file mode 100644 index 000000000..0a818f8af --- /dev/null +++ b/python/tests/attachments_compat_providers/test_provider_media.py @@ -0,0 +1,500 @@ +"""provider_media helper rules, through the real manager to a loopback F01 server.""" + +import builtins +import dataclasses +import io +import os +import subprocess +import sys +import textwrap +import traceback + +import pytest +from opentelemetry import context as otel_context + +from conftest import MP4, PDF, b64, data_uri, flush, jpeg, png, refs, wav # noqa: E402 +from fi_instrumentation import TraceConfig # noqa: E402 +from fi_instrumentation.attachments import AttachmentConfig, CapturePolicy, create_attachment_manager # noqa: E402 +from fi_instrumentation.attachments import media_capture, provider_media # noqa: E402 +from fi_instrumentation.attachments.provider_media import ( # noqa: E402 + PROVIDER_MEDIA_CAPABILITIES, + ProviderMedia, + capabilities, + consistent, + sniff, +) + +IMAGE = png(seed=31) +OTHER = png(seed=32) + + +def capture(manager, tracer_provider, fn, *, trace_config=None, name="llm"): + with tracer_provider.get_tracer("t").start_as_current_span(name) as span: + media = ProviderMedia(manager, span, "test", trace_config=trace_config) + fn(media) + return media, span + + +def test_compatible_messages_capture_data_uri_images_and_audio(f01, spans): + tracer_provider, exporter = spans + manager = f01.manager() + sound = wav(seed=3) + messages = [ + {"role": "system", "content": "be brief"}, + {"role": "user", "content": [ + {"type": "text", "text": "what is this"}, + {"type": "image_url", "image_url": {"url": data_uri(IMAGE)}}, + {"type": "image_url", "image_url": {"url": "https://example.invalid/x.png"}}, + {"type": "input_audio", "input_audio": {"data": b64(sound), "format": "wav"}}, + ]}, + ] + media, _ = capture(manager, tracer_provider, lambda m: m.compatible_messages(messages, "openai")) + flush(manager) + assert sorted(f01.payloads) == sorted([IMAGE, sound]) + assert set(media.captured) == {"chat/1/1", "chat/1/3"} + span = exporter.get_finished_spans()[-1] + by_mime = {ref["mime_type"]: ref for ref in refs(span)} + assert by_mime["image/png"]["message_index"] == 1 and by_mime["image/png"]["part_index"] == 1 + assert by_mime["audio/wav"]["role"] == "user" and by_mime["audio/wav"]["direction"] == "input" + manager.shutdown(5) + + +@pytest.mark.parametrize("dialect", ["groq", "mistral", "image"]) +def test_image_only_dialects_never_read_audio(f01, spans, dialect): + tracer_provider, _ = spans + manager = f01.manager() + messages = [{"role": "user", "content": [ + {"type": "input_audio", "input_audio": {"data": b64(wav()), "format": "wav"}}, + {"type": "image_url", "image_url": data_uri(IMAGE) if dialect == "mistral" else {"url": data_uri(IMAGE)}}, + ]}] + capture(manager, tracer_provider, lambda m: m.compatible_messages(messages, dialect)) + flush(manager) + assert f01.payloads == [IMAGE] + manager.shutdown(5) + + +def test_wrong_label_is_never_uploaded_and_cannot_bypass_hide_input_images(f01, spans): + """A PNG labelled as WAV audio would dodge hide_input_images; the magic-byte check refuses it.""" + tracer_provider, _ = spans + manager = f01.manager() + messages = [{"role": "user", "content": [ + {"type": "input_audio", "input_audio": {"data": b64(IMAGE), "format": "wav"}}]}] + contents = [{"role": "user", "parts": [{"inline_data": {"mime_type": "audio/wav", "data": OTHER}}]}] + capture(manager, tracer_provider, lambda m: (m.compatible_messages(messages, "openai"), m.gemini_contents(contents)), + trace_config=TraceConfig(hide_input_images=True)) + flush(manager) + assert f01.payloads == [] + assert f01.uploads == [] + manager.shutdown(5) + + +def test_declared_pdf_and_video_need_matching_bytes(f01, spans): + tracer_provider, _ = spans + f01.state["policy"] = {"allowed_modalities": ["audio", "image", "document", "file", "video"]} + manager = f01.manager() + contents = [{"role": "user", "parts": [ + {"inline_data": {"mime_type": "application/pdf", "data": PDF}}, + {"inline_data": {"mime_type": "video/mp4", "data": MP4}}, + {"inline_data": {"mime_type": "application/pdf", "data": IMAGE}}, # mislabelled: refused + {"inline_data": {"mime_type": "image/jpeg", "data": jpeg(4)}}, + ]}] + capture(manager, tracer_provider, lambda m: m.gemini_contents(contents)) + flush(manager) + assert sorted(f01.payloads) == sorted([PDF, MP4, jpeg(4)]) + purposes = sorted(u["purpose"] for u in f01.uploads) + assert purposes == ["document", "model_input", "model_input"] + manager.shutdown(5) + + +def test_server_policy_without_video_denies_video(f01, spans): + tracer_provider, _ = spans + manager = f01.manager() # the harness policy allows audio, image, document and file only + contents = [{"role": "user", "parts": [{"inline_data": {"mime_type": "video/mp4", "data": MP4}}]}] + capture(manager, tracer_provider, lambda m: m.gemini_contents(contents)) + flush(manager) + assert f01.payloads == [] + manager.shutdown(5) + + +def test_preflight_runs_before_any_read(f01, spans): + """A denied part is never read: the data-URI getter is not called under hide_input_images.""" + tracer_provider, _ = spans + manager = f01.manager() + reads = [] + + class Part(dict): + def get(self, key, default=None): + if key == "image_url": + reads.append(key) + return super().get(key, default) + + messages = [{"role": "user", "content": [Part(type="image_url", image_url={"url": data_uri(IMAGE)})]}] + capture(manager, tracer_provider, lambda m: m.compatible_messages(messages, "openai"), + trace_config=TraceConfig(hide_input_images=True)) + flush(manager) + assert reads == [] and f01.payloads == [] + capture(manager, tracer_provider, lambda m: m.compatible_messages(messages, "openai")) + flush(manager) + assert reads == ["image_url"] and f01.payloads == [IMAGE] + manager.shutdown(5) + + +def test_small_server_cap_resizes_data_uri_gate(f01, spans): + """The data-URI gate asks for 1 MiB; a server cap below that re-admits at the exact size.""" + tracer_provider, _ = spans + f01.state["limits"] = {"max_upload_bytes": 4096} + manager = f01.manager() + messages = [{"role": "user", "content": [{"type": "image_url", "image_url": {"url": data_uri(IMAGE)}}]}] + capture(manager, tracer_provider, lambda m: m.compatible_messages(messages, "openai")) + flush(manager) + assert f01.payloads == [IMAGE] + manager.shutdown(5) + + +def test_untyped_images_sniff_their_type_and_skip_paths(f01, spans, tmp_path, monkeypatch): + """Ollama/Titan images: bytes or base64 only. A path string is never opened or decoded into an upload.""" + tracer_provider, _ = spans + manager = f01.manager() + path = tmp_path / "picture.png" + path.write_bytes(OTHER) + opened = [] + real_open, real_io_open = builtins.open, io.open + + def watch(opener): + def wrapped(file, *args, **kwargs): + if any(frame.filename.endswith("provider_media.py") for frame in traceback.extract_stack()): + opened.append(str(file)) + return opener(file, *args, **kwargs) + return wrapped + + monkeypatch.setattr(builtins, "open", watch(real_open)) + monkeypatch.setattr(io, "open", watch(real_io_open)) + # A PDF has no image signature: untyped sources are images only, so it is never uploaded. + images = [IMAGE, b64(jpeg(7)), str(path), path, "not base64!", PDF, b64(PDF)] + media, _ = capture(manager, tracer_provider, lambda m: m.ollama_images(images)) + flush(manager) + monkeypatch.undo() + assert sorted(f01.payloads) == sorted([IMAGE, jpeg(7)]) + assert opened == [] + assert sorted(u["mime_type"] for u in f01.uploads) == ["image/jpeg", "image/png"] + manager.shutdown(5) + + +def test_context_policy_and_owner_keys_deny(f01, spans): + """The shared keys media_capture.CAPTURE_POLICY_KEY / OWNER_KEY apply to provider media (deny wins).""" + tracer_provider, _ = spans + manager = f01.manager() + messages = [{"role": "user", "content": [{"type": "image_url", "image_url": {"url": data_uri(IMAGE)}}]}] + for key, value in ((media_capture.CAPTURE_POLICY_KEY, CapturePolicy(capture_content=False)), + (media_capture.OWNER_KEY, True)): + token = otel_context.attach(otel_context.set_value(key, value)) + try: + media, _ = capture(manager, tracer_provider, lambda m: m.compatible_messages(messages, "openai")) + finally: + otel_context.detach(token) + assert media.captured == {} + flush(manager) + assert f01.payloads == [] + # claim() marks exactly that owner key. + assert otel_context.get_value(media_capture.OWNER_KEY, provider_media.claim()) is True + manager.shutdown(5) + + +@pytest.mark.parametrize("config", [ + TraceConfig(hide_input_images=True), TraceConfig(hide_inputs=True), TraceConfig(pii_redaction=True), + TraceConfig(hide_input_messages=True), TraceConfig(hide_input_text=True), +]) +def test_local_input_restrictions_deny(f01, spans, config): + tracer_provider, _ = spans + manager = f01.manager(allow_unredacted_media_with_text_pii=True) + messages = [{"role": "user", "content": [{"type": "image_url", "image_url": {"url": data_uri(IMAGE)}}]}] + contents = [{"role": "user", "parts": [{"inline_data": {"mime_type": "image/png", "data": OTHER}}]}] + capture(manager, tracer_provider, lambda m: (m.compatible_messages(messages), m.gemini_contents(contents)), + trace_config=config) + flush(manager) + assert f01.payloads == [] + manager.shutdown(5) + + +@pytest.mark.parametrize("config", [TraceConfig(hide_outputs=True), TraceConfig(hide_output_messages=True), + TraceConfig(hide_output_text=True), TraceConfig(pii_redaction=True)]) +def test_local_output_restrictions_deny_generated_media(f01, spans, config): + tracer_provider, _ = spans + manager = f01.manager(allow_unredacted_media_with_text_pii=True) + response = {"candidates": [{"content": {"role": "model", "parts": [ + {"inline_data": {"mime_type": "image/png", "data": IMAGE}}]}}]} + capture(manager, tracer_provider, lambda m: (m.gemini_response(response), + m.generated_images([{"b64_json": b64(OTHER)}])), + trace_config=config) + flush(manager) + assert f01.payloads == [] + manager.shutdown(5) + + +def test_unsampled_disabled_and_missing_managers_capture_nothing(f01, spans): + from opentelemetry.sdk.trace import TracerProvider + from opentelemetry.sdk.trace.sampling import ALWAYS_OFF + + manager = f01.manager() + messages = [{"role": "user", "content": [{"type": "image_url", "image_url": {"url": data_uri(IMAGE)}}]}] + unsampled = TracerProvider(sampler=ALWAYS_OFF) + media, _ = capture(manager, unsampled, lambda m: m.compatible_messages(messages)) + assert not media.usable + disabled = create_attachment_manager(config=AttachmentConfig(enabled=False)) + tracer_provider, _ = spans + for candidate in (None, disabled, object()): + media, _ = capture(candidate, tracer_provider, lambda m: m.compatible_messages(messages)) + assert not media.usable and media.captured == {} + flush(manager) + assert f01.payloads == [] + manager.shutdown(5) + + +def test_second_observer_of_same_span_uploads_nothing(f01, spans): + tracer_provider, _ = spans + manager = f01.manager() + messages = [{"role": "user", "content": [{"type": "image_url", "image_url": {"url": data_uri(IMAGE)}}]}] + with tracer_provider.get_tracer("t").start_as_current_span("llm") as span: + first = ProviderMedia(manager, span, "ns").compatible_messages(messages).captured + second = ProviderMedia(manager, span, "ns").compatible_messages(messages).captured + flush(manager) + assert first == second and len(first) == 1 and f01.payloads == [IMAGE] + manager.shutdown(5) + + +def test_part_cap_bounds_work(f01, spans): + tracer_provider, _ = spans + manager = f01.manager() + calls = [] + preflight = manager.preflight + + def counted(*args, **kwargs): + calls.append(kwargs.get("mime_type")) + return preflight(*args, **kwargs) + + manager.preflight = counted + # 80 parts over two messages: each list is within its own bound, so only the per-observation cap stops it. + messages = [{"role": "user", "content": [{"type": "image_url", "image_url": {"url": data_uri(png(seed=m * 40 + i))}} + for i in range(40)]} for m in range(2)] + media, _ = capture(manager, tracer_provider, lambda m: m.compatible_messages(messages)) + flush(manager) + assert len(f01.payloads) == provider_media.MAX_PARTS == len(media.captured) + # One preflight per admitted part (exact type and size already match the gate), none past the cap. + assert len(calls) == provider_media.MAX_PARTS + del manager.preflight + media, _ = capture(manager, tracer_provider, lambda m: m.compatible_messages( + [{"role": "user", "content": [{"type": "input_audio", "input_audio": {"data": b64(wav(seed=i)), "format": "wav"}} + for i in range(40)]}] * 2)) + flush(manager) + assert len(media.captured) == provider_media.MAX_PARTS + manager.shutdown(5) + + +def test_sniff_and_consistent(): + assert sniff(IMAGE) == "image/png" and sniff(jpeg()) == "image/jpeg" and sniff(PDF) == "application/pdf" + assert sniff(MP4) == "video/mp4" and sniff(wav()) == "audio/wav" and sniff(b"RIFF\x00\x00\x00\x00WEBPVP8 ") == "image/webp" + assert sniff(b"hello") is None and sniff("not bytes") is None + assert consistent("image/png", IMAGE) and not consistent("audio/wav", IMAGE) + assert consistent("video/webm", b"\x1a\x45\xdf\xa3" + b"\x00" * 12) + assert consistent("audio/webm", b"\x1a\x45\xdf\xa3" + b"\x00" * 12) + + +@pytest.mark.parametrize("brand,expected", [ + (b"isom", "video/mp4"), (b"mp42", "video/mp4"), (b"avc1", "video/mp4"), + # HEIF/HEIC and AVIF still images, QuickTime and audio-only M4A share the ftyp box: never video/mp4. + (b"heic", None), (b"heix", None), (b"mif1", None), (b"msf1", None), (b"avif", None), (b"avis", None), + (b"qt ", None), (b"M4A ", None), (b"zzzz", None), +]) +def test_sniff_checks_iso_bmff_major_brand(brand, expected): + head = b"\x00\x00\x00\x18ftyp" + brand + b"\x00\x00\x02\x00" + b"\x00" * 16 + assert sniff(head) == expected + assert consistent("video/mp4", head) is (expected == "video/mp4") + + +def test_heic_labelled_video_mp4_is_never_uploaded(f01, spans): + """A HEIC still image declared as video/mp4 would dodge hide_input_images; the brand check refuses it.""" + tracer_provider, _ = spans + f01.state["policy"] = {"allowed_modalities": ["audio", "image", "document", "file", "video"]} + manager = f01.manager() + heic = b"\x00\x00\x00\x18ftypheic\x00\x00\x00\x00mif1heic" + b"\x00" * 24 + contents = [{"role": "user", "parts": [{"inline_data": {"mime_type": "video/mp4", "data": heic}}, + {"inline_data": {"mime_type": "video/mp4", "data": MP4}}]}] + capture(manager, tracer_provider, lambda m: m.gemini_contents(contents), + trace_config=TraceConfig(hide_input_images=True)) + flush(manager) + assert f01.payloads == [MP4] + manager.shutdown(5) + + +def test_openai_dialect_file_parts_capture_pdf_once(f01, spans): + """Chat ``file`` (file.file_data) and Responses-style ``input_file`` PDF data URIs; IDs and URLs never read.""" + tracer_provider, exporter = spans + manager = f01.manager() + other_pdf = PDF + b"%%second\n" + messages = [{"role": "user", "content": [ + {"type": "text", "text": "summarize"}, + {"type": "file", "file": {"filename": "a.pdf", "file_data": data_uri(PDF, "application/pdf")}}, + {"type": "input_file", "filename": "b.pdf", "file_data": data_uri(other_pdf, "application/pdf")}, + {"type": "file", "file": {"file_id": "file-synthetic"}}, + {"type": "file", "file": {"file_data": data_uri(IMAGE, "application/pdf")}}, # mislabelled: refused + {"type": "file", "file": {"file_data": data_uri(PDF, "text/plain")}}, # not a captured type + ]}] + # hide_input_images does not hide documents. + # Without documents=True (every route but LiteLLM) no file part is read. + capture(manager, tracer_provider, lambda m: m.compatible_messages(messages, "openai")) + flush(manager) + assert f01.payloads == [] + media, span = capture(manager, tracer_provider, + lambda m: m.compatible_messages(messages, "openai", documents=True), + trace_config=TraceConfig(hide_input_images=True)) + flush(manager) + assert sorted(f01.payloads) == sorted([PDF, other_pdf]) + assert set(media.captured) == {"chat/0/1", "chat/0/2"} + span = exporter.get_finished_spans()[-1] + assert sorted((r["part_index"], r["purpose"], r["mime_type"]) for r in refs(span)) == [ + (1, "document", "application/pdf"), (2, "document", "application/pdf")] + manager.shutdown(5) + + +@pytest.mark.parametrize("dialect", ["groq", "mistral", "image"]) +def test_image_only_dialects_never_read_file_parts(f01, spans, dialect): + tracer_provider, _ = spans + manager = f01.manager() + messages = [{"role": "user", "content": [ + {"type": "file", "file": {"file_data": data_uri(PDF, "application/pdf")}}, + {"type": "input_file", "file_data": data_uri(PDF, "application/pdf")}]}] + capture(manager, tracer_provider, lambda m: m.compatible_messages(messages, dialect, documents=True)) + flush(manager) + assert f01.payloads == [] + manager.shutdown(5) + + +def test_data_uri_length_is_bounded_before_slicing(f01, spans): + """An oversized data URI is refused from its length alone: the string is never sliced or copied.""" + tracer_provider, _ = spans + manager = f01.manager() + slices = [] + + class Tracked(str): + def __getitem__(self, key): + slices.append(key) + return str.__getitem__(self, key) + + oversized = Tracked("data:image/png;base64," + "A" * (4 * provider_media.MAX_DECODED_BYTES // 3 + 8)) + exact = Tracked(data_uri(IMAGE)) + messages = [{"role": "user", "content": [{"type": "image_url", "image_url": {"url": oversized}}]}] + capture(manager, tracer_provider, lambda m: m.compatible_messages(messages, "openai")) + flush(manager) + assert f01.payloads == [] and slices == [] + messages = [{"role": "user", "content": [{"type": "image_url", "image_url": {"url": exact}}]}] + capture(manager, tracer_provider, lambda m: m.compatible_messages(messages, "openai")) + flush(manager) + assert f01.payloads == [IMAGE] and slices # in-bound values are sliced once, after the bound check + manager.shutdown(5) + + +def test_capability_matrix_is_explicit(): + for package in ("traceai_deepseek", "traceai_cerebras", "traceai_together"): + assert PROVIDER_MEDIA_CAPABILITIES[package]["operations"] == () + assert PROVIDER_MEDIA_CAPABILITIES[package]["note"] + for package, descriptor in PROVIDER_MEDIA_CAPABILITIES.items(): + assert descriptor["schema_version"] == 1 and isinstance(descriptor["operations"], tuple) + assert "remote_url" in descriptor["unsupported"] or package in ( + "traceai_google_genai", "traceai_vertexai", "traceai_bedrock", "traceai_ollama", "traceai_deepseek", + "traceai_cerebras") + assert capabilities("traceai_unknown")["operations"] == () + with pytest.raises(TypeError): + PROVIDER_MEDIA_CAPABILITIES["traceai_xai"] = {} # read-only + + +PROBE = textwrap.dedent(""" + import sys + from opentelemetry.sdk.trace import TracerProvider + names = sys.argv[1:] + import importlib + for name in names: + try: + module = importlib.import_module(name) + except ImportError: + continue + instrumentor = [getattr(module, attr) for attr in dir(module) if attr.endswith("Instrumentor") + and attr != "BaseInstrumentor"] + if instrumentor: + try: + instrumentor[0]().instrument(tracer_provider=TracerProvider()) + except Exception as error: + print("SKIP", name, type(error).__name__) + loaded = sorted(m for m in sys.modules if m.startswith("fi_instrumentation.attachments")) + print("LOADED", ",".join(loaded)) +""") + + +def test_disabled_path_never_imports_attachment_modules(tmp_path): + """Without a manager, instrumenting every provider package imports no attachment module.""" + script = tmp_path / "probe.py" + script.write_text(PROBE) + names = ["traceai_google_genai", "traceai_vertexai", "traceai_bedrock", "traceai_litellm", "traceai_groq", + "traceai_together", "traceai_fireworks", "traceai_xai", "traceai_mistralai", "traceai_openrouter", + "traceai_ollama", "traceai_portkey", "traceai_deepseek", "traceai_cerebras", "traceai_vllm", + "traceai_minimax"] + result = subprocess.run([sys.executable, str(script), *names], capture_output=True, text=True, timeout=120, + env=dict(os.environ, LITELLM_LOCAL_MODEL_COST_MAP="True")) + assert result.returncode == 0, result.stderr[-2000:] + line = [row for row in result.stdout.splitlines() if row.startswith("LOADED")][-1] + loaded = line.split(" ", 1)[1] if " " in line else "" + assert "provider_media" not in loaded and "media_capture" not in loaded, line + + +def test_dataclass_policy_unchanged_by_helper(): + """local_capture_policy is the only fold; the helper adds no permissive default.""" + policy = media_capture.local_capture_policy(None, direction="input", image=True) + assert dataclasses.asdict(policy) == dataclasses.asdict( + dataclasses.replace(CapturePolicy(), model_input=True)) + + +class _Unreadable: + """A TraceConfig stand-in whose flags cannot be read.""" + + def __getattr__(self, name): + raise RuntimeError(name) + + +@pytest.mark.parametrize("flags,expected", [ + ({}, {}), + ({"hide_input_images": True}, {"record_inputs": False}), + ({"hide_inputs": True}, {"record_inputs": False}), + ({"hide_input_messages": True}, {"record_inputs": False}), + ({"hide_input_text": True}, {"record_inputs": False}), + ({"hide_outputs": True}, {"record_outputs": False}), + ({"hide_output_text": True}, {"record_outputs": False}), + ({"pii_redaction": True}, {"capture_content": False}), +]) +def test_deferral_policy_folds_local_deny(flags, expected): + policy = provider_media.deferral_policy(TraceConfig(**flags)) + assert dataclasses.asdict(policy) == dataclasses.asdict(dataclasses.replace(CapturePolicy(), **expected)) + + +def test_deferred_context_merges_and_fails_closed(): + """A deferred call keeps the context policy and owner mark; local deny is added, never relaxed.""" + key = media_capture.CAPTURE_POLICY_KEY + assert provider_media.deferral_policy(None) == CapturePolicy() + unreadable = provider_media.deferral_policy(_Unreadable()) + assert not (unreadable.capture_content or unreadable.record_inputs or unreadable.record_outputs) + + outer = otel_context.set_value(media_capture.OWNER_KEY, True, + otel_context.set_value(key, CapturePolicy(record_outputs=False))) + merged = provider_media.deferred(TraceConfig(hide_input_images=True), outer) + policy = otel_context.get_value(key, merged) + assert (policy.record_inputs, policy.record_outputs, policy.capture_content) == (False, False, True) + assert media_capture.owned_elsewhere(merged) + + garbage = provider_media.deferred(TraceConfig(), otel_context.set_value(key, "not-a-policy")) + denied = otel_context.get_value(key, garbage) + assert denied.capture_content is False and denied.requested is False + + token = otel_context.attach(provider_media.deferred(TraceConfig(pii_redaction=True))) + try: + assert media_capture.context_capture_policy().capture_content is False + finally: + otel_context.detach(token) + assert media_capture.context_capture_policy() is None