Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
15 commits
Select commit Hold shift + click to select a range
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
151 changes: 151 additions & 0 deletions .github/workflows/attachments-python-compat-providers-tests.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,151 @@
name: Python provider media (Gemini, Vertex, Bedrock, LiteLLM, compatible chat)

# Real provider SDKs (declared floor and latest) through the Python
# instrumentors and fi_instrumentation.attachments.provider_media to a TLS
# loopback F01 attachment server. Outbound network is disabled while the tests run.

on:
pull_request:
paths:
- 'python/fi_instrumentation/attachments/**'
- 'python/frameworks/google-genai/**'
- 'python/frameworks/vertexai/**'
- 'python/frameworks/bedrock/**'
- 'python/frameworks/litellm/**'
- 'python/frameworks/groq/**'
- 'python/frameworks/together/**'
- 'python/frameworks/fireworks/**'
- 'python/frameworks/xai/**'
- 'python/frameworks/deepseek/**'
- 'python/frameworks/cerebras/**'
- 'python/frameworks/mistralai/**'
- 'python/frameworks/openrouter/**'
- 'python/frameworks/portkey/**'
- 'python/frameworks/ollama/**'
- 'python/frameworks/vllm/**'
- 'python/frameworks/minimax/**'
- 'python/frameworks/openai/**'
- 'python/tests/attachments_providers/conftest.py'
- 'python/tests/attachments_compat_providers/**'
- '.github/workflows/attachments-python-compat-providers-tests.yml'
push:
branches: [dev]
paths:
- 'python/fi_instrumentation/attachments/**'
- 'python/frameworks/google-genai/**'
- 'python/frameworks/vertexai/**'
- 'python/frameworks/bedrock/**'
- 'python/frameworks/litellm/**'
- 'python/frameworks/groq/**'
- 'python/frameworks/together/**'
- 'python/frameworks/fireworks/**'
- 'python/frameworks/xai/**'
- 'python/frameworks/deepseek/**'
- 'python/frameworks/cerebras/**'
- 'python/frameworks/mistralai/**'
- 'python/frameworks/openrouter/**'
- 'python/frameworks/portkey/**'
- 'python/frameworks/ollama/**'
- 'python/frameworks/vllm/**'
- 'python/frameworks/minimax/**'
- 'python/frameworks/openai/**'
- 'python/tests/attachments_providers/conftest.py'
- 'python/tests/attachments_compat_providers/**'
- '.github/workflows/attachments-python-compat-providers-tests.yml'
workflow_dispatch:

permissions:
contents: read

jobs:
media:
name: Provider media (${{ matrix.package }} ${{ matrix.tier }})
runs-on: ubuntu-24.04
timeout-minutes: 25
strategy:
fail-fast: false
matrix:
include:
- {package: core, tier: latest, python: '3.12', sdks: '', tests: core, min: 54}
- {package: google-genai, tier: floor, python: '3.10', sdks: 'google-genai==1.8.0', tests: google, min: 9}
- {package: google-genai, tier: latest, python: '3.12', sdks: 'google-genai==3.2.0', tests: google, min: 9}
- {package: vertexai, tier: floor, python: '3.10', sdks: 'google-cloud-aiplatform==1.61.0', tests: google, min: 5}
- {package: vertexai, tier: latest, python: '3.12', sdks: 'google-cloud-aiplatform==1.101.0', tests: google, min: 5}
- {package: bedrock, tier: floor, python: '3.10', sdks: 'boto3==1.38.17', tests: aws, min: 12}
- {package: bedrock, tier: latest, python: '3.12', sdks: 'boto3==1.44.1', tests: aws, min: 12}
- {package: litellm, tier: floor, python: '3.10', sdks: '-e python/frameworks/openai litellm==1.53.1 openai==1.69.0', tests: aws, min: 9}
- {package: litellm, tier: latest, python: '3.12', sdks: '-e python/frameworks/openai litellm==1.109.1', tests: aws, min: 10}
- {package: groq, tier: floor, python: '3.10', sdks: 'groq==0.9.0 httpx<0.28', tests: compat, min: 2}
- {package: groq, tier: latest, python: '3.12', sdks: 'groq==1.7.0', tests: compat, min: 2}
- {package: together, tier: floor, python: '3.10', sdks: 'together==1.3.0', tests: compat, min: 3}
- {package: together, tier: latest, python: '3.12', sdks: 'together==3.1.1', tests: compat, min: 3}
- {package: fireworks, tier: floor, python: '3.10', sdks: 'openai==1.69.0 httpx<0.28', tests: compat, min: 3}
- {package: fireworks, tier: latest, python: '3.12', sdks: 'openai==3.49.0', tests: compat, min: 3}
- {package: xai, tier: floor, python: '3.10', sdks: '-e python/frameworks/openai openai==1.69.0 httpx<0.28', tests: compat, min: 17}
- {package: xai, tier: latest, python: '3.12', sdks: '-e python/frameworks/openai openai==3.49.0', tests: compat, min: 17}
- {package: vllm, tier: floor, python: '3.10', sdks: '-e python/frameworks/openai openai==1.69.0 httpx<0.28', tests: compat, min: 4}
- {package: vllm, tier: latest, python: '3.12', sdks: '-e python/frameworks/openai openai==3.49.0', tests: compat, min: 4}
- {package: minimax, tier: floor, python: '3.10', sdks: '-e python/frameworks/openai openai==1.69.0 httpx<0.28', tests: compat, min: 18}
- {package: minimax, tier: latest, python: '3.12', sdks: '-e python/frameworks/openai openai==3.49.0', tests: compat, min: 18}
- {package: deepseek, tier: floor, python: '3.10', sdks: 'openai==1.69.0 httpx<0.28', tests: compat, min: 3}
- {package: deepseek, tier: latest, python: '3.12', sdks: 'openai==3.49.0', tests: compat, min: 3}
- {package: cerebras, tier: floor, python: '3.10', sdks: 'cerebras-cloud-sdk==1.0.0 httpx<0.28', tests: compat, min: 3}
- {package: cerebras, tier: latest, python: '3.12', sdks: 'cerebras-cloud-sdk==1.91.0', tests: compat, min: 3}
- {package: mistralai, tier: floor, python: '3.10', sdks: 'mistralai==1.1.0', tests: compat, min: 5}
- {package: mistralai, tier: latest, python: '3.12', sdks: 'mistralai==3.7.0', tests: compat, min: 5}
- {package: openrouter, tier: floor, python: '3.10', sdks: 'openrouter==1.3.0', tests: compat, min: 3}
- {package: openrouter, tier: latest, python: '3.12', sdks: 'openrouter==1.3.107', tests: compat, min: 3}
- {package: portkey, tier: floor, python: '3.10', sdks: 'portkey-ai==1.9.0 httpx<0.28', tests: compat, min: 3}
- {package: portkey, tier: latest, python: '3.12', sdks: 'portkey-ai==2.3.4', tests: compat, min: 3}
- {package: ollama, tier: floor, python: '3.10', sdks: 'ollama==0.3.0', tests: compat, min: 4}
- {package: ollama, tier: latest, python: '3.12', sdks: 'ollama==0.6.3', tests: compat, min: 4}
steps:
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
with:
persist-credentials: false
- uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: ${{ matrix.python }}
- name: Install the core, the integration and the selected SDKs
env:
PACKAGE: ${{ matrix.package }}
SDKS: ${{ matrix.sdks }}
run: |
python -m venv .venv
integration=""
if [ "$PACKAGE" != core ]; then integration="-e python/frameworks/$PACKAGE"; fi
# shellcheck disable=SC2086
.venv/bin/pip install -e python $integration opentelemetry-sdk 'pytest>=8,<9' 'pytest-asyncio<0.25' $SDKS
.venv/bin/pip freeze | grep -iE '^(google-genai|google-cloud-aiplatform|boto3|litellm|openai|groq|together|cerebras-cloud-sdk|mistralai|openrouter|portkey-ai|ollama|httpx)==' || true
- name: Run with outbound network disabled
# bash -eo pipefail: a failing pytest is not hidden by tee.
shell: bash
env:
SUITE: ${{ matrix.tests }}
MIN: ${{ matrix.min }}
run: |
case "$SUITE" in
core) tests="tests/attachments_compat_providers/test_provider_media.py" ;;
google) tests="tests/attachments_compat_providers/test_google_attachments.py" ;;
aws) tests="tests/attachments_compat_providers/test_bedrock_litellm_attachments.py" ;;
compat) tests="tests/attachments_compat_providers/test_compat_attachments.py" ;;
esac
# A file and a node ID inside the same file would collapse to the node ID only,
# so the descriptor rows are added only where the compat file is not already selected.
case "$SUITE" in
google|aws) tests="$tests tests/attachments_compat_providers/test_compat_attachments.py::test_package_descriptor" ;;
esac
if [ "$SUITE" != core ]; then
tests="$tests tests/attachments_compat_providers/test_provider_media.py::test_disabled_path_never_imports_attachment_modules"
fi
sudo unshare --net -- bash -e -c '
ip link set lo up
cd "$2/python"
export FI_BASE_URL=http://127.0.0.1:9 FI_GRPC_URL=http://127.0.0.1:9
export LITELLM_LOCAL_MODEL_COST_MAP=True AWS_EC2_METADATA_DISABLED=true
# shellcheck disable=SC2086
PYTHONPATH=. "$1" -m pytest $3 -v -rs -p no:cacheprovider -o addopts="" -o faulthandler_timeout=180
' _ "$PWD/.venv/bin/python" "$PWD" "$tests" | tee pytest.log
# A job whose SDK rows silently skipped must not pass: require the rows this package proves.
passed=$(grep -Eo '[0-9]+ passed' pytest.log | tail -1 | cut -d' ' -f1)
if [ "${passed:-0}" -lt "$MIN" ]; then echo "expected at least $MIN passed, got ${passed:-0}"; exit 1; fi
53 changes: 52 additions & 1 deletion python/fi_instrumentation/attachments/media_capture.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,11 @@
- One registry per manager remembers admitted (span, position) pairs, so a
second observer of the same span and position gets the FIRST occurrence back
and uploads nothing.
- A per-call ``CapturePolicy`` set in the OpenTelemetry context under
``CAPTURE_POLICY_KEY`` (``traceai.with_attachment_capture_policy``) can only
add restrictions. A context marked under ``OWNER_KEY``
(``traceai.with_attachment_ownership``) belongs to an adapter that captures
the media itself, so nothing is captured here for that call.

Every public function is non-throwing and returns ``None``/empty on denial.
"""
Expand All @@ -26,6 +31,8 @@
from collections import OrderedDict
from typing import Any, Callable, Iterable, Mapping, NamedTuple, Optional, Tuple

from opentelemetry import context as otel_context

from .policy import CapturePolicy
from .stream import StreamOptions
from .types import Bytes, CorrelationHandle
Expand All @@ -50,6 +57,13 @@
# mulaw have no F01 MIME, so they are never captured.
SPEECH_FORMATS = {"mp3": "audio/mpeg", "opus": "audio/ogg", "ogg": "audio/ogg", "flac": "audio/flac",
"wav": "audio/wav"}
# Context keys shared BY NAME with the auto module (``traceai``) and any adapter, like the
# TypeScript ``Symbol.for`` keys: plain strings, so no package has to import another.
CAPTURE_POLICY_KEY = "traceai.auto.attachment_policy.v1"
OWNER_KEY = "traceai.auto.attachment_owner.v1"
# CapturePolicy facts that grant capture; every other fact only restricts.
_PERMISSIONS = frozenset(("capture_content", "record_inputs", "record_outputs", "requested"))
_DENY_ALL = CapturePolicy(capture_content=False, requested=False)


class Occurrence(NamedTuple):
Expand Down Expand Up @@ -93,6 +107,40 @@ def _restricted(config, name: str) -> bool:
return True


def merge_capture_policies(*policies: Any) -> CapturePolicy:
"""Deny-wins intersection: a grant needs every policy, a restriction needs any one.

``None`` adds nothing. Any other value that is not a ``CapturePolicy`` denies.
"""
present = [policy for policy in policies if policy is not None]
if any(not isinstance(policy, CapturePolicy) for policy in present):
return _DENY_ALL
if not present:
return CapturePolicy()
values = {}
for item in dataclasses.fields(CapturePolicy):
facts = [getattr(policy, item.name) for policy in present]
values[item.name] = all(facts) if item.name in _PERMISSIONS else any(facts)
return CapturePolicy(**values)


def context_capture_policy(context=None) -> Any:
"""The per-call policy in the (current) OpenTelemetry context; None when unset."""
try:
return otel_context.get_value(CAPTURE_POLICY_KEY, context)
except Exception:
# An unreadable restriction is a restriction, never permission.
return _DENY_ALL


def owned_elsewhere(context=None) -> bool:
"""True when another adapter marked this call's media as its own."""
try:
return otel_context.get_value(OWNER_KEY, context) is not None
except Exception:
return True


def local_capture_policy(trace_config=None, *, direction: str, image: bool = False, error_detail: bool = False,
base: Optional[CapturePolicy] = None) -> CapturePolicy:
"""Fold the instrumentor's local TraceConfig into a deny-only CapturePolicy.
Expand All @@ -103,6 +151,7 @@ def local_capture_policy(trace_config=None, *, direction: str, image: bool = Fal
ancestry, so a tool screenshot is still treated as model input.
"""
base = base if isinstance(base, CapturePolicy) else CapturePolicy()
base = merge_capture_policies(base, context_capture_policy())
names = _INPUT_HIDE if direction == "input" else _OUTPUT_HIDE
hidden = any(_restricted(trace_config, name) for name in names) or _restricted(trace_config, "pii_redaction")
if image and direction == "input" and _restricted(trace_config, "hide_input_images"):
Expand All @@ -116,6 +165,8 @@ def local_capture_policy(trace_config=None, *, direction: str, image: bool = Fal
changes["model_input"] = True
if error_detail:
changes["error_detail"] = True
if owned_elsewhere():
changes["requested"] = False
return dataclasses.replace(base, **changes)


Expand Down Expand Up @@ -262,7 +313,7 @@ class Observation:

def __init__(self, manager, span, namespace: str):
self.manager, self.span, self.namespace = manager, span, namespace
self.key = span_key(span) if active(manager) else None
self.key = span_key(span) if active(manager) and not owned_elsewhere() else None
self.captured: "OrderedDict[str, Occurrence]" = OrderedDict()
self._visited = 0

Expand Down
Loading
Loading