From 5255968b11379f3e863d54de0dec61674ed8c7d7 Mon Sep 17 00:00:00 2001 From: jjcav84 Date: Tue, 6 Oct 2026 22:50:48 -0500 Subject: [PATCH 01/11] fix(litellm): keep full multi-message context in input.value MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Same defect as #151 (fixed for openai in #245): the completion instrumentation wrote INPUT_VALUE three times, ending with "query" = eval_input[0] — the first message's text only — after already writing the joined text of every message. Multi-turn inputs lost all but the first content piece. Drop the query write and the now-unused "query" key from _process_messages so the joined eval_input survives; image parts still surface as {{input_images.N}} placeholders alongside INPUT_IMAGES. Also repairs three stale attribute assertions in the litellm test file that referenced pre-migration names (FI_SPAN_KIND -> GEN_AI_SPAN_KIND, LLM_MODEL_NAME -> GEN_AI_REQUEST_MODEL) so the file actually runs green. Generated with Devin Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../litellm/traceai_litellm/__init__.py | 4 - python/tests/test_framework_litellm.py | 73 ++++++++++++++++++- 2 files changed, 71 insertions(+), 6 deletions(-) diff --git a/python/frameworks/litellm/traceai_litellm/__init__.py b/python/frameworks/litellm/traceai_litellm/__init__.py index 69b981964..aa3caa105 100644 --- a/python/frameworks/litellm/traceai_litellm/__init__.py +++ b/python/frameworks/litellm/traceai_litellm/__init__.py @@ -136,8 +136,6 @@ def _instrument_func_type_completion( ) if eval_input := process_messages.get("eval_input"): _set_span_attribute(span, SpanAttributes.INPUT_VALUE, eval_input) - if query := process_messages.get("query"): - _set_span_attribute(span, SpanAttributes.INPUT_VALUE, query) for index, input_message in list(enumerate(messages)): for key, value in _get_attributes_from_message_param(input_message): @@ -694,7 +692,6 @@ def _process_messages(messages): "input_images": input_images if input_images else None, "filtered_messages": filtered_messages if filtered_messages else messages, "eval_input": "\n".join(eval_input), - "query": str(eval_input[0]) if eval_input else None, } except Exception as e: print(f"Error in _process_messages: {e}") @@ -702,5 +699,4 @@ def _process_messages(messages): "input_images": None, "filtered_messages": messages, "eval_input": None, - "query": None, } diff --git a/python/tests/test_framework_litellm.py b/python/tests/test_framework_litellm.py index 25928ce2d..6e58816eb 100644 --- a/python/tests/test_framework_litellm.py +++ b/python/tests/test_framework_litellm.py @@ -696,7 +696,7 @@ def test_process_messages_basic(self): assert result is not None assert "filtered_messages" in result assert "eval_input" in result - assert "query" in result + assert "query" not in result assert result["filtered_messages"] == messages assert "Hello world" in result["eval_input"] @@ -896,5 +896,74 @@ def test_streaming_with_context_attributes(self, tracer_provider, config, mock_l mock_context.assert_called() +class TestInputValueMultiMessage: + """Regression: INPUT_VALUE must keep every message, not just the first. + + Same defect as traceAI#151 (openai handler): a final write overwrote the + joined eval_input with query = eval_input[0]. + """ + + def _final_input_value(self, mock_span): + values = [ + c.args[1] + for c in mock_span.set_attribute.call_args_list + if c.args[0] == SpanAttributes.INPUT_VALUE + ] + assert values, "INPUT_VALUE was never set" + return values[-1] + + def test_completion_keeps_all_message_texts(self): + mock_span = MagicMock() + _instrument_func_type_completion( + mock_span, + { + "model": "gpt-4o", + "messages": [ + {"role": "system", "content": "You are helpful."}, + {"role": "user", "content": "What is 2+2?"}, + {"role": "assistant", "content": "4"}, + {"role": "user", "content": "And 3+3?"}, + ], + }, + ) + final = self._final_input_value(mock_span) + for text in ("You are helpful.", "What is 2+2?", "4", "And 3+3?"): + assert text in final + + def test_completion_multimodal_text_and_image_placeholder(self): + mock_span = MagicMock() + _instrument_func_type_completion( + mock_span, + { + "model": "gpt-4o", + "messages": [ + { + "role": "user", + "content": [ + {"type": "text", "text": "describe this"}, + { + "type": "image_url", + "image_url": {"url": "https://example.com/a.png"}, + }, + {"type": "text", "text": "in detail"}, + ], + } + ], + }, + ) + attrs = {c.args[0]: c.args[1] for c in mock_span.set_attribute.call_args_list} + final = attrs[SpanAttributes.INPUT_VALUE] + assert "describe this" in final + assert "in detail" in final + assert f"{{{SpanAttributes.INPUT_IMAGES}.0}}" in final + assert "https://example.com/a.png" in attrs[SpanAttributes.INPUT_IMAGES] + + def test_process_messages_returns_no_query_key(self): + result = _process_messages([{"role": "user", "content": "hi"}]) + assert "query" not in result + assert result["eval_input"] == "hi" + + if __name__ == "__main__": - pytest.main([__file__, "-v"]) \ No newline at end of file + pytest.main([__file__, "-v"]) + \ No newline at end of file From e22a7cc71972130bd8255c9ad399250b9c089b20 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Mon, 2 Nov 2026 00:01:46 -0800 Subject: [PATCH 02/11] fix(litellm): preserve request semantics and close streaming spans Keep streaming context scoped to provider operations, finalize errors and cancellation, retain usage-only chunks, suppress nested calls, bind positional requests, and omit SDK credentials from telemetry. Restore owned Protect descriptors on uninstrument. Exercise minimum/latest SDKs and loopback OTLP export. Co-Authored-By: Claude Code --- .github/workflows/litellm-python-tests.yml | 70 ++ python/frameworks/litellm/README.md | 4 +- python/frameworks/litellm/poetry.lock | 18 +- python/frameworks/litellm/pyproject.toml | 4 +- .../litellm/tests/test_lifecycle.py | 419 ++++++++++++ .../litellm/tests/test_otlp_contract.py | 68 ++ .../litellm/traceai_litellm/__init__.py | 619 ++++++++++-------- 7 files changed, 910 insertions(+), 292 deletions(-) create mode 100644 .github/workflows/litellm-python-tests.yml create mode 100644 python/frameworks/litellm/tests/test_lifecycle.py create mode 100644 python/frameworks/litellm/tests/test_otlp_contract.py diff --git a/.github/workflows/litellm-python-tests.yml b/.github/workflows/litellm-python-tests.yml new file mode 100644 index 000000000..da97fe33a --- /dev/null +++ b/.github/workflows/litellm-python-tests.yml @@ -0,0 +1,70 @@ +name: LiteLLM Python compatibility + +on: + pull_request: + paths: + - 'python/frameworks/litellm/**' + - 'python/tests/test_framework_litellm.py' + - 'python/tests/harness/**' + - 'python/fi_instrumentation/**' + - '.github/workflows/litellm-python-tests.yml' + push: + branches: [dev] + paths: + - 'python/frameworks/litellm/**' + - 'python/tests/test_framework_litellm.py' + - 'python/tests/harness/**' + - 'python/fi_instrumentation/**' + - '.github/workflows/litellm-python-tests.yml' + +permissions: + contents: read + +jobs: + contract: + name: LiteLLM (${{ matrix.sdk }}, Python ${{ matrix.python }}) + runs-on: ubuntu-24.04 + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + include: + - sdk: floor + python: '3.10' + - sdk: latest + python: '3.12' + - sdk: latest + python: '3.13' + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: false + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: ${{ matrix.python }} + - name: Install integration and selected SDK + env: + SDK: ${{ matrix.sdk }} + run: | + python -m venv .venv + if [ "$SDK" = floor ]; then + # PyPI no longer serves 1.43.0; use its immutable upstream release commit. + .venv/bin/pip install -e python -e python/frameworks/litellm \ + 'litellm @ git+https://github.com/BerriAI/litellm.git@4603eedcc13bfab2da5a82aed209e6ef496c384d' \ + 'openai==1.40.0' 'httpx==0.27.2' pytest pytest-asyncio tenacity + else + .venv/bin/pip install -e python -e python/frameworks/litellm \ + --upgrade litellm pytest pytest-asyncio tenacity + fi + .venv/bin/pip check + - name: Run with outbound network disabled + run: | + sudo unshare --net -- bash -e -c ' + ip link set lo up + export LITELLM_LOCAL_MODEL_COST_MAP=True + export HOME="$2/.test-home" + mkdir -p "$HOME" + "$1" -I -m pytest --noconftest -q -o addopts="" \ + "$2/python/frameworks/litellm/tests" \ + "$2/python/tests/test_framework_litellm.py" + ' _ "$PWD/.venv/bin/python" "$PWD" diff --git a/python/frameworks/litellm/README.md b/python/frameworks/litellm/README.md index 5fd7f4065..3bd1c6770 100644 --- a/python/frameworks/litellm/README.md +++ b/python/frameworks/litellm/README.md @@ -8,7 +8,7 @@ This integration provides support for using OpenTelemetry with the Litellm frame 1. **Install traceAI Litellm** ```bash -pip install pip install traceAI-litellm +pip install traceAI-litellm ``` @@ -46,6 +46,8 @@ from traceai_litellm import LiteLLMInstrumentor LiteLLMInstrumentor().instrument(tracer_provider=trace_provider) ``` +Use one tracing integration for each LiteLLM call. Do not also enable LiteLLM's native OpenTelemetry callback for calls traced by this instrumentor. + ### Create Litellm Components Set up your Litellm client with built-in observability. diff --git a/python/frameworks/litellm/poetry.lock b/python/frameworks/litellm/poetry.lock index 9d22c0f2f..b033e2216 100644 --- a/python/frameworks/litellm/poetry.lock +++ b/python/frameworks/litellm/poetry.lock @@ -1922,6 +1922,22 @@ files = [ {file = "sniffio-1.3.1.tar.gz", hash = "sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc"}, ] +[[package]] +name = "tenacity" +version = "9.1.2" +description = "Retry code until it succeeds" +optional = false +python-versions = ">=3.9" +groups = ["dev"] +files = [ + {file = "tenacity-9.1.2-py3-none-any.whl", hash = "sha256:f77bf36710d8b73a50b2dd155c97b870017ad21afe6ab300326b0371b3b05138"}, + {file = "tenacity-9.1.2.tar.gz", hash = "sha256:1169d376c297e7de388d18b4481760d478b0e99a777cad3a9c86e556f4b697cb"}, +] + +[package.extras] +doc = ["reno", "sphinx"] +test = ["pytest", "tornado (>=4.5)", "typeguard"] + [[package]] name = "tiktoken" version = "0.9.0" @@ -2301,4 +2317,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.1" python-versions = ">3.9, <3.14" -content-hash = "bdee8eb7b356a448b33abd2a71a4e1fdcc305f9c81288cba5c4cce416cbf1522" +content-hash = "a46fea14faa1d824f4f7d79d660f573cb8100eb8737660f0b43556872b53e6bf" diff --git a/python/frameworks/litellm/pyproject.toml b/python/frameworks/litellm/pyproject.toml index a799c34d2..79bc049a4 100644 --- a/python/frameworks/litellm/pyproject.toml +++ b/python/frameworks/litellm/pyproject.toml @@ -32,7 +32,9 @@ python = ">3.9, <3.14" litellm = ">=1.43.0" fi-instrumentation-otel = ">=0.1.11" - +[tool.poetry.group.dev.dependencies] +# LiteLLM imports tenacity lazily for its optional retry helpers. +tenacity = ">=8.2.3" [tool.poetry.urls] Homepage = "https://futureagi.com" diff --git a/python/frameworks/litellm/tests/test_lifecycle.py b/python/frameworks/litellm/tests/test_lifecycle.py new file mode 100644 index 000000000..c6243e00c --- /dev/null +++ b/python/frameworks/litellm/tests/test_lifecycle.py @@ -0,0 +1,419 @@ +"""Offline regressions using real SDK response objects and exported OTel spans.""" + +import asyncio +from contextlib import contextmanager +from types import SimpleNamespace + +import litellm +import pytest +from litellm.types.utils import ModelResponse +from opentelemetry import context, trace +from opentelemetry.context import _SUPPRESS_INSTRUMENTATION_KEY +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +from opentelemetry.trace import StatusCode + +import traceai_litellm +from fi_instrumentation.fi_types import SpanAttributes +from traceai_litellm import LiteLLMInstrumentor + +MESSAGES = [ + {"role": "system", "content": "Be helpful"}, + {"role": "user", "content": "Hello"}, +] + + +def response(): + return ModelResponse( + model="gpt-4o-mini", + choices=[{"message": {"role": "assistant", "content": "hello"}}], + usage={"prompt_tokens": 2, "completion_tokens": 1, "total_tokens": 3}, + ) + + +@pytest.fixture +def instrument(monkeypatch): + exporter = InMemorySpanExporter() + provider = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + # Keep each SDK test independent of other instrumentors in the core suite. + inst = LiteLLMInstrumentor() + if inst.is_instrumented_by_opentelemetry: + inst.uninstrument() + inst.original_litellm_funcs.clear() + monkeypatch.setattr(traceai_litellm, "Protect", None) + + @contextmanager + def install(**functions): + for name, fn in functions.items(): + monkeypatch.setattr(litellm, name, fn) + inst.instrument(tracer_provider=provider) + try: + yield exporter, provider + finally: + inst.uninstrument() + + yield install + if inst.is_instrumented_by_opentelemetry: + inst.uninstrument() + provider.shutdown() + + +def only_span(exporter): + spans = exporter.get_finished_spans() + assert len(spans) == 1 + return spans[0] + + +def test_positional_request_and_full_context(instrument): + def completion(model, messages, **kwargs): + return response() + + with instrument(completion=completion) as (exporter, _): + litellm.completion("gpt-4o-mini", MESSAGES) + attrs = only_span(exporter).attributes + assert attrs[SpanAttributes.GEN_AI_REQUEST_MODEL] == "gpt-4o-mini" + assert all(m["content"] in attrs[SpanAttributes.INPUT_VALUE] for m in MESSAGES) + + +def test_non_json_options_do_not_break_request(instrument): + result = response() + with instrument(completion=lambda **kwargs: result) as (exporter, _): + assert ( + litellm.completion( + model="gpt-4o-mini", messages=MESSAGES, response_format=object() + ) + is result + ) + assert only_span(exporter).status.status_code is StatusCode.OK + + +def test_usage_can_be_absent(instrument): + result = response() + result.usage = None + with instrument(completion=lambda **kwargs: result) as (exporter, _): + assert litellm.completion(model="gpt-4o-mini", messages=MESSAGES) is result + assert only_span(exporter).status.status_code is StatusCode.OK + + +@pytest.mark.parametrize("async_call", [False, True]) +@pytest.mark.asyncio +async def test_stream_construction_error_preserves_exception_and_parent( + instrument, async_call +): + error = ValueError("synthetic error") + + def completion(**kwargs): + raise error + + async def acompletion(**kwargs): + raise error + + with instrument(completion=completion, acompletion=acompletion) as ( + exporter, + provider, + ): + with provider.get_tracer("test").start_as_current_span("parent") as parent: + with pytest.raises(ValueError) as caught: + if async_call: + await litellm.acompletion( + model="gpt-4o-mini", messages=MESSAGES, stream=True + ) + else: + litellm.completion( + model="gpt-4o-mini", messages=MESSAGES, stream=True + ) + assert caught.value is error + assert trace.get_current_span() is parent + assert only_span(exporter).status.status_code is StatusCode.ERROR + + +def stream_chunks(): + yield {"choices": [{"delta": {"content": "hello"}}]} + yield { + "choices": [], + "usage": {"prompt_tokens": 2, "completion_tokens": 1, "total_tokens": 3}, + } + + +def test_sync_stream_usage_and_context(instrument): + with instrument(completion=lambda **kwargs: stream_chunks()) as ( + exporter, + provider, + ): + with provider.get_tracer("test").start_as_current_span("parent") as parent: + stream = litellm.completion( + model="gpt-4o-mini", messages=MESSAGES, stream=True + ) + assert trace.get_current_span() is parent + assert list(stream) == list(stream_chunks()) + assert trace.get_current_span() is parent + span = only_span(exporter) + assert span.attributes[SpanAttributes.OUTPUT_VALUE] == "hello" + assert span.attributes[SpanAttributes.GEN_AI_USAGE_TOTAL_TOKENS] == 3 + assert span.parent.span_id == parent.get_span_context().span_id + stream.close() + assert len(exporter.get_finished_spans()) == 1 + + +@pytest.mark.asyncio +async def test_async_stream_usage_and_cross_task_close(instrument): + async def chunks(): + for chunk in stream_chunks(): + yield chunk + + async def acompletion(**kwargs): + return chunks() + + with instrument(acompletion=acompletion) as (exporter, _): + stream = await litellm.acompletion( + model="gpt-4o-mini", messages=MESSAGES, stream=True + ) + assert not trace.get_current_span().get_span_context().is_valid + + async def consume(): + assert [c async for c in stream] == list(stream_chunks()) + await stream.aclose() + + await asyncio.create_task(consume()) + span = only_span(exporter) + assert span.attributes[SpanAttributes.OUTPUT_VALUE] == "hello" + assert span.attributes[SpanAttributes.GEN_AI_USAGE_TOTAL_TOKENS] == 3 + + +def test_sync_stream_read_error_ends_span(instrument): + error = ValueError("stream read failed") + + def chunks(): + yield next(stream_chunks()) + raise error + + with instrument(completion=lambda **kwargs: chunks()) as (exporter, _): + stream = litellm.completion(model="gpt-4o-mini", messages=MESSAGES, stream=True) + with pytest.raises(ValueError) as caught: + list(stream) + assert caught.value is error + assert only_span(exporter).status.status_code is StatusCode.ERROR + + +@pytest.mark.asyncio +async def test_async_stream_cancel_ends_span(instrument): + async def chunks(): + yield next(stream_chunks()) + raise asyncio.CancelledError() + + async def acompletion(**kwargs): + return chunks() + + with instrument(acompletion=acompletion) as (exporter, _): + stream = await litellm.acompletion( + model="gpt-4o-mini", messages=MESSAGES, stream=True + ) + with pytest.raises(asyncio.CancelledError): + async for _ in stream: + pass + assert only_span(exporter).status.status_code is StatusCode.ERROR + + +@pytest.mark.parametrize("async_call", [False, True]) +@pytest.mark.asyncio +async def test_early_close_closes_upstream_once(instrument, async_call): + closed = [] + + def chunks(): + try: + yield next(stream_chunks()) + yield next(stream_chunks()) + finally: + closed.append(True) + + async def achunks(): + try: + yield next(stream_chunks()) + yield next(stream_chunks()) + finally: + closed.append(True) + + async def acompletion(**kwargs): + return achunks() + + with instrument(completion=lambda **kwargs: chunks(), acompletion=acompletion) as ( + exporter, + _, + ): + if async_call: + stream = await litellm.acompletion( + model="gpt-4o-mini", messages=MESSAGES, stream=True + ) + await stream.__anext__() + await stream.aclose() + await stream.aclose() + else: + stream = litellm.completion( + model="gpt-4o-mini", messages=MESSAGES, stream=True + ) + next(stream) + stream.close() + stream.close() + assert closed == [True] + assert only_span(exporter).attributes[SpanAttributes.OUTPUT_VALUE] == "hello" + + +def test_retries_have_one_span(instrument): + def retry(**kwargs): + return litellm.completion(**kwargs) + + with instrument( + completion=lambda **kwargs: response(), completion_with_retries=retry + ) as (exporter, _): + litellm.completion_with_retries(model="gpt-4o-mini", messages=MESSAGES) + assert only_span(exporter).name == "completion_with_retries" + + +@pytest.mark.parametrize("async_call", [False, True]) +@pytest.mark.asyncio +async def test_real_sdk_mock_completion(instrument, async_call): + # This uses LiteLLM's own mock transport, not a mocked public API function. + with instrument() as (exporter, _): + kwargs = dict( + model="gpt-4o-mini", + messages=MESSAGES, + mock_response="hello", + api_key="synthetic-not-a-key", + ) + result = ( + await litellm.acompletion(**kwargs) + if async_call + else litellm.completion(**kwargs) + ) + assert result.choices[0].message.content == "hello" + assert only_span(exporter).attributes[SpanAttributes.OUTPUT_VALUE] == "hello" + + +def test_suppression_and_uninstrument_restore_original(instrument): + original = lambda **kwargs: response() + with instrument(completion=original) as (exporter, _): + token = context.attach(context.set_value(_SUPPRESS_INSTRUMENTATION_KEY, True)) + try: + litellm.completion(model="gpt-4o-mini", messages=MESSAGES) + finally: + context.detach(token) + assert not exporter.get_finished_spans() + assert litellm.completion is original + + +@pytest.mark.parametrize("async_call", [False, True]) +@pytest.mark.asyncio +async def test_real_sdk_stream(instrument, async_call): + with instrument() as (exporter, _): + kwargs = dict( + model="gpt-4o-mini", + messages=MESSAGES, + mock_response="hello", + api_key="synthetic-not-a-key", + stream=True, + ) + if async_call: + stream = await litellm.acompletion(**kwargs) + chunks = [chunk async for chunk in stream] + await stream.aclose() + else: + stream = litellm.completion(**kwargs) + chunks = list(stream) + stream.close() + assert chunks + assert only_span(exporter).attributes[SpanAttributes.OUTPUT_VALUE] == "hello" + + +def test_real_retry_positional_arguments(instrument): + with instrument() as (exporter, _): + result = litellm.completion_with_retries( + "gpt-4o-mini", + MESSAGES, + mock_response="hello", + api_key="synthetic-not-a-key", + ) + assert result.choices[0].message.content == "hello" + span = only_span(exporter) + assert span.attributes[SpanAttributes.GEN_AI_REQUEST_MODEL] == "gpt-4o-mini" + assert all( + m["content"] in span.attributes[SpanAttributes.INPUT_VALUE] for m in MESSAGES + ) + + +def test_uninstrument_restores_protect(monkeypatch): + original = lambda *args, **kwargs: None + protect = SimpleNamespace(protect=original) + monkeypatch.setattr(traceai_litellm, "Protect", protect) + + def install_wrapper(**kwargs): + protect.protect = lambda *args, **kwargs: original(*args, **kwargs) + + monkeypatch.setattr(traceai_litellm, "wrap_function_wrapper", install_wrapper) + provider = TracerProvider() + inst = LiteLLMInstrumentor() + try: + inst.instrument(tracer_provider=provider) + assert protect.protect is not original + inst.uninstrument() + assert protect.protect is original + finally: + if inst.is_instrumented_by_opentelemetry: + inst.uninstrument() + provider.shutdown() + + +def test_provider_credentials_are_not_exported(instrument): + import json + + seen = [] + + def completion(**kwargs): + seen.append(kwargs) + return response() + + with instrument(completion=completion) as (exporter, _): + litellm.completion( + model="gpt-4o-mini", + messages=MESSAGES, + api_key="synthetic-provider-secret", + extra_headers={"Authorization": "Bearer synthetic-header-secret"}, + ) + attrs = json.dumps(dict(only_span(exporter).attributes)) + assert "synthetic-provider-secret" not in attrs + assert "synthetic-header-secret" not in attrs + assert seen[0]["api_key"] == "synthetic-provider-secret" + assert seen[0]["extra_headers"] == { + "Authorization": "Bearer synthetic-header-secret" + } + + +def test_real_protect_descriptor_restored(monkeypatch): + import sys + import types + + class FakeProtect: + def protect(self, value): + return value + + original = FakeProtect.protect + fake_module = types.ModuleType("fi.evals") + fake_module.Protect = FakeProtect + fake_parent = types.ModuleType("fi") + fake_parent.evals = fake_module + monkeypatch.setitem(sys.modules, "fi", fake_parent) + monkeypatch.setitem(sys.modules, "fi.evals", fake_module) + monkeypatch.setattr(traceai_litellm, "Protect", FakeProtect) + provider = TracerProvider() + inst = LiteLLMInstrumentor() + try: + inst.instrument(tracer_provider=provider) + assert FakeProtect.protect is not original + inst.uninstrument() + assert FakeProtect.protect is original + assert FakeProtect().protect("ok") == "ok" + finally: + if inst.is_instrumented_by_opentelemetry: + inst.uninstrument() + provider.shutdown() diff --git a/python/frameworks/litellm/tests/test_otlp_contract.py b/python/frameworks/litellm/tests/test_otlp_contract.py new file mode 100644 index 000000000..ae688ac2b --- /dev/null +++ b/python/frameworks/litellm/tests/test_otlp_contract.py @@ -0,0 +1,68 @@ +"""Exercise the documented registration route against a loopback OTLP receiver.""" + +import importlib.util +from pathlib import Path + +import litellm +import pytest + +from fi_instrumentation import register +from fi_instrumentation.fi_types import ProjectType, SpanAttributes +from fi_instrumentation.otel import HTTPSpanExporter +from traceai_litellm import LiteLLMInstrumentor + +spec = importlib.util.spec_from_file_location( + "litellm_test_receiver", + Path(__file__).resolve().parents[3] / "tests" / "harness" / "__init__.py", +) +harness = importlib.util.module_from_spec(spec) +spec.loader.exec_module(harness) + + +@pytest.mark.parametrize("streaming", [False, True]) +def test_registered_litellm_exports_to_collector(monkeypatch, streaming): + monkeypatch.setenv("FI_API_KEY", "synthetic-api") + monkeypatch.setenv("FI_SECRET_KEY", "synthetic-secret") + with harness.Receiver() as receiver: + provider = register( + project_name="litellm-contract", + project_type=ProjectType.OBSERVE, + batch=False, + verbose=False, + span_exporter=HTTPSpanExporter(endpoint=receiver.collector_endpoint), + ) + instrumentor = LiteLLMInstrumentor() + instrumentor.instrument(tracer_provider=provider) + try: + result = litellm.completion( + model="gpt-4o-mini", + messages=[ + {"role": "system", "content": "Be helpful"}, + {"role": "user", "content": "Hello"}, + ], + mock_response="hello", + api_key="synthetic-provider-key", + stream=streaming, + ) + if streaming: + list(result) + provider.force_flush() + finally: + instrumentor.uninstrument() + provider.shutdown() + spans = receiver.spans() + assert len(spans) == 1 + assert spans[0]["status"]["code"] == "STATUS_CODE_OK" + attrs = harness._flatten_attributes(spans[0]["attributes"]) + assert attrs[SpanAttributes.GEN_AI_SPAN_KIND] == "LLM" + assert attrs[SpanAttributes.GEN_AI_REQUEST_MODEL] == "gpt-4o-mini" + assert "Be helpful" in attrs[SpanAttributes.INPUT_VALUE] + assert "Hello" in attrs[SpanAttributes.INPUT_VALUE] + assert attrs[SpanAttributes.OUTPUT_VALUE] == "hello" + request = receiver.requests()[0] + assert request["path"] == "/tracer/v1/traces" + assert request["headers"]["x-api-key"] == "synthetic-api" + assert request["headers"]["x-secret-key"] == "synthetic-secret" + resource = request["resource_attributes"][0] + assert resource["project_name"] == "litellm-contract" + assert resource["project_type"] == ProjectType.OBSERVE.value diff --git a/python/frameworks/litellm/traceai_litellm/__init__.py b/python/frameworks/litellm/traceai_litellm/__init__.py index aa3caa105..b9b534e86 100644 --- a/python/frameworks/litellm/traceai_litellm/__init__.py +++ b/python/frameworks/litellm/traceai_litellm/__init__.py @@ -1,4 +1,6 @@ import json +import inspect +from contextlib import contextmanager from enum import Enum from functools import wraps from typing import ( @@ -16,6 +18,7 @@ import litellm import logging + logger = logging.getLogger(__name__) try: from fi.evals import Protect @@ -49,6 +52,38 @@ from traceai_litellm.version import __version__ from wrapt import wrap_function_wrapper +_REQUEST_FIELDS = frozenset( + { + "model", + "messages", + "input", + "prompt", + "temperature", + "top_p", + "n", + "stream", + "stream_options", + "stop", + "max_tokens", + "max_completion_tokens", + "presence_penalty", + "frequency_penalty", + "logit_bias", + "seed", + "response_format", + "tools", + "tool_choice", + "functions", + "function_call", + "parallel_tool_calls", + "encoding_format", + "dimensions", + "size", + "quality", + "style", + } +) + # Helper functions to set span attributes def _set_span_attribute(span: trace_api.Span, name: str, value: AttributeValue) -> None: @@ -119,7 +154,9 @@ def _instrument_func_type_completion( litellm.acompletion_with_retries() (async version of completion_with_retries) """ _set_span_attribute(span, SpanAttributes.INPUT_VALUE, safe_json_dumps(kwargs)) - _set_span_attribute(span, SpanAttributes.GEN_AI_SPAN_KIND, FiSpanKindValues.LLM.value) + _set_span_attribute( + span, SpanAttributes.GEN_AI_SPAN_KIND, FiSpanKindValues.LLM.value + ) _set_span_attribute( span, SpanAttributes.GEN_AI_REQUEST_MODEL, kwargs.get("model", "unknown_model") ) @@ -147,7 +184,9 @@ def _instrument_func_type_completion( k: v for k, v in kwargs.items() if k not in ["model", "messages"] } _set_span_attribute( - span, SpanAttributes.GEN_AI_REQUEST_PARAMETERS, json.dumps(invocation_params) + span, + SpanAttributes.GEN_AI_REQUEST_PARAMETERS, + safe_json_dumps(invocation_params), ) @@ -185,7 +224,9 @@ def _instrument_func_type_image_generation( litellm.aimage_generation() (async version of image_generation) """ _set_span_attribute(span, SpanAttributes.INPUT_VALUE, safe_json_dumps(kwargs)) - _set_span_attribute(span, SpanAttributes.GEN_AI_SPAN_KIND, FiSpanKindValues.LLM.value) + _set_span_attribute( + span, SpanAttributes.GEN_AI_SPAN_KIND, FiSpanKindValues.LLM.value + ) if model := kwargs.get("model"): _set_span_attribute(span, SpanAttributes.GEN_AI_REQUEST_MODEL, model) if prompt := kwargs.get("prompt"): @@ -216,32 +257,37 @@ def _finalize_span(span: trace_api.Span, result: Any) -> None: span, SpanAttributes.EMBEDDING_EMBEDDINGS, safe_json_dumps(result_data) ) elif isinstance(result, ImageResponse): - _set_span_attribute(span, f"{SpanAttributes.GEN_AI_OUTPUT_MESSAGES}.{0}.{MessageAttributes.MESSAGE_ROLE}", "assistant") - for idx,img_data in enumerate(result.data): - _set_span_attribute(span, f"{SpanAttributes.GEN_AI_OUTPUT_MESSAGES}.{0}.{MessageAttributes.MESSAGE_CONTENT}.{idx}.{MessageContentAttributes.MESSAGE_CONTENT_TYPE}", "image") + _set_span_attribute( + span, + f"{SpanAttributes.GEN_AI_OUTPUT_MESSAGES}.{0}.{MessageAttributes.MESSAGE_ROLE}", + "assistant", + ) + for idx, img_data in enumerate(result.data): + _set_span_attribute( + span, + f"{SpanAttributes.GEN_AI_OUTPUT_MESSAGES}.{0}.{MessageAttributes.MESSAGE_CONTENT}.{idx}.{MessageContentAttributes.MESSAGE_CONTENT_TYPE}", + "image", + ) if img_data: if isinstance(img_data, Image) and ( url := (img_data.url or img_data.b64_json) ): - _set_span_attribute(span, f"{SpanAttributes.GEN_AI_OUTPUT_MESSAGES}.{0}.{MessageAttributes.MESSAGE_CONTENT}.{idx}.{MessageContentAttributes.MESSAGE_CONTENT_IMAGE}", url) + _set_span_attribute( + span, + f"{SpanAttributes.GEN_AI_OUTPUT_MESSAGES}.{0}.{MessageAttributes.MESSAGE_CONTENT}.{idx}.{MessageContentAttributes.MESSAGE_CONTENT_IMAGE}", + url, + ) _set_span_attribute(span, SpanAttributes.OUTPUT_VALUE, url) elif isinstance(img_data, dict) and ( url := (img_data.get("url") or img_data.get("b64_json")) ): - _set_span_attribute(span, f"{SpanAttributes.GEN_AI_OUTPUT_MESSAGES}.{0}.{MessageAttributes.MESSAGE_CONTENT}.{idx}.{MessageContentAttributes.MESSAGE_CONTENT_IMAGE}", url) + _set_span_attribute( + span, + f"{SpanAttributes.GEN_AI_OUTPUT_MESSAGES}.{0}.{MessageAttributes.MESSAGE_CONTENT}.{idx}.{MessageContentAttributes.MESSAGE_CONTENT_IMAGE}", + url, + ) _set_span_attribute(span, SpanAttributes.OUTPUT_VALUE, url) - if hasattr(result, "usage"): - _set_span_attribute( - span, SpanAttributes.GEN_AI_USAGE_INPUT_TOKENS, result.usage["prompt_tokens"] - ) - _set_span_attribute( - span, - SpanAttributes.GEN_AI_USAGE_OUTPUT_TOKENS, - result.usage["completion_tokens"], - ) - _set_span_attribute( - span, SpanAttributes.GEN_AI_USAGE_TOTAL_TOKENS, result.usage["total_tokens"] - ) + _set_usage(span, getattr(result, "usage", None)) class LiteLLMInstrumentor(BaseInstrumentor): # type: ignore @@ -276,14 +322,16 @@ def _instrument(self, **kwargs: Any) -> None: "aimage_generation": self._aimage_generation_wrapper, } if Protect is not None: - self._original_protect = Protect.protect + self._original_protect = inspect.getattr_static(Protect, "protect") wrap_function_wrapper( module="fi.evals", name="Protect.protect", wrapper=GuardrailProtectWrapper(tracer=self._tracer), ) + self._wrapped_protect = inspect.getattr_static(Protect, "protect") else: self._original_protect = None + self._wrapped_protect = None for func_name, func_wrapper in functions_to_instrument.items(): if hasattr(litellm, func_name): @@ -303,332 +351,325 @@ def _uninstrument(self, **kwargs: Any) -> None: ) in LiteLLMInstrumentor.original_litellm_funcs.items(): setattr(litellm, func_name, original_func) self.original_litellm_funcs.clear() + if Protect is not None and inspect.getattr_static( + Protect, "protect" + ) is getattr(self, "_wrapped_protect", None): + Protect.protect = self._original_protect + self._original_protect = None + self._wrapped_protect = None self._tracer = None + def _request(self, name, args, kwargs): + """Bind positional arguments without changing the provider call.""" + try: + signature = inspect.signature(self.original_litellm_funcs[name]) + bound = signature.bind_partial(*args, **kwargs) + request = dict(bound.arguments) + for key, param in signature.parameters.items(): + if param.kind is inspect.Parameter.VAR_KEYWORD: + request.update(request.pop(key, {})) + # Retry helpers accept only *args/**kwargs; their arguments are completion's. + if "args" in request: + request.pop("args") + request.update( + inspect.signature(self.original_litellm_funcs["completion"]) + .bind_partial(*args) + .arguments + ) + except (TypeError, ValueError): + request = dict(kwargs) + # Capture generation inputs/settings, not arbitrary SDK control objects, + # credentials, HTTP headers, callbacks, or provider-specific secrets. + return {key: value for key, value in request.items() if key in _REQUEST_FIELDS} + + def _call(self, name, annotate, args, kwargs): + original = self.original_litellm_funcs[name] + if context_api.get_value(_SUPPRESS_INSTRUMENTATION_KEY): + return original(*args, **kwargs) + request = self._request(name, args, kwargs) + span = self._tracer.start_span( + name, attributes=dict(get_attributes_from_context()) + ) + _observe(annotate, span, request) + try: + with _provider_context(span): + result = original(*args, **kwargs) + if request.get("stream"): + return StreamingIteratorWrapper(result, span) + _observe(_finalize_span, span, result) + span.set_status(trace_api.StatusCode.OK) + except BaseException as error: + _record_error(span, error) + span.end() + raise + span.end() + return result + + async def _acall(self, name, annotate, args, kwargs): + original = self.original_litellm_funcs[name] + if context_api.get_value(_SUPPRESS_INSTRUMENTATION_KEY): + return await original(*args, **kwargs) + request = self._request(name, args, kwargs) + span = self._tracer.start_span( + name, attributes=dict(get_attributes_from_context()) + ) + _observe(annotate, span, request) + try: + with _provider_context(span): + result = await original(*args, **kwargs) + if request.get("stream"): + return AsyncStreamingIteratorWrapper(result, span) + _observe(_finalize_span, span, result) + span.set_status(trace_api.StatusCode.OK) + except BaseException as error: + _record_error(span, error) + span.end() + raise + span.end() + return result + @wraps(litellm.completion) def _completion_wrapper(self, *args: Any, **kwargs: Any) -> Any: - if context_api.get_value(_SUPPRESS_INSTRUMENTATION_KEY): - return self.original_litellm_funcs["completion"](*args, **kwargs) - streaming = kwargs.get("stream", False) - if not streaming: - with self._tracer.start_as_current_span( - name="completion", attributes=dict(get_attributes_from_context()) - ) as span: - try: - _instrument_func_type_completion(span, kwargs) - result = self.original_litellm_funcs["completion"](*args, **kwargs) - _finalize_span(span, result) - span.set_status(trace_api.StatusCode.OK) - return result - except Exception as e: - span.record_exception(e) - raise - else: - span_cm = self._tracer.start_as_current_span( - name="completion", attributes=dict(get_attributes_from_context()) - ) - span = span_cm.__enter__() - try: - _instrument_func_type_completion(span, kwargs) - original_iterator = self.original_litellm_funcs["completion"]( - *args, **kwargs - ) - return StreamingIteratorWrapper(original_iterator, span, span_cm) - except Exception as e: - span.record_exception(e) - raise + return self._call("completion", _instrument_func_type_completion, args, kwargs) @wraps(litellm.acompletion) async def _acompletion_wrapper(self, *args: Any, **kwargs: Any) -> Any: - if context_api.get_value(_SUPPRESS_INSTRUMENTATION_KEY): - return await self.original_litellm_funcs["acompletion"](*args, **kwargs) - streaming = kwargs.get("stream", False) - if not streaming: - with self._tracer.start_as_current_span( - name="acompletion", attributes=dict(get_attributes_from_context()) - ) as span: - try: - _instrument_func_type_completion(span, kwargs) - result = await self.original_litellm_funcs["acompletion"]( - *args, **kwargs - ) - _finalize_span(span, result) - span.set_status(trace_api.StatusCode.OK) - return result - except Exception as e: - span.record_exception(e) - raise - else: - span_cm = self._tracer.start_as_current_span( - name="acompletion", attributes=dict(get_attributes_from_context()) - ) - span = span_cm.__enter__() - try: - _instrument_func_type_completion(span, kwargs) - original_async_iterator = await self.original_litellm_funcs[ - "acompletion" - ](*args, **kwargs) - return AsyncStreamingIteratorWrapper( - original_async_iterator, span, span_cm - ) - except Exception as e: - span.set_status( - trace_api.StatusCode.ERROR, - description=f"{e.__class__.__name__}: {e}", - ) - span.record_exception(e) - span_cm.__exit__(e, type(e), e.__traceback__) - raise + return await self._acall( + "acompletion", _instrument_func_type_completion, args, kwargs + ) @wraps(litellm.completion_with_retries) - def _completion_with_retries_wrapper( - self, *args: Any, **kwargs: Any - ) -> ModelResponse: - if context_api.get_value(_SUPPRESS_INSTRUMENTATION_KEY): - return self.original_litellm_funcs["completion_with_retries"]( - *args, **kwargs - ) # type:ignore - with self._tracer.start_as_current_span( - name="completion_with_retries", - attributes=dict(get_attributes_from_context()), - ) as span: - try: - _instrument_func_type_completion(span, kwargs) - result = self.original_litellm_funcs["completion_with_retries"]( - *args, **kwargs - ) - _finalize_span(span, result) - span.set_status(trace_api.StatusCode.OK) - return result # type:ignore - except Exception as e: - span.record_exception(e) - raise + def _completion_with_retries_wrapper(self, *args: Any, **kwargs: Any) -> Any: + return self._call( + "completion_with_retries", _instrument_func_type_completion, args, kwargs + ) @wraps(litellm.acompletion_with_retries) - async def _acompletion_with_retries_wrapper( - self, *args: Any, **kwargs: Any - ) -> ModelResponse: - if context_api.get_value(_SUPPRESS_INSTRUMENTATION_KEY): - return await self.original_litellm_funcs["acompletion_with_retries"]( - *args, **kwargs - ) # type:ignore - with self._tracer.start_as_current_span( - name="acompletion_with_retries", - attributes=dict(get_attributes_from_context()), - ) as span: - try: - _instrument_func_type_completion(span, kwargs) - result = await self.original_litellm_funcs[ - "acompletion_with_retries" - ](*args, **kwargs) - _finalize_span(span, result) - span.set_status(trace_api.StatusCode.OK) - return result # type:ignore - except Exception as e: - span.record_exception(e) - raise + async def _acompletion_with_retries_wrapper(self, *args: Any, **kwargs: Any) -> Any: + return await self._acall( + "acompletion_with_retries", _instrument_func_type_completion, args, kwargs + ) @wraps(litellm.embedding) - def _embedding_wrapper(self, *args: Any, **kwargs: Any) -> EmbeddingResponse: - if context_api.get_value(_SUPPRESS_INSTRUMENTATION_KEY): - return self.original_litellm_funcs["embedding"]( - *args, **kwargs - ) # type:ignore - with self._tracer.start_as_current_span( - name="embedding", attributes=dict(get_attributes_from_context()) - ) as span: - try: - _instrument_func_type_embedding(span, kwargs) - result = self.original_litellm_funcs["embedding"](*args, **kwargs) - _finalize_span(span, result) - span.set_status(trace_api.StatusCode.OK) - return result # type:ignore - except Exception as e: - span.record_exception(e) - raise + def _embedding_wrapper(self, *args: Any, **kwargs: Any) -> Any: + return self._call("embedding", _instrument_func_type_embedding, args, kwargs) @wraps(litellm.aembedding) - async def _aembedding_wrapper(self, *args: Any, **kwargs: Any) -> EmbeddingResponse: - if context_api.get_value(_SUPPRESS_INSTRUMENTATION_KEY): - return await self.original_litellm_funcs["aembedding"]( - *args, **kwargs - ) # type:ignore - with self._tracer.start_as_current_span( - name="aembedding", attributes=dict(get_attributes_from_context()) - ) as span: - try: - _instrument_func_type_embedding(span, kwargs) - result = await self.original_litellm_funcs["aembedding"]( - *args, **kwargs - ) - _finalize_span(span, result) - span.set_status(trace_api.StatusCode.OK) - return result # type:ignore - except Exception as e: - span.record_exception(e) - raise + async def _aembedding_wrapper(self, *args: Any, **kwargs: Any) -> Any: + return await self._acall( + "aembedding", _instrument_func_type_embedding, args, kwargs + ) @wraps(litellm.image_generation) - def _image_generation_wrapper(self, *args: Any, **kwargs: Any) -> ImageResponse: - if context_api.get_value(_SUPPRESS_INSTRUMENTATION_KEY): - return self.original_litellm_funcs["image_generation"]( - *args, **kwargs - ) # type:ignore - with self._tracer.start_as_current_span( - name="image_generation", attributes=dict(get_attributes_from_context()) - ) as span: - try: - _instrument_func_type_image_generation(span, kwargs) - result = self.original_litellm_funcs["image_generation"]( - *args, **kwargs - ) - _finalize_span(span, result) - span.set_status(trace_api.StatusCode.OK) - return result # type:ignore - except Exception as e: - span.record_exception(e) - raise + def _image_generation_wrapper(self, *args: Any, **kwargs: Any) -> Any: + return self._call( + "image_generation", _instrument_func_type_image_generation, args, kwargs + ) @wraps(litellm.aimage_generation) - async def _aimage_generation_wrapper( - self, *args: Any, **kwargs: Any - ) -> ImageResponse: - if context_api.get_value(_SUPPRESS_INSTRUMENTATION_KEY): - return await self.original_litellm_funcs["aimage_generation"]( - *args, **kwargs - ) # type:ignore - with self._tracer.start_as_current_span( - name="aimage_generation", attributes=dict(get_attributes_from_context()) - ) as span: - try: - _instrument_func_type_image_generation(span, kwargs) - result = await self.original_litellm_funcs["aimage_generation"]( - *args, **kwargs - ) - _finalize_span(span, result) - span.set_status(trace_api.StatusCode.OK) - return result # type:ignore - except Exception as e: - span.record_exception(e) - raise + async def _aimage_generation_wrapper(self, *args: Any, **kwargs: Any) -> Any: + return await self._acall( + "aimage_generation", _instrument_func_type_image_generation, args, kwargs + ) def _set_wrapper_attr(self, func_wrapper: Any) -> None: func_wrapper.__func__.is_wrapper = True -class StreamingIteratorWrapper: - def __init__(self, iterator: Iterator[Any], span: trace_api.Span, span_cm): - self.iterator = iterator +def _observe(function, *args): + """Telemetry extraction must not change a successful SDK operation.""" + try: + function(*args) + except Exception: + # Do not log request, response, or exception content (which can include secrets). + logger.debug("Could not extract LiteLLM telemetry") + + +@contextmanager +def _provider_context(span): + # Keep a span current only during provider work, never across a yielded chunk. + # Suppress nested LiteLLM/OpenAI instrumentation for the same request. + with trace_api.use_span( + span, end_on_exit=False, record_exception=False, set_status_on_exception=False + ): + token = context_api.attach( + context_api.set_value(_SUPPRESS_INSTRUMENTATION_KEY, True) + ) + try: + yield + finally: + context_api.detach(token) + + +def _record_error(span, error): + span.set_status(trace_api.StatusCode.ERROR) + span.record_exception(error) + + +def _set_usage(span, usage): + if hasattr(usage, "model_dump"): + usage = usage.model_dump() + if not isinstance(usage, Mapping): + return + for key, attribute in ( + ("prompt_tokens", SpanAttributes.GEN_AI_USAGE_INPUT_TOKENS), + ("completion_tokens", SpanAttributes.GEN_AI_USAGE_OUTPUT_TOKENS), + ("total_tokens", SpanAttributes.GEN_AI_USAGE_TOTAL_TOKENS), + ): + _set_span_attribute(span, attribute, usage.get(key)) + + +class _StreamState: + def __init__(self, span, span_cm=None): self.span = span self.span_cm = span_cm self.processed_chunks = [] self.content_string = "" + self._ended = False + self._closed = False + + def _process(self, chunk): + if hasattr(chunk, "to_dict"): + chunk = chunk.to_dict() + elif hasattr(chunk, "__dict__"): + chunk = chunk.__dict__ + if not isinstance(chunk, Mapping): + return + self.processed_chunks.append(chunk) + choices = chunk.get("choices") or [] + if choices: + delta = choices[0].get("delta") or {} + content = delta.get("content") + if isinstance(content, str): + self.content_string += content + _set_usage(self.span, chunk.get("usage")) + _set_span_attribute( + self.span, SpanAttributes.GEN_AI_RESPONSE_MODEL, chunk.get("model") + ) - def __iter__(self): - return self - - def __next__(self): + def _finish(self, error=None): + if self._ended: + return + self._ended = True try: - chunk = next(self.iterator) - # Process chunk - if hasattr(chunk, "to_dict"): - chunk_dict = chunk.to_dict() - elif hasattr(chunk, "__dict__"): - chunk_dict = chunk.__dict__ + _set_span_attribute( + self.span, + SpanAttributes.OUTPUT_VALUE, + self.content_string or safe_json_dumps(self.processed_chunks), + ) + if error is None: + self.span.set_status(trace_api.StatusCode.OK) else: - chunk_dict = chunk + _record_error(self.span, error) + finally: + if self.span_cm is not None: + self.span_cm.__exit__( + type(error) if error else None, + error, + error.__traceback__ if error else None, + ) + else: + self.span.end() - self.processed_chunks.append(chunk_dict) - if ( - content := chunk_dict.get("choices", [{}])[0] - .get("delta", {}) - .get("content") - ): - self.content_string += content +class StreamingIteratorWrapper(_StreamState): + def __init__(self, iterator: Iterator[Any], span: trace_api.Span, span_cm=None): + super().__init__(span, span_cm) + self.iterator = iterator - return chunk + def __iter__(self): + return self + + def __next__(self): + if self._ended: + raise StopIteration + try: + with _provider_context(self.span): + chunk = next(self.iterator) except StopIteration: - # After the iterator is exhausted, set the span attributes - _set_span_attribute( - self.span, - SpanAttributes.OUTPUT_VALUE, - safe_json_dumps(self.processed_chunks), - ) - _set_span_attribute( - self.span, SpanAttributes.OUTPUT_VALUE, self.content_string - ) - self.span.set_status(trace_api.StatusCode.OK) - # End the span - self.span_cm.__exit__(None, None, None) + self._finish() + raise + except BaseException as error: + self._finish(error) raise - except Exception as e: - self.span.record_exception(e) + _observe(self._process, chunk) + return chunk + + def close(self): + if self._closed: + return + self._closed = True + try: + close = getattr(self.iterator, "close", None) + if close is not None: + with _provider_context(self.span): + close() + except BaseException as error: + self._finish(error) raise + finally: + self._finish() def __enter__(self): return self def __exit__(self, exc_type, exc_value, traceback): - # If the iterator exits early, we need to close the span - self.span_cm.__exit__(exc_type, exc_value, traceback) + if exc_value is not None: + self._finish(exc_value) + self.close() -class AsyncStreamingIteratorWrapper: +class AsyncStreamingIteratorWrapper(_StreamState): def __init__( - self, async_iterator: AsyncIterator[Any], span: trace_api.Span, span_cm + self, async_iterator: AsyncIterator[Any], span: trace_api.Span, span_cm=None ): + super().__init__(span, span_cm) self.async_iterator = async_iterator - self.span = span - self.span_cm = span_cm - self.processed_chunks = [] - self.content_string = "" def __aiter__(self): return self async def __anext__(self): + if self._ended: + raise StopAsyncIteration try: - chunk = await self.async_iterator.__anext__() - # Process chunk - if hasattr(chunk, "to_dict"): - chunk_dict = chunk.to_dict() - elif hasattr(chunk, "__dict__"): - chunk_dict = chunk.__dict__ - else: - chunk_dict = chunk - - self.processed_chunks.append(chunk_dict) - - if ( - content := chunk_dict.get("choices", [{}])[0] - .get("delta", {}) - .get("content") - ): - self.content_string += content - - return chunk + with _provider_context(self.span): + chunk = await self.async_iterator.__anext__() except StopAsyncIteration: - # After the iterator is exhausted, set the span attributes - _set_span_attribute( - self.span, - SpanAttributes.OUTPUT_VALUE, - safe_json_dumps(self.processed_chunks), - ) - _set_span_attribute( - self.span, SpanAttributes.OUTPUT_VALUE, self.content_string - ) - self.span.set_status(trace_api.StatusCode.OK) - # End the span - self.span_cm.__exit__(None, None, None) + self._finish() + raise + except BaseException as error: + self._finish(error) raise - except Exception as e: - self.span.record_exception(e) + _observe(self._process, chunk) + return chunk + + async def aclose(self): + if self._closed: + return + self._closed = True + try: + close = getattr(self.async_iterator, "aclose", None) + if close is not None: + with _provider_context(self.span): + await close() + except BaseException as error: + self._finish(error) raise + finally: + self._finish() async def close(self): - # Explicitly close the span if needed - self.span_cm.__exit__(None, None, None) + # Compatibility with the previous wrapper's async close() method. + await self.aclose() + + async def __aenter__(self): + return self + + async def __aexit__(self, exc_type, exc_value, traceback): + if exc_value is not None: + self._finish(exc_value) + await self.aclose() def _set_raw_output(span: trace_api.Span, result: Any) -> None: From 8889ad120a6f0bbd4415851ecac9fa5e6ac73487 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Mon, 2 Nov 2026 00:53:21 -0800 Subject: [PATCH 03/11] fix(litellm): close provider streams and preserve private structured context Address recall review: keep embedding vectors exclusively in privacy-filtered attributes, preserve completed response types and tool conversations, bind async endpoint arguments correctly, and declare/test usable runtime dependencies and provider credentials. Co-Authored-By: Claude Code --- .github/workflows/litellm-python-tests.yml | 9 +- python/frameworks/litellm/README.md | 12 +- python/frameworks/litellm/poetry.lock | 14 +- python/frameworks/litellm/pyproject.toml | 4 +- .../litellm/tests/test_install_contract.py | 40 +++ .../litellm/tests/test_provider_contract.py | 322 ++++++++++++++++++ .../litellm/tests/test_readme_example.py | 63 ++++ .../litellm/traceai_litellm/__init__.py | 119 +++++-- 8 files changed, 538 insertions(+), 45 deletions(-) create mode 100644 python/frameworks/litellm/tests/test_install_contract.py create mode 100644 python/frameworks/litellm/tests/test_provider_contract.py create mode 100644 python/frameworks/litellm/tests/test_readme_example.py diff --git a/.github/workflows/litellm-python-tests.yml b/.github/workflows/litellm-python-tests.yml index da97fe33a..c59a24484 100644 --- a/.github/workflows/litellm-python-tests.yml +++ b/.github/workflows/litellm-python-tests.yml @@ -49,12 +49,13 @@ jobs: python -m venv .venv if [ "$SDK" = floor ]; then # PyPI no longer serves 1.43.0; use its immutable upstream release commit. - .venv/bin/pip install -e python -e python/frameworks/litellm \ + .venv/bin/pip install -e python/frameworks/litellm \ 'litellm @ git+https://github.com/BerriAI/litellm.git@4603eedcc13bfab2da5a82aed209e6ef496c384d' \ - 'openai==1.40.0' 'httpx==0.27.2' pytest pytest-asyncio tenacity + 'fi-instrumentation-otel==1.2.2' 'openai==1.40.0' 'httpx==0.27.2' \ + pytest pytest-asyncio else - .venv/bin/pip install -e python -e python/frameworks/litellm \ - --upgrade litellm pytest pytest-asyncio tenacity + .venv/bin/pip install -e python/frameworks/litellm \ + --upgrade litellm pytest pytest-asyncio fi .venv/bin/pip check - name: Run with outbound network disabled diff --git a/python/frameworks/litellm/README.md b/python/frameworks/litellm/README.md index 3bd1c6770..10ce1dc38 100644 --- a/python/frameworks/litellm/README.md +++ b/python/frameworks/litellm/README.md @@ -12,14 +12,14 @@ pip install traceAI-litellm ``` -### Set Environment Variables -Set up your environment variables to authenticate with FutureAGI +### Set environment variables -```python -import os +Set your Future AGI credentials for telemetry and your own OpenAI API key for the examples below. Future AGI credentials do not authenticate model inference; OpenAI bills inference to your OpenAI account. When you use another provider through LiteLLM, configure that provider's credentials instead. -os.environ["FI_API_KEY"] = FI_API_KEY -os.environ["FI_SECRET_KEY"] = FI_SECRET_KEY +```bash +export FI_API_KEY="your-future-agi-api-key" +export FI_SECRET_KEY="your-future-agi-secret-key" +export OPENAI_API_KEY="your-openai-api-key" ``` ## Quickstart diff --git a/python/frameworks/litellm/poetry.lock b/python/frameworks/litellm/poetry.lock index b033e2216..33b20538d 100644 --- a/python/frameworks/litellm/poetry.lock +++ b/python/frameworks/litellm/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 2.1.2 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.1.3 and should not be changed by hand. [[package]] name = "aiohappyeyeballs" @@ -378,14 +378,14 @@ test = ["pytest (>=6)"] [[package]] name = "fi-instrumentation-otel" -version = "1.1.0" -description = "OpenTelemetry instrumentation" +version = "1.2.2" +description = "OpenTelemetry tracing for AI applications: tracer setup, span helpers, exporters, and semantic conventions for Future AGI." optional = false python-versions = "<3.14,>3.9" groups = ["main"] files = [ - {file = "fi_instrumentation_otel-1.1.0-py3-none-any.whl", hash = "sha256:4fa2bf663e0ce09447aca92ae798cd7b0dee35c5c92c2e033f1d89d158a9e62c"}, - {file = "fi_instrumentation_otel-1.1.0.tar.gz", hash = "sha256:7ec324c3c9242aa87a88cf23a21c11919b8736c6451c5fb50917f592a853b059"}, + {file = "fi_instrumentation_otel-1.2.2-py3-none-any.whl", hash = "sha256:25cf88f93e33aa928d4fae0f631afe8a32565639490862b715869e0633becbaf"}, + {file = "fi_instrumentation_otel-1.2.2.tar.gz", hash = "sha256:b675df47d7a03c506fbd2e4b0420fe4f74d12a6353a88b219bfd97797b276d4c"}, ] [package.dependencies] @@ -1928,7 +1928,7 @@ version = "9.1.2" description = "Retry code until it succeeds" optional = false python-versions = ">=3.9" -groups = ["dev"] +groups = ["main"] files = [ {file = "tenacity-9.1.2-py3-none-any.whl", hash = "sha256:f77bf36710d8b73a50b2dd155c97b870017ad21afe6ab300326b0371b3b05138"}, {file = "tenacity-9.1.2.tar.gz", hash = "sha256:1169d376c297e7de388d18b4481760d478b0e99a777cad3a9c86e556f4b697cb"}, @@ -2317,4 +2317,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.1" python-versions = ">3.9, <3.14" -content-hash = "a46fea14faa1d824f4f7d79d660f573cb8100eb8737660f0b43556872b53e6bf" +content-hash = "1c9c37790807d6f7e6de6462eb0d25a17a72af60d42e48e98523bf900dfb4ff5" diff --git a/python/frameworks/litellm/pyproject.toml b/python/frameworks/litellm/pyproject.toml index 79bc049a4..2d33fc305 100644 --- a/python/frameworks/litellm/pyproject.toml +++ b/python/frameworks/litellm/pyproject.toml @@ -30,9 +30,7 @@ classifiers = [ [tool.poetry.dependencies] python = ">3.9, <3.14" litellm = ">=1.43.0" -fi-instrumentation-otel = ">=0.1.11" - -[tool.poetry.group.dev.dependencies] +fi-instrumentation-otel = ">=1.2.2" # LiteLLM imports tenacity lazily for its optional retry helpers. tenacity = ">=8.2.3" diff --git a/python/frameworks/litellm/tests/test_install_contract.py b/python/frameworks/litellm/tests/test_install_contract.py new file mode 100644 index 000000000..19782dace --- /dev/null +++ b/python/frameworks/litellm/tests/test_install_contract.py @@ -0,0 +1,40 @@ +"""Installed distribution and documented setup must support the public examples.""" + +from importlib.metadata import requires, version +from pathlib import Path + +import litellm +from packaging.requirements import Requirement +from packaging.version import Version + + +def test_required_core_conventions_are_in_the_declared_floor(): + core = next( + Requirement(value) + for value in requires("traceAI-litellm") + if Requirement(value).name == "fi-instrumentation-otel" + ) + assert Version("0.1.11") not in core.specifier + assert Version("1.2.2") in core.specifier + + +def test_retry_helper_works_with_runtime_dependencies(): + dependencies = [Requirement(value) for value in requires("traceAI-litellm")] + assert any(dep.name == "tenacity" and dep.marker is None for dep in dependencies) + assert Version(version("tenacity")) >= Version("8.2.3") + response = litellm.completion_with_retries( + model="gpt-4o-mini", + messages=[{"role": "user", "content": "hello"}], + api_key="synthetic-key", + mock_response="hello", + ) + assert response.choices[0].message.content == "hello" + + +def test_readme_separates_provider_and_telemetry_credentials(): + readme = (Path(__file__).parents[1] / "README.md").read_text() + setup = readme.split("## Quickstart")[0] + assert "export OPENAI_API_KEY=" in setup + assert "export FI_API_KEY=" in setup + assert "export FI_SECRET_KEY=" in setup + assert "not" in setup and "inference" in setup diff --git a/python/frameworks/litellm/tests/test_provider_contract.py b/python/frameworks/litellm/tests/test_provider_contract.py new file mode 100644 index 000000000..3341477b4 --- /dev/null +++ b/python/frameworks/litellm/tests/test_provider_contract.py @@ -0,0 +1,322 @@ +"""Real LiteLLM/OpenAI objects with synthetic HTTP transports; no provider traffic.""" + +import copy +import json +from contextlib import contextmanager + +import httpx +import litellm +import openai +import pytest +from fi_instrumentation import TraceConfig +from fi_instrumentation.fi_types import SpanAttributes +from litellm.types.utils import EmbeddingResponse, ModelResponse +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +from opentelemetry.trace import StatusCode + +import traceai_litellm +from traceai_litellm import LiteLLMInstrumentor + +MESSAGES = [{"role": "user", "content": "synthetic-input"}] + + +@pytest.fixture +def capture(monkeypatch): + monkeypatch.setattr(traceai_litellm, "Protect", None) + + @contextmanager + def setup(config=None): + exporter = InMemorySpanExporter() + provider = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + inst = LiteLLMInstrumentor() + inst.instrument(tracer_provider=provider, config=config or TraceConfig()) + try: + yield exporter + finally: + inst.uninstrument() + provider.shutdown() + + return setup + + +def attributes(exporter): + spans = exporter.get_finished_spans() + assert len(spans) == 1 + assert spans[0].status.status_code is StatusCode.OK + return dict(spans[0].attributes) + + +class Bytes(httpx.SyncByteStream): + def __init__(self): + self.close_count = 0 + + def __iter__(self): + for text in ("first", "second"): + obj = { + "id": "synthetic", + "object": "chat.completion.chunk", + "created": 1, + "model": "gpt-4o-mini", + "choices": [ + { + "index": 0, + "delta": {"role": "assistant", "content": text}, + "finish_reason": None, + } + ], + } + yield ("data: " + json.dumps(obj) + "\n\n").encode() + yield b"data: [DONE]\n\n" + + def close(self): + self.close_count += 1 + + +class AsyncBytes(httpx.AsyncByteStream): + def __init__(self): + self.close_count = 0 + + async def __aiter__(self): + for chunk in Bytes(): + yield chunk + + async def aclose(self): + self.close_count += 1 + + +@pytest.mark.parametrize("consume_first", [False, True]) +def test_real_sync_provider_stream_early_close(capture, consume_first): + body = Bytes() + transport = httpx.MockTransport( + lambda request: httpx.Response( + 200, headers={"content-type": "text/event-stream"}, stream=body + ) + ) + with openai.OpenAI( + api_key="synthetic-key", http_client=httpx.Client(transport=transport) + ) as client: + with capture() as exporter: + stream = litellm.completion( + model="gpt-4o-mini", messages=MESSAGES, stream=True, client=client + ) + if consume_first: + next(stream) + stream.close() + stream.close() + assert body.close_count == 1 + attributes(exporter) + + +@pytest.mark.parametrize("consume_first", [False, True]) +@pytest.mark.asyncio +async def test_real_async_provider_stream_early_close(capture, consume_first): + body = AsyncBytes() + transport = httpx.MockTransport( + lambda request: httpx.Response( + 200, headers={"content-type": "text/event-stream"}, stream=body + ) + ) + async with openai.AsyncOpenAI( + api_key="synthetic-key", http_client=httpx.AsyncClient(transport=transport) + ) as client: + with capture() as exporter: + stream = await litellm.acompletion( + model="gpt-4o-mini", messages=MESSAGES, stream=True, client=client + ) + if consume_first: + await stream.__anext__() + await stream.aclose() + await stream.aclose() + assert body.close_count == 1 + attributes(exporter) + + +@pytest.mark.asyncio +async def test_real_async_embedding_positional_input(capture): + response = { + "object": "list", + "model": "text-embedding-ada-002", + "data": [{"object": "embedding", "index": 0, "embedding": [0.12345, 0.56789]}], + "usage": {"prompt_tokens": 1, "total_tokens": 1}, + } + transport = httpx.MockTransport(lambda request: httpx.Response(200, json=response)) + async with openai.AsyncOpenAI( + api_key="synthetic-key", http_client=httpx.AsyncClient(transport=transport) + ) as client: + with capture() as exporter: + result = await litellm.aembedding( + "text-embedding-ada-002", ["synthetic-embedding-input"], client=client + ) + assert result.data + attrs = attributes(exporter) + assert "synthetic-embedding-input" in attrs[SpanAttributes.INPUT_VALUE] + assert ( + "synthetic-embedding-input" + in attrs["embedding.embeddings.0.embedding.text"] + ) + assert attrs[SpanAttributes.GEN_AI_SPAN_KIND] == "EMBEDDING" + assert attrs[SpanAttributes.EMBEDDING_MODEL_NAME] == "text-embedding-ada-002" + assert attrs[SpanAttributes.GEN_AI_USAGE_TOTAL_TOKENS] == 1 + + +def test_real_complete_response_type_preserved(capture): + kwargs = dict( + model="gpt-4o-mini", + messages=MESSAGES, + stream=True, + complete_response=True, + mock_response="hello", + api_key="synthetic-key", + ) + baseline = litellm.completion(**kwargs) + with capture() as exporter: + result = litellm.completion(**kwargs) + assert type(result) is type(baseline) + assert result.choices[0].message.content == baseline.choices[0].message.content + assert attributes(exporter)[SpanAttributes.OUTPUT_VALUE] == "hello" + + +@pytest.mark.parametrize("hide", [False, True]) +@pytest.mark.parametrize("from_environment", [False, True]) +def test_embedding_vectors_only_use_privacy_filtered_attributes( + capture, monkeypatch, hide, from_environment +): + data = [ + {"object": "embedding", "index": 0, "embedding": [0.12345, 0.56789]}, + {"object": "embedding", "index": 1, "embedding": [0.98765, 0.43210]}, + ] + response = EmbeddingResponse( + model="text-embedding-ada-002", + data=copy.deepcopy(data), + usage={"prompt_tokens": 1, "total_tokens": 1}, + ) + monkeypatch.setattr(litellm, "embedding", lambda **kwargs: response) + if from_environment: + monkeypatch.setenv("FI_HIDE_EMBEDDING_VECTORS", str(hide).lower()) + config = TraceConfig() + else: + config = TraceConfig(hide_embedding_vectors=hide) + with capture(config) as exporter: + result = litellm.embedding(model="text-embedding-ada-002", input=["one", "two"]) + assert result is response + assert result.data == data + attrs = attributes(exporter) + assert SpanAttributes.OUTPUT_VALUE not in attrs + assert SpanAttributes.EMBEDDING_EMBEDDINGS not in attrs + for index, item in enumerate(data): + key = f"embedding.embeddings.{index}.embedding.vector" + if hide: + assert key not in attrs + for value in item["embedding"]: + assert str(value) not in json.dumps(attrs) + else: + assert json.loads(attrs[key]) == item["embedding"] + assert attrs[SpanAttributes.GEN_AI_USAGE_TOTAL_TOKENS] == 1 + + +def tool_call(suffix, arguments): + return { + "id": f"call_synthetic_{suffix}", + "type": "function", + "function": {"name": f"lookup_{suffix}", "arguments": arguments}, + } + + +def test_tool_calls_preserved(capture, monkeypatch): + response = ModelResponse( + model="gpt-4o-mini", + choices=[ + { + "message": { + "role": "assistant", + "content": "Calling a tool", + "tool_calls": [tool_call("out", '{"x":2}')], + } + } + ], + ) + monkeypatch.setattr(litellm, "completion", lambda **kwargs: response) + messages = MESSAGES + [ + { + "role": "assistant", + "content": None, + "tool_calls": [tool_call("in", '{"x":1}')], + }, + {"role": "tool", "tool_call_id": "call_synthetic_in", "content": "tool result"}, + ] + original_messages = copy.deepcopy(messages) + with capture() as exporter: + assert litellm.completion(model="gpt-4o-mini", messages=messages) is response + assert messages == original_messages + attrs = attributes(exporter) + assert json.loads(attrs[SpanAttributes.INPUT_VALUE]) == messages + output = json.loads(attrs[SpanAttributes.OUTPUT_VALUE])["choices"][0]["message"] + assert output["content"] == "Calling a tool" + assert output["tool_calls"][0]["function"]["name"] == "lookup_out" + for direction, index, suffix, arguments in ( + ("input", 1, "in", '{"x":1}'), + ("output", 0, "out", '{"x":2}'), + ): + prefix = f"gen_ai.{direction}.messages.{index}.message.tool_calls.0" + assert attrs[f"{prefix}.tool_call.id"] == f"call_synthetic_{suffix}" + assert attrs[f"{prefix}.tool_call.function.name"] == f"lookup_{suffix}" + assert attrs[f"{prefix}.tool_call.function.arguments"] == arguments + assert attrs["gen_ai.input.messages.2.message.tool_call_id"] == "call_synthetic_in" + + +@pytest.mark.parametrize("async_call", [False, True]) +@pytest.mark.asyncio +async def test_stream_mixed_text_and_tools_preserved(capture, monkeypatch, async_call): + chunks = [ + { + "choices": [ + { + "delta": { + "content": "Calling a tool", + "tool_calls": [tool_call("out", '{"x":')], + } + } + ] + }, + { + "choices": [ + { + "delta": { + "tool_calls": [{"index": 0, "function": {"arguments": "2}"}}] + } + } + ] + }, + ] + + def completion(**kwargs): + return iter(chunks) + + async def async_chunks(): + for chunk in chunks: + yield chunk + + async def acompletion(**kwargs): + return async_chunks() + + monkeypatch.setattr(litellm, "completion", completion) + monkeypatch.setattr(litellm, "acompletion", acompletion) + with capture() as exporter: + if async_call: + stream = await litellm.acompletion( + model="gpt-4o-mini", messages=MESSAGES, stream=True + ) + assert [chunk async for chunk in stream] == chunks + else: + assert ( + list( + litellm.completion( + model="gpt-4o-mini", messages=MESSAGES, stream=True + ) + ) + == chunks + ) + assert json.loads(attributes(exporter)[SpanAttributes.OUTPUT_VALUE]) == chunks diff --git a/python/frameworks/litellm/tests/test_readme_example.py b/python/frameworks/litellm/tests/test_readme_example.py new file mode 100644 index 000000000..675dab3c4 --- /dev/null +++ b/python/frameworks/litellm/tests/test_readme_example.py @@ -0,0 +1,63 @@ +"""Execute the README's first request with only its documented credential names.""" + +import ast +import re +from pathlib import Path + +import httpx +import litellm +import openai + + +def test_readme_first_request(monkeypatch): + readme = (Path(__file__).parents[1] / "README.md").read_text() + for name in ("OPENAI_API_KEY", "FI_API_KEY", "FI_SECRET_KEY"): + monkeypatch.delenv(name, raising=False) + setup = readme.split("## Quickstart")[0] + for name in re.findall(r"^export ([A-Z_]+)=", setup, re.MULTILINE): + monkeypatch.setenv(name, "synthetic-" + name.lower()) + + example = None + for block in re.findall(r"```python\n(.*?)```", readme, re.DOTALL): + module = ast.parse(block) + compile(module, "README.md", "exec") + for node in module.body: + if isinstance(node, ast.AsyncFunctionDef) and node.name == "run_examples": + example = node.body[0].value + assert example is not None + kwargs = { + argument.arg: ast.literal_eval(argument.value) for argument in example.keywords + } + requests = [] + + def respond(request): + requests.append(request) + assert request.headers["authorization"] == "Bearer synthetic-openai_api_key" + return httpx.Response( + 200, + json={ + "id": "synthetic", + "object": "chat.completion", + "created": 1, + "model": kwargs["model"], + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "Beijing"}, + "finish_reason": "stop", + } + ], + "usage": { + "prompt_tokens": 2, + "completion_tokens": 1, + "total_tokens": 3, + }, + }, + ) + + with openai.OpenAI( + http_client=httpx.Client(transport=httpx.MockTransport(respond)) + ) as client: + response = litellm.completion(**kwargs, client=client) + assert response.choices[0].message.content == "Beijing" + assert len(requests) == 1 diff --git a/python/frameworks/litellm/traceai_litellm/__init__.py b/python/frameworks/litellm/traceai_litellm/__init__.py index b9b534e86..1d42fdc06 100644 --- a/python/frameworks/litellm/traceai_litellm/__init__.py +++ b/python/frameworks/litellm/traceai_litellm/__init__.py @@ -39,6 +39,7 @@ MessageAttributes, MessageContentAttributes, SpanAttributes, + ToolCallAttributes, ) from fi_instrumentation.instrumentation._protect_wrapper import GuardrailProtectWrapper from litellm.types.utils import Choices, EmbeddingResponse, ImageResponse, ModelResponse @@ -101,6 +102,8 @@ def is_iterable_of(lst: Iterable[object], tp: T) -> bool: def _get_attributes_from_message_param( message: Mapping[str, Any], ) -> Iterator[Tuple[str, AttributeValue]]: + if hasattr(message, "model_dump"): + message = message.model_dump() if not hasattr(message, "get"): return if role := message.get("role"): @@ -117,6 +120,34 @@ def _get_attributes_from_message_param( for key, value in _get_attributes_from_message_content(c): yield f"{MessageAttributes.MESSAGE_CONTENTS}.{index}.{key}", value + for field, attribute in ( + ("name", MessageAttributes.MESSAGE_NAME), + ("tool_call_id", MessageAttributes.MESSAGE_TOOL_CALL_ID), + ): + if value := message.get(field): + yield attribute, value + if function := message.get("function_call"): + if name := function.get("name"): + yield MessageAttributes.MESSAGE_FUNCTION_CALL_NAME, name + if arguments := function.get("arguments"): + yield MessageAttributes.MESSAGE_FUNCTION_CALL_ARGUMENTS_JSON, arguments + for index, tool in enumerate(message.get("tool_calls") or []): + prefix = f"{MessageAttributes.MESSAGE_TOOL_CALLS}.{index}" + if tool_id := tool.get("id"): + yield f"{prefix}.{ToolCallAttributes.TOOL_CALL_ID}", tool_id + function = tool.get("function") or {} + if name := function.get("name"): + yield f"{prefix}.{ToolCallAttributes.TOOL_CALL_FUNCTION_NAME}", name + if arguments := function.get("arguments"): + yield f"{prefix}.{ToolCallAttributes.TOOL_CALL_FUNCTION_ARGUMENTS_JSON}", arguments + + +def _has_tool_context(message): + return any( + message.get(key) + for key in ("tool_calls", "tool_call_id", "function_call", "name") + ) + def _get_attributes_from_message_content( content: Mapping[str, Any], @@ -171,7 +202,12 @@ def _instrument_func_type_completion( _set_span_attribute( span, SpanAttributes.INPUT_IMAGES, json.dumps(input_images) ) - if eval_input := process_messages.get("eval_input"): + if any(_has_tool_context(message) for message in messages): + # Keep tool IDs, arguments and results connected in the full conversation. + _set_span_attribute( + span, SpanAttributes.INPUT_VALUE, safe_json_dumps(messages) + ) + elif eval_input := process_messages.get("eval_input"): _set_span_attribute(span, SpanAttributes.INPUT_VALUE, eval_input) for index, input_message in list(enumerate(messages)): @@ -234,27 +270,28 @@ def _instrument_func_type_image_generation( def _finalize_span(span: trace_api.Span, result: Any) -> None: - _set_raw_output(span, result) + # Embedding vectors must only use the canonical, privacy-filtered attributes. + # Serializing the response elsewhere bypasses FI_HIDE_EMBEDDING_VECTORS. + if not isinstance(result, EmbeddingResponse): + _set_raw_output(span, result) if isinstance(result, ModelResponse): if (choices := result.choices) and len(choices) > 0: choice = choices[0] if isinstance(choice, Choices): - if output := choice.message.content: - _set_span_attribute(span, SpanAttributes.OUTPUT_VALUE, output) + if choice.message.content and not _has_tool_context(choice.message): + _set_span_attribute( + span, SpanAttributes.OUTPUT_VALUE, choice.message.content + ) for key, value in _get_attributes_from_message_param(choice.message): _set_span_attribute( span, f"{SpanAttributes.GEN_AI_OUTPUT_MESSAGES}.0.{key}", value ) elif isinstance(result, EmbeddingResponse): - if result_data := result.data: - first_embedding = result_data[0] + for index, embedding in enumerate(result.data or []): _set_span_attribute( span, - f"{SpanAttributes.EMBEDDING_EMBEDDINGS}.{0}.{EmbeddingAttributes.EMBEDDING_VECTOR}", - json.dumps(first_embedding.get("embedding", [])), - ) - _set_span_attribute( - span, SpanAttributes.EMBEDDING_EMBEDDINGS, safe_json_dumps(result_data) + f"{SpanAttributes.EMBEDDING_EMBEDDINGS}.{index}.{EmbeddingAttributes.EMBEDDING_VECTOR}", + json.dumps(embedding.get("embedding", [])), ) elif isinstance(result, ImageResponse): _set_span_attribute( @@ -368,14 +405,23 @@ def _request(self, name, args, kwargs): for key, param in signature.parameters.items(): if param.kind is inspect.Parameter.VAR_KEYWORD: request.update(request.pop(key, {})) - # Retry helpers accept only *args/**kwargs; their arguments are completion's. - if "args" in request: - request.pop("args") - request.update( - inspect.signature(self.original_litellm_funcs["completion"]) - .bind_partial(*args) - .arguments - ) + # Async endpoints and retry helpers may expose only *args/**kwargs. + endpoint = { + "acompletion": "completion", + "completion_with_retries": "completion", + "acompletion_with_retries": "completion", + "aembedding": "embedding", + "aimage_generation": "image_generation", + }.get(name) + for key, param in signature.parameters.items(): + if param.kind is inspect.Parameter.VAR_POSITIONAL and key in request: + positional = request.pop(key) + if endpoint: + request.update( + inspect.signature(self.original_litellm_funcs[endpoint]) + .bind_partial(*positional) + .arguments + ) except (TypeError, ValueError): request = dict(kwargs) # Capture generation inputs/settings, not arbitrary SDK control objects, @@ -394,7 +440,7 @@ def _call(self, name, annotate, args, kwargs): try: with _provider_context(span): result = original(*args, **kwargs) - if request.get("stream"): + if request.get("stream") and hasattr(result, "__next__"): return StreamingIteratorWrapper(result, span) _observe(_finalize_span, span, result) span.set_status(trace_api.StatusCode.OK) @@ -417,7 +463,7 @@ async def _acall(self, name, annotate, args, kwargs): try: with _provider_context(span): result = await original(*args, **kwargs) - if request.get("stream"): + if request.get("stream") and hasattr(result, "__anext__"): return AsyncStreamingIteratorWrapper(result, span) _observe(_finalize_span, span, result) span.set_status(trace_api.StatusCode.OK) @@ -525,6 +571,7 @@ def __init__(self, span, span_cm=None): self.span_cm = span_cm self.processed_chunks = [] self.content_string = "" + self.has_tool_context = False self._ended = False self._closed = False @@ -539,6 +586,7 @@ def _process(self, chunk): choices = chunk.get("choices") or [] if choices: delta = choices[0].get("delta") or {} + self.has_tool_context |= _has_tool_context(delta) content = delta.get("content") if isinstance(content, str): self.content_string += content @@ -555,7 +603,11 @@ def _finish(self, error=None): _set_span_attribute( self.span, SpanAttributes.OUTPUT_VALUE, - self.content_string or safe_json_dumps(self.processed_chunks), + ( + self.content_string + if self.content_string and not self.has_tool_context + else safe_json_dumps(self.processed_chunks) + ), ) if error is None: self.span.set_status(trace_api.StatusCode.OK) @@ -572,6 +624,21 @@ def _finish(self, error=None): self.span.end() +def _stream_close_method(stream, *, asynchronous=False): + """Use the SDK close API, or its provider stream on older LiteLLM versions.""" + seen = set() + while stream is not None and id(stream) not in seen: + seen.add(id(stream)) + if asynchronous and callable(close := getattr(stream, "aclose", None)): + return close + if callable(close := getattr(stream, "close", None)): + return close + if not isinstance(stream, litellm.CustomStreamWrapper): + break + stream = stream.completion_stream + return None + + class StreamingIteratorWrapper(_StreamState): def __init__(self, iterator: Iterator[Any], span: trace_api.Span, span_cm=None): super().__init__(span, span_cm) @@ -600,7 +667,7 @@ def close(self): return self._closed = True try: - close = getattr(self.iterator, "close", None) + close = _stream_close_method(self.iterator) if close is not None: with _provider_context(self.span): close() @@ -649,10 +716,12 @@ async def aclose(self): return self._closed = True try: - close = getattr(self.async_iterator, "aclose", None) + close = _stream_close_method(self.async_iterator, asynchronous=True) if close is not None: with _provider_context(self.span): - await close() + result = close() + if inspect.isawaitable(result): + await result except BaseException as error: self._finish(error) raise From f3aee499cf215353ac62e3600b01c788854f9cae Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Mon, 2 Nov 2026 01:44:34 -0800 Subject: [PATCH 04/11] fix(litellm): protect image and embedding inputs and retain all choices Apply image privacy and size limits to every captured representation, retain tool linkage without raw image copies, and mask indexed embedding inputs against released core. Match batch vectors by provider index and preserve every completion choice for sync and async streams and responses. Add real SDK transport regressions covering privacy configuration and environment overrides, supported input shapes, image limits, and multi-choice completions. Co-Authored-By: Claude Code --- .../litellm/tests/test_privacy_and_choices.py | 427 ++++++++++++++++++ .../litellm/traceai_litellm/__init__.py | 201 +++++---- 2 files changed, 548 insertions(+), 80 deletions(-) create mode 100644 python/frameworks/litellm/tests/test_privacy_and_choices.py diff --git a/python/frameworks/litellm/tests/test_privacy_and_choices.py b/python/frameworks/litellm/tests/test_privacy_and_choices.py new file mode 100644 index 000000000..9cb9e9e14 --- /dev/null +++ b/python/frameworks/litellm/tests/test_privacy_and_choices.py @@ -0,0 +1,427 @@ +"""Privacy and multi-result contracts using real SDKs and synthetic transports.""" + +import copy +import json +from contextlib import contextmanager + +import httpx +import litellm +import openai +import pytest +from fi_instrumentation import TraceConfig +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +from opentelemetry.trace import StatusCode +from traceai_litellm import LiteLLMInstrumentor + + +@pytest.fixture(autouse=True) +def offline_tokenizers(monkeypatch): + # LiteLLM's background cost accounting must not download tokenizers. + monkeypatch.setenv("HF_HUB_OFFLINE", "1") + monkeypatch.setenv("TRANSFORMERS_OFFLINE", "1") + + +@contextmanager +def capture(config=None): + provider = TracerProvider() + exporter = InMemorySpanExporter() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + inst = LiteLLMInstrumentor() + inst.instrument(tracer_provider=provider, config=config or TraceConfig()) + try: + yield exporter + finally: + inst.uninstrument() + provider.shutdown() + + +def attrs(exporter): + spans = exporter.get_finished_spans() + assert len(spans) == 1 + assert spans[0].status.status_code is StatusCode.OK + return dict(spans[0].attributes) + + +def image_messages(marker, with_tool): + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "Describe this synthetic image"}, + {"type": "image_url", "image_url": {"url": marker}}, + ], + } + ] + if with_tool: + messages += [ + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "synthetic_call", + "type": "function", + "function": { + "name": "lookup", + "arguments": "{}", + }, + } + ], + }, + { + "role": "tool", + "tool_call_id": "synthetic_call", + "content": "synthetic-result", + }, + ] + return messages + + +@pytest.mark.parametrize("with_tool", [False, True]) +@pytest.mark.parametrize("asynchronous", [False, True]) +@pytest.mark.parametrize("from_environment", [False, True]) +@pytest.mark.parametrize("hide", [False, True]) +@pytest.mark.asyncio +async def test_hide_input_images_with_tool_history( + with_tool, asynchronous, from_environment, hide, monkeypatch +): + marker = "data:image/png;base64,c3ludGhldGljLXByaXZhdGUtaW1hZ2U=" + messages = image_messages(marker, with_tool) + if from_environment: + monkeypatch.setenv("FI_HIDE_INPUT_IMAGES", str(hide).lower()) + config = TraceConfig() + else: + # Explicit configuration must override the environment in both directions. + monkeypatch.setenv("FI_HIDE_INPUT_IMAGES", str(not hide).lower()) + config = TraceConfig(hide_input_images=hide) + original = copy.deepcopy(messages) + with capture(config) as exporter: + kwargs = dict( + model="gpt-4o-mini", + messages=messages, + mock_response="hello", + api_key="synthetic-key", + ) + result = ( + await litellm.acompletion(**kwargs) + if asynchronous + else litellm.completion(**kwargs) + ) + assert result.choices[0].message.content == "hello" + assert messages == original + actual = attrs(exporter) + assert marker not in actual["input.value"] + assert (marker in json.dumps(actual)) is not hide + if with_tool: + conversation = json.loads(actual["input.value"]) + assert conversation[1:] == messages[1:] + assert "{{gen_ai.input.images.0}}" in json.dumps(conversation[0]) + if hide: + assert "gen_ai.input.images" not in actual + else: + assert json.loads(actual["gen_ai.input.images"]) == [marker] + assert ( + actual[ + "gen_ai.input.messages.0.message.contents.1.message_content.image.image.url" + ] + == marker + ) + + +@pytest.mark.parametrize("asynchronous", [False, True]) +@pytest.mark.parametrize("from_environment", [False, True]) +@pytest.mark.parametrize("hide", [False, True]) +@pytest.mark.parametrize( + "inputs,expected_texts", + [ + ("synthetic-private-single", ["synthetic-private-single"]), + ( + ["synthetic-private-first", "synthetic-private-second"], + ["synthetic-private-first", "synthetic-private-second"], + ), + ([101, 102], ["[101, 102]"]), + ([[101, 102], [103, 104]], ["[101, 102]", "[103, 104]"]), + ], +) +@pytest.mark.asyncio +async def test_embedding_inputs_match_vectors_and_respect_privacy( + asynchronous, from_environment, hide, inputs, expected_texts, monkeypatch +): + # Provider indices, not response list positions, associate vectors with inputs. + data = [ + {"object": "embedding", "index": i, "embedding": [i + 0.1, i + 0.2]} + for i in reversed(range(len(expected_texts))) + ] + payload = { + "object": "list", + "model": "text-embedding-ada-002", + "data": data, + "usage": {"prompt_tokens": 2, "total_tokens": 2}, + } + requests = [] + + def respond(request): + requests.append(json.loads(request.content)) + return httpx.Response(200, json=payload) + + transport = httpx.MockTransport(respond) + if from_environment: + monkeypatch.setenv("FI_HIDE_INPUTS", str(hide).lower()) + config = TraceConfig() + else: + monkeypatch.setenv("FI_HIDE_INPUTS", str(not hide).lower()) + config = TraceConfig(hide_inputs=hide) + original = copy.deepcopy(inputs) + with capture(config) as exporter: + if asynchronous: + async with openai.AsyncOpenAI( + api_key="synthetic-key", + http_client=httpx.AsyncClient(transport=transport), + ) as client: + result = await litellm.aembedding( + "text-embedding-ada-002", inputs, client=client + ) + else: + with openai.OpenAI( + api_key="synthetic-key", http_client=httpx.Client(transport=transport) + ) as client: + result = litellm.embedding( + "text-embedding-ada-002", inputs, client=client + ) + assert inputs == original + assert requests[0]["input"] == inputs + assert result.data == data + actual = attrs(exporter) + assert actual["input.value"] == ("__REDACTED__" if hide else str(inputs)) + for index, text in enumerate(expected_texts): + assert actual[f"embedding.embeddings.{index}.embedding.text"] == ( + "__REDACTED__" if hide else text + ) + assert json.loads(actual[f"embedding.embeddings.{index}.embedding.vector"]) == [ + index + 0.1, + index + 0.2, + ] + if hide: + assert text not in json.dumps(actual) + assert f"embedding.embeddings.{len(expected_texts)}.embedding.text" not in actual + assert actual["gen_ai.usage.total_tokens"] == 2 + + +@pytest.mark.parametrize("with_tool", [False, True]) +@pytest.mark.parametrize("limit", [None, 32]) +@pytest.mark.parametrize("image_kind", ["oversized", "at-limit", "remote"]) +def test_base64_image_size_limit(with_tool, limit, image_kind): + max_length = limit if limit is not None else 32000 + prefix = "data:image/png;base64," + marker = prefix + "A" * (max_length - len(prefix) + (image_kind == "oversized")) + if image_kind == "remote": + marker = "https://example.invalid/" + "x" * 100 + ".png" + messages = image_messages(marker, with_tool) + original = copy.deepcopy(messages) + with capture(TraceConfig(base64_image_max_length=limit)) as exporter: + result = litellm.completion( + model="gpt-4o-mini", + messages=messages, + mock_response="hello", + api_key="synthetic-key", + ) + assert result.choices[0].message.content == "hello" + assert messages == original + actual = attrs(exporter) + expected = "__REDACTED__" if image_kind == "oversized" else marker + assert ( + actual[ + "gen_ai.input.messages.0.message.contents.1.message_content.image.image.url" + ] + == expected + ) + assert json.loads(actual["gen_ai.input.images"]) == [expected] + assert marker not in actual["input.value"] + if image_kind == "oversized": + assert marker not in json.dumps(actual) + if with_tool: + assert json.loads(actual["input.value"])[1:] == messages[1:] + + +@pytest.mark.parametrize("streaming", [False, True]) +@pytest.mark.parametrize("asynchronous", [False, True]) +@pytest.mark.parametrize("hide", [False, True]) +@pytest.mark.asyncio +async def test_multiple_completion_choices_are_preserved(streaming, asynchronous, hide): + messages = [{"role": "user", "content": "synthetic-question"}] + texts = ["synthetic-choice-one", "synthetic-choice-two"] + payload = { + "id": "synthetic", + "object": "chat.completion", + "created": 1, + "model": "gpt-4o-mini", + "choices": [ + { + "index": i, + "message": {"role": "assistant", "content": texts[i]}, + "finish_reason": "stop", + } + for i in (0, 1) + ], + "usage": {"prompt_tokens": 2, "completion_tokens": 4, "total_tokens": 6}, + } + if streaming: + chunks = [] + # Split and interleave choices; every chunk need not contain choice zero. + for part in (0, 1): + for index in (1, 0): + delta = { + "content": texts[index][:10] if part == 0 else texts[index][10:] + } + if part == 0: + delta["role"] = "assistant" + chunks.append( + { + "id": "synthetic", + "object": "chat.completion.chunk", + "created": 1, + "model": "gpt-4o-mini", + "choices": [ + { + "index": index, + "delta": delta, + "finish_reason": "stop" if part else None, + } + ], + } + ) + chunks.append( + { + "id": "synthetic", + "object": "chat.completion.chunk", + "created": 1, + "model": "gpt-4o-mini", + "choices": [], + "usage": payload["usage"], + } + ) + raw = ( + "".join("data: " + json.dumps(chunk) + "\n\n" for chunk in chunks) + + "data: [DONE]\n\n" + ).encode() + transport = httpx.MockTransport( + lambda request: httpx.Response( + 200, headers={"content-type": "text/event-stream"}, content=raw + ) + ) + else: + transport = httpx.MockTransport( + lambda request: httpx.Response(200, json=payload) + ) + kwargs = dict(model="gpt-4o-mini", messages=messages, n=2, stream=streaming) + if streaming: + kwargs["stream_options"] = {"include_usage": True} + with capture(TraceConfig(hide_outputs=hide)) as exporter: + if asynchronous: + async with openai.AsyncOpenAI( + api_key="synthetic-key", + http_client=httpx.AsyncClient(transport=transport), + ) as client: + result = await litellm.acompletion(**kwargs, client=client) + if streaming: + received = [chunk async for chunk in result] + else: + with openai.OpenAI( + api_key="synthetic-key", http_client=httpx.Client(transport=transport) + ) as client: + result = litellm.completion(**kwargs, client=client) + if streaming: + received = list(result) + roles = {} + if streaming: + content_by_choice = {} + for chunk in received: + for choice in chunk.choices: + content_by_choice.setdefault(choice.index, "") + content_by_choice[choice.index] += choice.delta.content or "" + if choice.delta.role: + roles[choice.index] = choice.delta.role + assert content_by_choice == dict(enumerate(texts)) + else: + assert { + choice.index: choice.message.content for choice in result.choices + } == dict(enumerate(texts)) + roles = {choice.index: choice.message.role for choice in result.choices} + actual = attrs(exporter) + if hide: + assert actual["output.value"] == "__REDACTED__" + for text in texts: + assert text not in json.dumps(actual) + assert not any(key.startswith("gen_ai.output.messages.") for key in actual) + else: + for index, text in enumerate(texts): + assert actual[f"gen_ai.output.messages.{index}.message.content"] == text + if index in roles: + assert ( + actual[f"gen_ai.output.messages.{index}.message.role"] + == roles[index] + ) + output = json.loads(actual["output.value"]) + choices = ( + [choice for chunk in output for choice in chunk["choices"]] + if streaming + else output["choices"] + ) + assert {choice["index"] for choice in choices} == {0, 1} + assert actual["gen_ai.usage.total_tokens"] == 6 + + +@pytest.mark.parametrize("streaming", [False, True]) +def test_error_privacy_is_preserved(streaming, monkeypatch): + marker = "synthetic-private-error-detail" + transport = httpx.MockTransport( + lambda request: httpx.Response( + 400, json={"error": {"message": marker, "type": "invalid_request_error"}} + ) + ) + monkeypatch.setenv("FI_HIDE_ERROR_DETAILS", "true") + with openai.OpenAI( + api_key="synthetic-key", + max_retries=0, + http_client=httpx.Client(transport=transport), + ) as client: + with capture() as exporter: + with pytest.raises(Exception): + litellm.completion( + model="gpt-4o-mini", + messages=[{"role": "user", "content": "hello"}], + stream=streaming, + client=client, + num_retries=0, + ) + spans = exporter.get_finished_spans() + assert len(spans) == 1 + assert spans[0].status.status_code is StatusCode.ERROR + assert marker not in spans[0].to_json() + + +def test_image_sanitization_preserves_other_tool_conversation_content(): + marker = "data:image/png;base64,c3ludGhldGlj" + messages = image_messages(marker, True) + audio = { + "type": "input_audio", + "input_audio": {"data": "synthetic-audio", "format": "wav"}, + } + # No text in the multimodal message: it must not fall back to its raw images. + messages[0]["content"] = [messages[0]["content"][1], audio] + original = copy.deepcopy(messages) + with capture(TraceConfig(hide_input_images=True)) as exporter: + litellm.completion( + model="gpt-4o-mini", + messages=messages, + mock_response="hello", + api_key="synthetic-key", + ) + actual = attrs(exporter) + conversation = json.loads(actual["input.value"]) + assert audio in conversation[0]["content"] + assert conversation[1:] == original[1:] + assert marker not in json.dumps(actual) + assert messages == original diff --git a/python/frameworks/litellm/traceai_litellm/__init__.py b/python/frameworks/litellm/traceai_litellm/__init__.py index 1d42fdc06..f4313732f 100644 --- a/python/frameworks/litellm/traceai_litellm/__init__.py +++ b/python/frameworks/litellm/traceai_litellm/__init__.py @@ -12,6 +12,7 @@ Iterable, Iterator, Mapping, + Optional, Tuple, TypeVar, ) @@ -161,8 +162,9 @@ def _get_attributes_from_message_content( elif type_ == "image_url": if image := content.pop("image_url"): yield f"{MessageContentAttributes.MESSAGE_CONTENT_TYPE}", "image" - yield f"{MessageContentAttributes.MESSAGE_CONTENT_IMAGE}", image.get( - "url", "" + yield ( + f"{MessageContentAttributes.MESSAGE_CONTENT_IMAGE}.{ImageAttributes.IMAGE_URL}", + image.get("url", ""), ) @@ -175,7 +177,9 @@ def _get_attributes_from_image( def _instrument_func_type_completion( - span: trace_api.Span, kwargs: Dict[str, Any] + span: trace_api.Span, + kwargs: Dict[str, Any], + config: Optional[TraceConfig] = None, ) -> None: """ Currently instruments the functions: @@ -184,7 +188,6 @@ def _instrument_func_type_completion( litellm.completion_with_retries() litellm.acompletion_with_retries() (async version of completion_with_retries) """ - _set_span_attribute(span, SpanAttributes.INPUT_VALUE, safe_json_dumps(kwargs)) _set_span_attribute( span, SpanAttributes.GEN_AI_SPAN_KIND, FiSpanKindValues.LLM.value ) @@ -193,7 +196,7 @@ def _instrument_func_type_completion( ) if messages := kwargs.get("messages"): - process_messages = _process_messages(messages) + process_messages = _process_messages(messages, config) if filtered_messages := process_messages.get("filtered_messages"): _set_span_attribute( span, SpanAttributes.INPUT_VALUE, json.dumps(filtered_messages) @@ -203,9 +206,10 @@ def _instrument_func_type_completion( span, SpanAttributes.INPUT_IMAGES, json.dumps(input_images) ) if any(_has_tool_context(message) for message in messages): - # Keep tool IDs, arguments and results connected in the full conversation. + # Preserve tool linkage without putting image bytes in input.value, + # whose privacy filter cannot inspect nested image content. _set_span_attribute( - span, SpanAttributes.INPUT_VALUE, safe_json_dumps(messages) + span, SpanAttributes.INPUT_VALUE, safe_json_dumps(filtered_messages) ) elif eval_input := process_messages.get("eval_input"): _set_span_attribute(span, SpanAttributes.INPUT_VALUE, eval_input) @@ -227,7 +231,9 @@ def _instrument_func_type_completion( def _instrument_func_type_embedding( - span: trace_api.Span, kwargs: Dict[str, Any] + span: trace_api.Span, + kwargs: Dict[str, Any], + config: Optional[TraceConfig] = None, ) -> None: """ Currently instruments the functions: @@ -243,16 +249,31 @@ def _instrument_func_type_embedding( _set_span_attribute( span, SpanAttributes.EMBEDDING_MODEL_NAME, kwargs.get("model", "unknown_model") ) - _set_span_attribute( - span, - f"{SpanAttributes.EMBEDDING_EMBEDDINGS}.{0}.{EmbeddingAttributes.EMBEDDING_TEXT}", - str(kwargs.get("input")), + inputs = kwargs.get("input") + # A string or a list of token IDs is one input; strings/token lists form a batch. + batch = ( + inputs + if isinstance(inputs, list) and ( + not inputs or not all(isinstance(item, int) for item in inputs) + ) + else [inputs] ) - _set_span_attribute(span, SpanAttributes.INPUT_VALUE, str(kwargs.get("input"))) + config = config or TraceConfig() + for index, text in enumerate(batch): + # Released core 1.2.2 does not yet mask embedding.text. Apply the input + # policy before emitting any indexed copy, including for batch requests. + _set_span_attribute( + span, + f"{SpanAttributes.EMBEDDING_EMBEDDINGS}.{index}.{EmbeddingAttributes.EMBEDDING_TEXT}", + config.mask(SpanAttributes.INPUT_VALUE, str(text)), + ) + _set_span_attribute(span, SpanAttributes.INPUT_VALUE, str(inputs)) def _instrument_func_type_image_generation( - span: trace_api.Span, kwargs: Dict[str, Any] + span: trace_api.Span, + kwargs: Dict[str, Any], + config: Optional[TraceConfig] = None, ) -> None: """ Currently instruments the functions: @@ -275,19 +296,25 @@ def _finalize_span(span: trace_api.Span, result: Any) -> None: if not isinstance(result, EmbeddingResponse): _set_raw_output(span, result) if isinstance(result, ModelResponse): - if (choices := result.choices) and len(choices) > 0: - choice = choices[0] + choices = result.choices or [] + for position, choice in enumerate(choices): if isinstance(choice, Choices): - if choice.message.content and not _has_tool_context(choice.message): + if ( + len(choices) == 1 + and choice.message.content + and not _has_tool_context(choice.message) + ): _set_span_attribute( span, SpanAttributes.OUTPUT_VALUE, choice.message.content ) + index = choice.index if choice.index is not None else position for key, value in _get_attributes_from_message_param(choice.message): _set_span_attribute( - span, f"{SpanAttributes.GEN_AI_OUTPUT_MESSAGES}.0.{key}", value + span, f"{SpanAttributes.GEN_AI_OUTPUT_MESSAGES}.{index}.{key}", value ) elif isinstance(result, EmbeddingResponse): - for index, embedding in enumerate(result.data or []): + for position, embedding in enumerate(result.data or []): + index = embedding.get("index", position) _set_span_attribute( span, f"{SpanAttributes.EMBEDDING_EMBEDDINGS}.{index}.{EmbeddingAttributes.EMBEDDING_VECTOR}", @@ -342,6 +369,7 @@ def _instrument(self, **kwargs: Any) -> None: config = TraceConfig() else: assert isinstance(config, TraceConfig) + self._config = config self._tracer = FITracer( trace_api.get_tracer(__name__, __version__, tracer_provider), config=config, @@ -436,7 +464,7 @@ def _call(self, name, annotate, args, kwargs): span = self._tracer.start_span( name, attributes=dict(get_attributes_from_context()) ) - _observe(annotate, span, request) + _observe(annotate, span, request, self._config) try: with _provider_context(span): result = original(*args, **kwargs) @@ -459,7 +487,7 @@ async def _acall(self, name, annotate, args, kwargs): span = self._tracer.start_span( name, attributes=dict(get_attributes_from_context()) ) - _observe(annotate, span, request) + _observe(annotate, span, request, self._config) try: with _provider_context(span): result = await original(*args, **kwargs) @@ -570,7 +598,7 @@ def __init__(self, span, span_cm=None): self.span = span self.span_cm = span_cm self.processed_chunks = [] - self.content_string = "" + self.content_by_choice = {} self.has_tool_context = False self._ended = False self._closed = False @@ -584,12 +612,19 @@ def _process(self, chunk): return self.processed_chunks.append(chunk) choices = chunk.get("choices") or [] - if choices: - delta = choices[0].get("delta") or {} + for position, choice in enumerate(choices): + index = choice.get("index", position) + delta = choice.get("delta") or {} self.has_tool_context |= _has_tool_context(delta) + self.content_by_choice.setdefault(index, "") content = delta.get("content") if isinstance(content, str): - self.content_string += content + self.content_by_choice[index] += content + _set_span_attribute( + self.span, + f"{SpanAttributes.GEN_AI_OUTPUT_MESSAGES}.{index}.{MessageAttributes.MESSAGE_ROLE}", + delta.get("role"), + ) _set_usage(self.span, chunk.get("usage")) _set_span_attribute( self.span, SpanAttributes.GEN_AI_RESPONSE_MODEL, chunk.get("model") @@ -600,15 +635,24 @@ def _finish(self, error=None): return self._ended = True try: + content = next(iter(self.content_by_choice.values()), "") _set_span_attribute( self.span, SpanAttributes.OUTPUT_VALUE, ( - self.content_string - if self.content_string and not self.has_tool_context + content + if len(self.content_by_choice) == 1 + and content + and not self.has_tool_context else safe_json_dumps(self.processed_chunks) ), ) + for index, text in self.content_by_choice.items(): + _set_span_attribute( + self.span, + f"{SpanAttributes.GEN_AI_OUTPUT_MESSAGES}.{index}.{MessageAttributes.MESSAGE_CONTENT}", + text, + ) if error is None: self.span.set_status(trace_api.StatusCode.OK) else: @@ -756,57 +800,54 @@ def _set_raw_output(span: trace_api.Span, result: Any) -> None: _set_span_attribute(span, SpanAttributes.OUTPUT_VALUE, safe_json_dumps(result)) -def _process_messages(messages): - try: - input_images = [] - filtered_messages = [] - eval_input = [] - - if isinstance(messages, list): - for message in messages: - filtered_content = [] - content = message.get("content", []) - - # Handle string content - if isinstance(content, str): - filtered_messages.append(message) - eval_input.append(content) +def _process_messages(messages, config=None): + """Keep conversation structure, with images referenced rather than copied.""" + if not isinstance(messages, list): + return {"input_images": None, "filtered_messages": messages, "eval_input": None} + + config = config or TraceConfig() + input_images = [] + filtered_messages = [] + eval_input = [] + for message_index, message in enumerate(messages): + if not isinstance(message, Mapping): + continue + filtered_message = dict(message) + content = message.get("content") + if isinstance(content, str): + eval_input.append(content) + elif isinstance(content, list): + filtered_content = [] + for content_index, item in enumerate(content): + if not isinstance(item, Mapping): continue - - # Handle list content - elif isinstance(content, list): - for item in content: - if isinstance(item, dict): - if item.get("type") == "image_url": - image_url = item.get("image_url", {}).get("url") - if image_url: - input_images.append(image_url) - image_index = len(input_images) - 1 - eval_input.append( - "{{" - + f"{SpanAttributes.INPUT_IMAGES}.{image_index}" - + "}}" - ) - elif item.get("type") == "text": - filtered_content.append(item) - if text := item.get("text"): - eval_input.append(str(text)) - - # Create new message with filtered content - if filtered_content: - filtered_message = message.copy() - filtered_message["content"] = filtered_content - filtered_messages.append(filtered_message) - - return { - "input_images": input_images if input_images else None, - "filtered_messages": filtered_messages if filtered_messages else messages, - "eval_input": "\n".join(eval_input), - } - except Exception as e: - print(f"Error in _process_messages: {e}") - return { - "input_images": None, - "filtered_messages": messages, - "eval_input": None, - } + if item.get("type") == "image_url": + image = item.get("image_url") or {} + if image_url := image.get("url"): + # The aggregate must use the same privacy/size policy as + # the canonical image.url attribute, not bypass it via JSON. + key = ( + f"{SpanAttributes.GEN_AI_INPUT_MESSAGES}.{message_index}." + f"{MessageAttributes.MESSAGE_CONTENTS}.{content_index}." + f"{MessageContentAttributes.MESSAGE_CONTENT_IMAGE}.{ImageAttributes.IMAGE_URL}" + ) + image_index = len(input_images) + input_images.append(config.mask(key, image_url)) + reference = ( + "{{" + f"{SpanAttributes.INPUT_IMAGES}.{image_index}" + "}}" + ) + filtered_content.append({"type": "text", "text": reference}) + eval_input.append(reference) + else: + filtered_content.append(dict(item)) + if item.get("type") == "text" and (text := item.get("text")): + eval_input.append(str(text)) + filtered_message["content"] = filtered_content + # Preserve content=None assistant tool calls and tool results as well. + filtered_messages.append(filtered_message) + + return { + "input_images": input_images or None, + "filtered_messages": filtered_messages, + "eval_input": "\n".join(eval_input), + } From 10aa43a5d74a7b9d067a0fa2305ba5b19da063c7 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Mon, 2 Nov 2026 13:02:34 -0800 Subject: [PATCH 05/11] fix(litellm): honor embedding privacy flags and retain audio inputs Co-Authored-By: Claude Code --- .../litellm/tests/test_round3_regressions.py | 110 ++++++++++++++++++ .../litellm/traceai_litellm/__init__.py | 29 ++++- 2 files changed, 133 insertions(+), 6 deletions(-) create mode 100644 python/frameworks/litellm/tests/test_round3_regressions.py diff --git a/python/frameworks/litellm/tests/test_round3_regressions.py b/python/frameworks/litellm/tests/test_round3_regressions.py new file mode 100644 index 000000000..bd51f785d --- /dev/null +++ b/python/frameworks/litellm/tests/test_round3_regressions.py @@ -0,0 +1,110 @@ +"""Independent review probes. Only synthetic SDK transports and local OTel export.""" +import json +from contextlib import contextmanager + +import httpx +import litellm +import openai +import pytest +from fi_instrumentation import TraceConfig +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +from traceai_litellm import LiteLLMInstrumentor + + +@contextmanager +def capture(config=None): + provider = TracerProvider() + exporter = InMemorySpanExporter() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + instrumentor = LiteLLMInstrumentor() + instrumentor.instrument(tracer_provider=provider, config=config or TraceConfig()) + try: + yield exporter + finally: + instrumentor.uninstrument() + provider.shutdown() + + +def attrs(exporter): + spans = exporter.get_finished_spans() + assert len(spans) == 1 + return dict(spans[0].attributes) + + +def response_payload(): + return { + "id": "synthetic", + "object": "chat.completion", + "created": 1, + "model": "gpt-4o-mini", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "synthetic-private-response"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 2, "completion_tokens": 1, "total_tokens": 3}, + } + + +@pytest.mark.parametrize("flag", ["hide_inputs", "hide_input_text", "hide_input_messages"]) +@pytest.mark.parametrize("asynchronous", [False, True]) +@pytest.mark.parametrize("from_environment", [False, True]) +@pytest.mark.asyncio +async def test_embedding_text_hide_flags(flag, asynchronous, from_environment, monkeypatch): + inputs = ["synthetic-private-first", "synthetic-private-second"] + payload = { + "object": "list", "model": "text-embedding-ada-002", + "data": [{"object": "embedding", "index": i, "embedding": [0.1, 0.2]} for i in range(2)], + "usage": {"prompt_tokens": 2, "total_tokens": 2}, + } + transport = httpx.MockTransport(lambda request: httpx.Response(200, json=payload)) + if from_environment: + monkeypatch.setenv("FI_" + flag.upper(), "true") + config = TraceConfig() + else: + config = TraceConfig(**{flag: True}) + with capture(config) as exporter: + if asynchronous: + async with openai.AsyncOpenAI(api_key="synthetic-key", http_client=httpx.AsyncClient(transport=transport)) as client: + result = await litellm.aembedding("text-embedding-ada-002", inputs, client=client) + else: + with openai.OpenAI(api_key="synthetic-key", http_client=httpx.Client(transport=transport)) as client: + result = litellm.embedding("text-embedding-ada-002", inputs, client=client) + assert len(result.data) == 2 + actual = attrs(exporter) + for i, text in enumerate(inputs): + key = f"embedding.embeddings.{i}.embedding.text" + assert actual.get(key) != text, {"flag": flag, "key": key, "actual": actual.get(key)} + + +@pytest.mark.parametrize("with_tool", [False, True]) +@pytest.mark.parametrize("asynchronous", [False, True]) +@pytest.mark.asyncio +async def test_audio_and_text_input_both_preserved(with_tool, asynchronous): + marker = "c3ludGhldGljLWF1ZGlv" + messages = [{"role": "user", "content": [ + {"type": "text", "text": "What was said in this recording?"}, + {"type": "input_audio", "input_audio": {"data": marker, "format": "wav"}}, + ]}] + if with_tool: + messages += [ + {"role": "assistant", "content": None, "tool_calls": [{"id": "call_synthetic", "type": "function", "function": {"name": "lookup", "arguments": "{}"}}]}, + {"role": "tool", "tool_call_id": "call_synthetic", "content": "ready"}, + ] + requests = [] + def respond(request): + requests.append(json.loads(request.content)) + return httpx.Response(200, json=response_payload()) + transport = httpx.MockTransport(respond) + with capture() as exporter: + kwargs = dict(model="openai/gpt-audio", messages=messages) + if asynchronous: + async with openai.AsyncOpenAI(api_key="synthetic-key", http_client=httpx.AsyncClient(transport=transport)) as client: + result = await litellm.acompletion(**kwargs, client=client) + else: + with openai.OpenAI(api_key="synthetic-key", http_client=httpx.Client(transport=transport)) as client: + result = litellm.completion(**kwargs, client=client) + assert result.choices[0].message.content == "synthetic-private-response" + assert marker in json.dumps(requests[0]) + actual = attrs(exporter) + assert marker in json.dumps(actual), {"with_tool": with_tool, "actual": actual} + + diff --git a/python/frameworks/litellm/traceai_litellm/__init__.py b/python/frameworks/litellm/traceai_litellm/__init__.py index f4313732f..c20fa7f2b 100644 --- a/python/frameworks/litellm/traceai_litellm/__init__.py +++ b/python/frameworks/litellm/traceai_litellm/__init__.py @@ -150,6 +150,15 @@ def _has_tool_context(message): ) +def _has_non_text_content(message): + """Keep blocks without dedicated text/image attributes in input.value.""" + content = message.get("content") + return isinstance(content, list) and any( + isinstance(item, Mapping) and item.get("type") not in ("text", "image_url") + for item in content + ) + + def _get_attributes_from_message_content( content: Mapping[str, Any], ) -> Iterator[Tuple[str, AttributeValue]]: @@ -205,9 +214,12 @@ def _instrument_func_type_completion( _set_span_attribute( span, SpanAttributes.INPUT_IMAGES, json.dumps(input_images) ) - if any(_has_tool_context(message) for message in messages): - # Preserve tool linkage without putting image bytes in input.value, - # whose privacy filter cannot inspect nested image content. + if any( + _has_tool_context(message) or _has_non_text_content(message) + for message in messages + ): + # Preserve tool linkage and non-text blocks (such as input_audio) + # without copying image bytes into input.value. _set_span_attribute( span, SpanAttributes.INPUT_VALUE, safe_json_dumps(filtered_messages) ) @@ -260,12 +272,17 @@ def _instrument_func_type_embedding( ) config = config or TraceConfig() for index, text in enumerate(batch): - # Released core 1.2.2 does not yet mask embedding.text. Apply the input - # policy before emitting any indexed copy, including for batch requests. + # Released core 1.2.2 does not yet mask embedding.text. All three + # input-content flags cover this indexed copy in the source core. + safe_text = ( + "__REDACTED__" + if config.hide_inputs or config.hide_input_messages or config.hide_input_text + else str(text) + ) _set_span_attribute( span, f"{SpanAttributes.EMBEDDING_EMBEDDINGS}.{index}.{EmbeddingAttributes.EMBEDDING_TEXT}", - config.mask(SpanAttributes.INPUT_VALUE, str(text)), + safe_text, ) _set_span_attribute(span, SpanAttributes.INPUT_VALUE, str(inputs)) From 6542e8c8c3e7a6a2e9db5f419cad27392f249610 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Mon, 2 Nov 2026 13:04:55 -0800 Subject: [PATCH 06/11] test(litellm): remove trailing test-file whitespace Co-Authored-By: Claude Code --- python/frameworks/litellm/tests/test_round3_regressions.py | 2 -- python/tests/test_framework_litellm.py | 1 - 2 files changed, 3 deletions(-) diff --git a/python/frameworks/litellm/tests/test_round3_regressions.py b/python/frameworks/litellm/tests/test_round3_regressions.py index bd51f785d..bd4d89302 100644 --- a/python/frameworks/litellm/tests/test_round3_regressions.py +++ b/python/frameworks/litellm/tests/test_round3_regressions.py @@ -106,5 +106,3 @@ def respond(request): assert marker in json.dumps(requests[0]) actual = attrs(exporter) assert marker in json.dumps(actual), {"with_tool": with_tool, "actual": actual} - - diff --git a/python/tests/test_framework_litellm.py b/python/tests/test_framework_litellm.py index 6e58816eb..2d68877da 100644 --- a/python/tests/test_framework_litellm.py +++ b/python/tests/test_framework_litellm.py @@ -966,4 +966,3 @@ def test_process_messages_returns_no_query_key(self): if __name__ == "__main__": pytest.main([__file__, "-v"]) - \ No newline at end of file From e6963193f805b3b735778aa8a09e5ab67f99a473 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Mon, 2 Nov 2026 13:30:26 -0800 Subject: [PATCH 07/11] fix(litellm): prevent aggregate message privacy leaks Co-Authored-By: Claude Code --- .../litellm/tests/test_round3_regressions.py | 47 +++++++++++ .../litellm/traceai_litellm/__init__.py | 78 +++++++++++-------- 2 files changed, 94 insertions(+), 31 deletions(-) diff --git a/python/frameworks/litellm/tests/test_round3_regressions.py b/python/frameworks/litellm/tests/test_round3_regressions.py index bd4d89302..1d443371e 100644 --- a/python/frameworks/litellm/tests/test_round3_regressions.py +++ b/python/frameworks/litellm/tests/test_round3_regressions.py @@ -106,3 +106,50 @@ def respond(request): assert marker in json.dumps(requests[0]) actual = attrs(exporter) assert marker in json.dumps(actual), {"with_tool": with_tool, "actual": actual} + +@pytest.mark.parametrize("flag", ["hide_input_text", "hide_input_messages"]) +@pytest.mark.parametrize("kind", ["completion", "audio", "embedding"]) +@pytest.mark.parametrize("asynchronous", [False, True]) +@pytest.mark.asyncio +async def test_message_input_flags_do_not_leak_aggregate(flag, kind, asynchronous): + marker = "synthetic-private-input-aaa" + with capture(TraceConfig(**{flag: True})) as exporter: + if kind == "embedding": + payload = {"object": "list", "model": "text-embedding-ada-002", "data": [{"object": "embedding", "index": 0, "embedding": [0.1]}]} + transport = httpx.MockTransport(lambda request: httpx.Response(200, json=payload)) + if asynchronous: + async with openai.AsyncOpenAI(api_key="synthetic-key", http_client=httpx.AsyncClient(transport=transport)) as client: + await litellm.aembedding("text-embedding-ada-002", marker, client=client) + else: + with openai.OpenAI(api_key="synthetic-key", http_client=httpx.Client(transport=transport)) as client: + litellm.embedding("text-embedding-ada-002", marker, client=client) + else: + content = marker if kind == "completion" else [{"type": "text", "text": marker}, {"type": "input_audio", "input_audio": {"data": "c3ludGhldGljLWF1ZGlv", "format": "wav"}}] + kwargs = dict(model="gpt-4o-mini", messages=[{"role": "user", "content": content}], mock_response="ok", api_key="synthetic-key") + if asynchronous: + await litellm.acompletion(**kwargs) + else: + litellm.completion(**kwargs) + actual = attrs(exporter) + assert marker not in actual.get("input.value", ""), (flag, kind, actual.get("input.value")) + + +@pytest.mark.parametrize("flag", ["hide_output_text", "hide_output_messages"]) +@pytest.mark.parametrize("asynchronous", [False, True]) +@pytest.mark.parametrize("streaming", [False, True]) +@pytest.mark.asyncio +async def test_message_output_flags_do_not_leak_aggregate(flag, asynchronous, streaming): + marker = "synthetic-private-output-bbb" + kwargs = dict(model="gpt-4o-mini", messages=[{"role": "user", "content": "hello"}], mock_response=marker, api_key="synthetic-key", stream=streaming) + with capture(TraceConfig(**{flag: True})) as exporter: + if asynchronous: + result = await litellm.acompletion(**kwargs) + if streaming: + async for _ in result: + pass + else: + result = litellm.completion(**kwargs) + if streaming: + list(result) + actual = attrs(exporter) + assert marker not in actual.get("output.value", ""), (flag, streaming, actual.get("output.value")) diff --git a/python/frameworks/litellm/traceai_litellm/__init__.py b/python/frameworks/litellm/traceai_litellm/__init__.py index c20fa7f2b..3944aeca9 100644 --- a/python/frameworks/litellm/traceai_litellm/__init__.py +++ b/python/frameworks/litellm/traceai_litellm/__init__.py @@ -206,7 +206,10 @@ def _instrument_func_type_completion( if messages := kwargs.get("messages"): process_messages = _process_messages(messages, config) - if filtered_messages := process_messages.get("filtered_messages"): + # Core masks indexed message attributes, but input.value is an independent + # aggregate. Omit it when either message-content privacy flag applies. + hide_message_input = bool(config and (config.hide_input_text or config.hide_input_messages)) + if not hide_message_input and (filtered_messages := process_messages.get("filtered_messages")): _set_span_attribute( span, SpanAttributes.INPUT_VALUE, json.dumps(filtered_messages) ) @@ -214,7 +217,7 @@ def _instrument_func_type_completion( _set_span_attribute( span, SpanAttributes.INPUT_IMAGES, json.dumps(input_images) ) - if any( + if not hide_message_input and any( _has_tool_context(message) or _has_non_text_content(message) for message in messages ): @@ -223,7 +226,7 @@ def _instrument_func_type_completion( _set_span_attribute( span, SpanAttributes.INPUT_VALUE, safe_json_dumps(filtered_messages) ) - elif eval_input := process_messages.get("eval_input"): + elif not hide_message_input and (eval_input := process_messages.get("eval_input")): _set_span_attribute(span, SpanAttributes.INPUT_VALUE, eval_input) for index, input_message in list(enumerate(messages)): @@ -252,7 +255,10 @@ def _instrument_func_type_embedding( litellm.embedding() litellm.aembedding() (async version of embedding) """ - _set_span_attribute(span, SpanAttributes.INPUT_VALUE, safe_json_dumps(kwargs)) + config = config or TraceConfig() + hide_message_input = config.hide_input_text or config.hide_input_messages + if not hide_message_input: + _set_span_attribute(span, SpanAttributes.INPUT_VALUE, safe_json_dumps(kwargs)) _set_span_attribute( span, SpanAttributes.GEN_AI_SPAN_KIND, @@ -284,7 +290,8 @@ def _instrument_func_type_embedding( f"{SpanAttributes.EMBEDDING_EMBEDDINGS}.{index}.{EmbeddingAttributes.EMBEDDING_TEXT}", safe_text, ) - _set_span_attribute(span, SpanAttributes.INPUT_VALUE, str(inputs)) + if not hide_message_input: + _set_span_attribute(span, SpanAttributes.INPUT_VALUE, str(inputs)) def _instrument_func_type_image_generation( @@ -297,20 +304,26 @@ def _instrument_func_type_image_generation( litellm.image_generation() litellm.aimage_generation() (async version of image_generation) """ - _set_span_attribute(span, SpanAttributes.INPUT_VALUE, safe_json_dumps(kwargs)) + if not (config and (config.hide_input_text or config.hide_input_messages)): + _set_span_attribute(span, SpanAttributes.INPUT_VALUE, safe_json_dumps(kwargs)) _set_span_attribute( span, SpanAttributes.GEN_AI_SPAN_KIND, FiSpanKindValues.LLM.value ) if model := kwargs.get("model"): _set_span_attribute(span, SpanAttributes.GEN_AI_REQUEST_MODEL, model) - if prompt := kwargs.get("prompt"): + if (prompt := kwargs.get("prompt")) and not ( + config and (config.hide_input_text or config.hide_input_messages) + ): _set_span_attribute(span, SpanAttributes.INPUT_VALUE, str(prompt)) -def _finalize_span(span: trace_api.Span, result: Any) -> None: - # Embedding vectors must only use the canonical, privacy-filtered attributes. - # Serializing the response elsewhere bypasses FI_HIDE_EMBEDDING_VECTORS. - if not isinstance(result, EmbeddingResponse): +def _finalize_span(span: trace_api.Span, result: Any, config: Optional[TraceConfig] = None) -> None: + # Embedding vectors and private message content must not reappear in the + # independent raw response aggregate after core filters semantic attributes. + hide_message_output = bool(config and (config.hide_output_text or config.hide_output_messages)) + if not isinstance(result, EmbeddingResponse) and not ( + hide_message_output and isinstance(result, ModelResponse) + ): _set_raw_output(span, result) if isinstance(result, ModelResponse): choices = result.choices or [] @@ -319,6 +332,7 @@ def _finalize_span(span: trace_api.Span, result: Any) -> None: if ( len(choices) == 1 and choice.message.content + and not hide_message_output and not _has_tool_context(choice.message) ): _set_span_attribute( @@ -486,8 +500,8 @@ def _call(self, name, annotate, args, kwargs): with _provider_context(span): result = original(*args, **kwargs) if request.get("stream") and hasattr(result, "__next__"): - return StreamingIteratorWrapper(result, span) - _observe(_finalize_span, span, result) + return StreamingIteratorWrapper(result, span, config=self._config) + _observe(_finalize_span, span, result, self._config) span.set_status(trace_api.StatusCode.OK) except BaseException as error: _record_error(span, error) @@ -509,8 +523,8 @@ async def _acall(self, name, annotate, args, kwargs): with _provider_context(span): result = await original(*args, **kwargs) if request.get("stream") and hasattr(result, "__anext__"): - return AsyncStreamingIteratorWrapper(result, span) - _observe(_finalize_span, span, result) + return AsyncStreamingIteratorWrapper(result, span, config=self._config) + _observe(_finalize_span, span, result, self._config) span.set_status(trace_api.StatusCode.OK) except BaseException as error: _record_error(span, error) @@ -611,9 +625,10 @@ def _set_usage(span, usage): class _StreamState: - def __init__(self, span, span_cm=None): + def __init__(self, span, span_cm=None, config=None): self.span = span self.span_cm = span_cm + self.config = config self.processed_chunks = [] self.content_by_choice = {} self.has_tool_context = False @@ -653,17 +668,18 @@ def _finish(self, error=None): self._ended = True try: content = next(iter(self.content_by_choice.values()), "") - _set_span_attribute( - self.span, - SpanAttributes.OUTPUT_VALUE, - ( - content - if len(self.content_by_choice) == 1 - and content - and not self.has_tool_context - else safe_json_dumps(self.processed_chunks) - ), - ) + if not (self.config and (self.config.hide_output_text or self.config.hide_output_messages)): + _set_span_attribute( + self.span, + SpanAttributes.OUTPUT_VALUE, + ( + content + if len(self.content_by_choice) == 1 + and content + and not self.has_tool_context + else safe_json_dumps(self.processed_chunks) + ), + ) for index, text in self.content_by_choice.items(): _set_span_attribute( self.span, @@ -701,8 +717,8 @@ def _stream_close_method(stream, *, asynchronous=False): class StreamingIteratorWrapper(_StreamState): - def __init__(self, iterator: Iterator[Any], span: trace_api.Span, span_cm=None): - super().__init__(span, span_cm) + def __init__(self, iterator: Iterator[Any], span: trace_api.Span, span_cm=None, config=None): + super().__init__(span, span_cm, config) self.iterator = iterator def __iter__(self): @@ -749,9 +765,9 @@ def __exit__(self, exc_type, exc_value, traceback): class AsyncStreamingIteratorWrapper(_StreamState): def __init__( - self, async_iterator: AsyncIterator[Any], span: trace_api.Span, span_cm=None + self, async_iterator: AsyncIterator[Any], span: trace_api.Span, span_cm=None, config=None ): - super().__init__(span, span_cm) + super().__init__(span, span_cm, config) self.async_iterator = async_iterator def __aiter__(self): From 45a0dfecbecf05f133aeb2f4de88e5bf68353ca4 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Mon, 2 Nov 2026 13:56:01 -0800 Subject: [PATCH 08/11] fix(litellm): protect generated image output aggregates Co-Authored-By: Claude Code --- .../tests/test_image_output_privacy.py | 49 +++++++++++++++++++ .../litellm/traceai_litellm/__init__.py | 8 +-- 2 files changed, 54 insertions(+), 3 deletions(-) create mode 100644 python/frameworks/litellm/tests/test_image_output_privacy.py diff --git a/python/frameworks/litellm/tests/test_image_output_privacy.py b/python/frameworks/litellm/tests/test_image_output_privacy.py new file mode 100644 index 000000000..57426ecfb --- /dev/null +++ b/python/frameworks/litellm/tests/test_image_output_privacy.py @@ -0,0 +1,49 @@ +"""Generated image content follows output-message privacy across response formats.""" +import json + +import litellm +import pytest +from fi_instrumentation import TraceConfig +from litellm.types.utils import ImageResponse +from openai.types.image import Image +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +from traceai_litellm import LiteLLMInstrumentor + + +@pytest.mark.parametrize("flag", ["hide_output_text", "hide_output_messages", None]) +@pytest.mark.parametrize("asynchronous", [False, True]) +@pytest.mark.parametrize("format", ["url", "b64_json"]) +@pytest.mark.parametrize("object_response", [False, True]) +@pytest.mark.asyncio +async def test_image_generation_output_privacy(flag, asynchronous, format, object_response, monkeypatch): + marker = "synthetic-private-generated-image-data" + item = {format: marker} + response = ImageResponse(data=[Image(**item) if object_response else item]) + async def async_image_generation(*args, **kwargs): + return response + def image_generation(*args, **kwargs): + return response + monkeypatch.setattr(litellm, "image_generation", image_generation) + monkeypatch.setattr(litellm, "aimage_generation", async_image_generation) + provider, exporter = TracerProvider(), InMemorySpanExporter() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + instrumentor = LiteLLMInstrumentor() + instrumentor.instrument(tracer_provider=provider, config=TraceConfig(**({flag: True} if flag else {}))) + try: + actual = (await litellm.aimage_generation(model="synthetic-model", prompt="public-prompt") + if asynchronous else litellm.image_generation(model="synthetic-model", prompt="public-prompt")) + assert actual is response + finally: + instrumentor.uninstrument() + provider.shutdown() + spans = exporter.get_finished_spans() + assert len(spans) == 1 + attributes = dict(spans[0].attributes) + if flag == "hide_output_messages": + assert marker not in json.dumps(attributes), attributes + if flag is not None: + assert marker not in attributes.get("output.value", ""), attributes + else: + assert marker in attributes.get("output.value", ""), attributes diff --git a/python/frameworks/litellm/traceai_litellm/__init__.py b/python/frameworks/litellm/traceai_litellm/__init__.py index 3944aeca9..cdf325252 100644 --- a/python/frameworks/litellm/traceai_litellm/__init__.py +++ b/python/frameworks/litellm/traceai_litellm/__init__.py @@ -322,7 +322,7 @@ def _finalize_span(span: trace_api.Span, result: Any, config: Optional[TraceConf # independent raw response aggregate after core filters semantic attributes. hide_message_output = bool(config and (config.hide_output_text or config.hide_output_messages)) if not isinstance(result, EmbeddingResponse) and not ( - hide_message_output and isinstance(result, ModelResponse) + hide_message_output and isinstance(result, (ModelResponse, ImageResponse)) ): _set_raw_output(span, result) if isinstance(result, ModelResponse): @@ -372,7 +372,8 @@ def _finalize_span(span: trace_api.Span, result: Any, config: Optional[TraceConf f"{SpanAttributes.GEN_AI_OUTPUT_MESSAGES}.{0}.{MessageAttributes.MESSAGE_CONTENT}.{idx}.{MessageContentAttributes.MESSAGE_CONTENT_IMAGE}", url, ) - _set_span_attribute(span, SpanAttributes.OUTPUT_VALUE, url) + if not hide_message_output: + _set_span_attribute(span, SpanAttributes.OUTPUT_VALUE, url) elif isinstance(img_data, dict) and ( url := (img_data.get("url") or img_data.get("b64_json")) ): @@ -381,7 +382,8 @@ def _finalize_span(span: trace_api.Span, result: Any, config: Optional[TraceConf f"{SpanAttributes.GEN_AI_OUTPUT_MESSAGES}.{0}.{MessageAttributes.MESSAGE_CONTENT}.{idx}.{MessageContentAttributes.MESSAGE_CONTENT_IMAGE}", url, ) - _set_span_attribute(span, SpanAttributes.OUTPUT_VALUE, url) + if not hide_message_output: + _set_span_attribute(span, SpanAttributes.OUTPUT_VALUE, url) _set_usage(span, getattr(result, "usage", None)) From 640b305fb928a8261fe3a2e5625c98f4d8c47665 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Wed, 4 Nov 2026 11:21:40 -0800 Subject: [PATCH 09/11] fix: complete LiteLLM endpoint and stream telemetry Trace optional Responses endpoints, preserve provider/model and detailed usage metadata, accumulate streaming tools, and finalize abandoned streams once. Cover real loopback SDK requests, privacy, suppression, errors and lifecycle behavior on current and floor dependencies. Co-Authored-By: Claude Code --- python/frameworks/litellm/README.md | 4 +- .../litellm/tests/test_http_regressions.py | 689 ++++++++++++++++++ .../litellm/tests/test_stream_extensions.py | 381 ++++++++++ .../litellm/traceai_litellm/__init__.py | 250 ++++++- .../litellm/traceai_litellm/_responses.py | 126 ++++ python/tests/test_framework_litellm.py | 35 +- 6 files changed, 1427 insertions(+), 58 deletions(-) create mode 100644 python/frameworks/litellm/tests/test_http_regressions.py create mode 100644 python/frameworks/litellm/tests/test_stream_extensions.py create mode 100644 python/frameworks/litellm/traceai_litellm/_responses.py diff --git a/python/frameworks/litellm/README.md b/python/frameworks/litellm/README.md index 10ce1dc38..947d48577 100644 --- a/python/frameworks/litellm/README.md +++ b/python/frameworks/litellm/README.md @@ -38,7 +38,9 @@ trace_provider = register( ``` ### Configure Litellm Instrumentation -Instrument the Litellm client to enable telemetry collection. This step ensures that all interactions with the Litellm SDK are tracked and monitored. +Instrument the LiteLLM client to trace `completion` / `acompletion`, `completion_with_retries`, `embedding` / `aembedding`, and `image_generation` / `aimage_generation`. On LiteLLM versions that expose them, `responses` / `aresponses` are also traced. Completion and Responses streams are traced as you consume them; other SDK endpoints are not covered. + +Exhaust streams or explicitly call `close()` / `await aclose()` to finish telemetry and release provider resources promptly. Discarding a stream finalizes its span on garbage collection, but does not guarantee immediate transport cleanup. ```python from traceai_litellm import LiteLLMInstrumentor diff --git a/python/frameworks/litellm/tests/test_http_regressions.py b/python/frameworks/litellm/tests/test_http_regressions.py new file mode 100644 index 000000000..0448e88f1 --- /dev/null +++ b/python/frameworks/litellm/tests/test_http_regressions.py @@ -0,0 +1,689 @@ +"""Independent local-HTTP contract. Run only in E2B, with outbound network disabled.""" + +import gc +import json +import threading +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +import litellm +import pytest +from fi_instrumentation import TraceConfig +from opentelemetry import context, trace +from opentelemetry.context import _SUPPRESS_INSTRUMENTATION_KEY +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +from opentelemetry.trace import StatusCode +from traceai_litellm import LiteLLMInstrumentor + +MODEL = "gpt-4o-mini" +RETURNED_MODEL = "gpt-4o-mini-2024-07-18" +USAGE = { + "prompt_tokens": 9, + "completion_tokens": 4, + "total_tokens": 13, + "prompt_tokens_details": {"cached_tokens": 3, "audio_tokens": 0}, + "completion_tokens_details": {"reasoning_tokens": 2, "audio_tokens": 1}, +} + + +class Handler(BaseHTTPRequestHandler): + def log_message(self, *args): + pass + + def do_POST(self): + request = json.loads(self.rfile.read(int(self.headers["Content-Length"]))) + self.server.requests.append((self.path, request)) + if "error" in self.path: + self.send_response(400) + self.send_header("Content-Type", "application/json") + self.end_headers() + self.wfile.write( + json.dumps( + { + "error": { + "message": "synthetic-private-error", + "type": "invalid_request_error", + } + } + ).encode() + ) + return + if self.path.endswith("/embeddings"): + response = { + "object": "list", + "model": "text-embedding-ada-002", + "data": [{"object": "embedding", "index": 0, "embedding": [0.1]}], + "usage": {"prompt_tokens": 9, "total_tokens": 9}, + } + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.end_headers() + self.wfile.write(json.dumps(response).encode()) + return + if self.path.endswith("/responses"): + response = { + "id": "resp_synthetic", + "object": "response", + "instructions": request.get("instructions"), + "metadata": {"private": "synthetic-echoed-metadata"}, + "created_at": 1, + "status": "completed", + "model": RETURNED_MODEL, + "output": [ + { + "id": "msg_synthetic", + "type": "message", + "role": "assistant", + "status": "completed", + "content": [ + { + "type": "output_text", + "text": "synthetic-output", + "annotations": [], + } + ], + } + ], + "parallel_tool_calls": False, + "tool_choice": "auto", + "tools": [], + "usage": { + "input_tokens": 9, + "output_tokens": 4, + "total_tokens": 13, + "input_tokens_details": {"cached_tokens": 3}, + "output_tokens_details": {"reasoning_tokens": 2}, + }, + } + tool = { + "type": "function_call", + "id": "fc_synthetic", + "call_id": "call_synthetic", + "name": "lookup", + "arguments": '{"x":2}', + "status": "completed", + } + if request.get("tools"): + response["output"].append(tool) + self.send_response(200) + if request.get("stream"): + self.send_header("Content-Type", "text/event-stream") + self.end_headers() + message = response["output"][0] + events = [ + { + "type": "response.created", + "response": { + **response, + "status": "in_progress", + "output": [], + "usage": None, + }, + }, + { + "type": "response.output_item.added", + "output_index": 0, + "item": {**message, "content": []}, + }, + { + "type": "response.content_part.added", + "output_index": 0, + "content_index": 0, + "item_id": message["id"], + "part": {"type": "output_text", "text": "", "annotations": []}, + }, + ] + for delta in ("synthetic-", "output"): + events.append( + { + "type": "response.output_text.delta", + "output_index": 0, + "content_index": 0, + "item_id": message["id"], + "delta": delta, + } + ) + events.extend( + [ + { + "type": "response.output_text.done", + "output_index": 0, + "content_index": 0, + "item_id": message["id"], + "text": "synthetic-output", + }, + { + "type": "response.content_part.done", + "output_index": 0, + "content_index": 0, + "item_id": message["id"], + "part": message["content"][0], + }, + { + "type": "response.output_item.done", + "output_index": 0, + "item": message, + }, + ] + ) + if request.get("tools"): + events.append( + { + "type": "response.output_item.added", + "output_index": 1, + "item": {**tool, "arguments": ""}, + } + ) + for delta in ('{"x":', "2}"): + events.append( + { + "type": "response.function_call_arguments.delta", + "output_index": 1, + "item_id": tool["id"], + "delta": delta, + } + ) + events.extend( + [ + { + "type": "response.function_call_arguments.done", + "output_index": 1, + "item_id": tool["id"], + "arguments": tool["arguments"], + }, + { + "type": "response.output_item.done", + "output_index": 1, + "item": tool, + }, + ] + ) + events.append({"type": "response.completed", "response": response}) + for seq, event in enumerate(events): + event["sequence_number"] = seq + self.wfile.write( + ( + "event: " + + event["type"] + + "\ndata: " + + json.dumps(event) + + "\n\n" + ).encode() + ) + else: + self.send_header("Content-Type", "application/json") + self.end_headers() + self.wfile.write(json.dumps(response).encode()) + return + tools = bool(request.get("tools")) + if request.get("stream"): + self.send_response(200) + self.send_header("Content-Type", "text/event-stream") + self.end_headers() + if tools: + deltas = [ + { + "role": "assistant", + "tool_calls": [ + { + "index": 0, + "id": "call_synthetic", + "type": "function", + "function": {"name": "lookup", "arguments": '{"x":'}, + } + ], + }, + {"tool_calls": [{"index": 0, "function": {"arguments": "2}"}}]}, + ] + else: + deltas = [ + {"role": "assistant", "content": "synthetic-"}, + {"content": "output"}, + ] + chunks = [ + { + "id": "chatcmpl_synthetic", + "object": "chat.completion.chunk", + "created": 1, + "model": RETURNED_MODEL, + "choices": [{"index": 0, "delta": d, "finish_reason": None}], + } + for d in deltas + ] + chunks += [ + { + "id": "chatcmpl_synthetic", + "object": "chat.completion.chunk", + "created": 1, + "model": RETURNED_MODEL, + "choices": [ + { + "index": 0, + "delta": {}, + "finish_reason": "tool_calls" if tools else "stop", + } + ], + "usage": USAGE, + } + ] + for chunk in chunks: + self.wfile.write(("data: " + json.dumps(chunk) + "\n\n").encode()) + self.wfile.write(b"data: [DONE]\n\n") + else: + message = {"role": "assistant", "content": "synthetic-output"} + if tools: + message["tool_calls"] = [ + { + "id": "call_synthetic", + "type": "function", + "function": {"name": "lookup", "arguments": '{"x":2}'}, + } + ] + response = { + "id": "chatcmpl_synthetic", + "object": "chat.completion", + "created": 1, + "model": RETURNED_MODEL, + "choices": [ + { + "index": 0, + "message": message, + "finish_reason": "tool_calls" if tools else "stop", + } + ], + "usage": USAGE, + } + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.end_headers() + self.wfile.write(json.dumps(response).encode()) + + +@pytest.fixture +def server(): + server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + server.requests = [] + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + yield server, "http://127.0.0.1:%d/v1" % server.server_port + server.shutdown() + server.server_close() + thread.join() + + +@pytest.fixture +def capture(): + exporter = InMemorySpanExporter() + provider = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + inst = LiteLLMInstrumentor() + inst.instrument(tracer_provider=provider, config=TraceConfig()) + yield exporter, provider + inst.uninstrument() + provider.shutdown() + + +async def call(base, asynchronous=False, streaming=False, **kwargs): + params = dict( + model=MODEL, + messages=[ + {"role": "system", "content": "synthetic-system"}, + {"role": "user", "content": "synthetic-input"}, + ], + api_base=base, + api_key="synthetic-provider-key", + num_retries=0, + stream=streaming, + **kwargs, + ) + if streaming: + params["stream_options"] = {"include_usage": True} + result = ( + await litellm.acompletion(**params) + if asynchronous + else litellm.completion(**params) + ) + if streaming: + if asynchronous: + return [part async for part in result] + return list(result) + return result + + +@pytest.mark.parametrize("asynchronous", [False, True]) +@pytest.mark.parametrize("streaming", [False, True]) +@pytest.mark.asyncio +async def test_real_http_core_contract(server, capture, asynchronous, streaming): + http, base = server + exporter, provider = capture + with provider.get_tracer("review").start_as_current_span("application") as parent: + result = await call(base, asynchronous, streaming) + assert result + assert trace.get_current_span() is parent + spans = exporter.get_finished_spans() + assert len(spans) == 1 + span = spans[0] + assert span.parent.span_id == parent.get_span_context().span_id + assert span.status.status_code is StatusCode.OK + attrs = dict(span.attributes) + assert attrs["gen_ai.span.kind"] == "LLM" + assert attrs["gen_ai.request.model"] == MODEL + assert "synthetic-input" in attrs["input.value"] + assert "synthetic-system" in attrs["input.value"] + assert attrs["output.value"] == "synthetic-output" + assert attrs["gen_ai.usage.input_tokens"] == 9 + assert attrs["gen_ai.usage.output_tokens"] == 4 + assert attrs["gen_ai.usage.total_tokens"] == 13 + assert "synthetic-provider-key" not in span.to_json() + assert len(http.requests) == 1 + assert len(exporter.get_finished_spans()) == 2 + + +@pytest.mark.parametrize("asynchronous", [False, True]) +@pytest.mark.parametrize("streaming", [False, True]) +@pytest.mark.asyncio +async def test_provider_and_response_model(server, capture, streaming, asynchronous): + _, base = server + exporter, _ = capture + result = await call( + base, + asynchronous=asynchronous, + streaming=streaming, + custom_llm_provider="openai", + ) + response_model = result[-1].model if streaming else result.model + attrs = dict(exporter.get_finished_spans()[0].attributes) + assert attrs.get("gen_ai.provider.name") == "openai" + assert attrs.get("gen_ai.response.model") == response_model + + +@pytest.mark.parametrize("asynchronous", [False, True]) +@pytest.mark.parametrize("streaming", [False, True]) +@pytest.mark.asyncio +async def test_detailed_usage(server, capture, streaming, asynchronous): + _, base = server + exporter, _ = capture + result = await call(base, asynchronous=asynchronous, streaming=streaming) + attrs = dict(exporter.get_finished_spans()[0].attributes) + reported_usage = (result[-1] if streaming else result).usage.model_dump() + if reported_usage.get("cost") is not None: + assert attrs["gen_ai.cost.total"] == reported_usage["cost"] + from importlib.metadata import version + + if streaming and version("litellm") == "1.43.0": + # The floor SDK strips nested usage before yielding its final chunk. + # Do not invent the fields it never exposes to an instrumentor. + assert not reported_usage.get("prompt_tokens_details") + assert not reported_usage.get("completion_tokens_details") + assert "gen_ai.usage.input_tokens.cache_read" not in attrs + assert "gen_ai.usage.output_tokens.reasoning" not in attrs + else: + assert attrs["gen_ai.usage.input_tokens.cache_read"] == 3 + assert attrs["gen_ai.usage.output_tokens.reasoning"] == 2 + assert attrs["gen_ai.usage.input_tokens.audio"] == 0 + assert attrs["gen_ai.usage.output_tokens.audio"] == 1 + + +@pytest.mark.parametrize("asynchronous", [False, True]) +@pytest.mark.parametrize("streaming", [False, True]) +@pytest.mark.asyncio +async def test_real_http_error_privacy( + server, capture, monkeypatch, asynchronous, streaming +): + monkeypatch.setenv("FI_HIDE_ERROR_DETAILS", "true") + _, base = server + exporter, provider = capture + inst = LiteLLMInstrumentor() + inst.uninstrument() + inst.instrument(tracer_provider=provider, config=TraceConfig()) + with pytest.raises(Exception): + await call(base + "/error", asynchronous, streaming) + spans = exporter.get_finished_spans() + assert len(spans) == 1 + assert spans[0].status.status_code is StatusCode.ERROR + assert "synthetic-private-error" not in spans[0].to_json() + + +@pytest.mark.parametrize("asynchronous", [False, True]) +@pytest.mark.asyncio +async def test_stream_tool_semantics(server, capture, asynchronous): + _, base = server + exporter, _ = capture + tool = { + "type": "function", + "function": { + "name": "lookup", + "parameters": {"type": "object", "properties": {"x": {"type": "integer"}}}, + }, + } + await call(base, asynchronous=asynchronous, streaming=True, tools=[tool]) + attrs = dict(exporter.get_finished_spans()[0].attributes) + assert ( + attrs.get( + "gen_ai.output.messages.0.message.tool_calls.0.tool_call.function.name" + ) + == "lookup" + ) + assert ( + attrs.get( + "gen_ai.output.messages.0.message.tool_calls.0.tool_call.function.arguments" + ) + == '{"x":2}' + ) + + +@pytest.mark.skipif( + not hasattr(litellm, "responses"), + reason="Responses API did not exist at declared floor", +) +def test_responses_endpoint_is_traced(server, capture): + http, base = server + exporter, _ = capture + result = litellm.responses( + model="openai/" + MODEL, + input="synthetic-input", + api_base=base, + api_key="synthetic-provider-key", + num_retries=0, + ) + assert result.output[0].content[0].text == "synthetic-output" + assert http.requests[-1][0].endswith("/responses") + spans = exporter.get_finished_spans() + assert len(spans) == 1, "Successful real Responses request exported no span" + + +def test_unread_stream_is_eventually_finalized(server, capture): + _, base = server + exporter, _ = capture + stream = litellm.completion( + model=MODEL, + messages=[{"role": "user", "content": "synthetic-input"}], + api_base=base, + api_key="synthetic-provider-key", + num_retries=0, + stream=True, + ) + span = stream.span + del stream + gc.collect() + assert len(exporter.get_finished_spans()) == 1, ( + "Discarded unread stream left a recording span open" + ) + assert not span.is_recording() + + +@pytest.mark.parametrize("asynchronous", [False, True]) +@pytest.mark.asyncio +async def test_embedding_provider_and_model(server, capture, asynchronous): + _, base = server + exporter, _ = capture + params = dict( + model="openai/text-embedding-ada-002", + input=["hello"], + api_base=base, + api_key="synthetic-provider-key", + custom_llm_provider="openai", + ) + result = ( + await litellm.aembedding(**params) + if asynchronous + else litellm.embedding(**params) + ) + attrs = exporter.get_finished_spans()[0].attributes + assert attrs["gen_ai.provider.name"] == "openai" + assert attrs["gen_ai.response.model"] == result.model + assert "synthetic-provider-key" not in json.dumps(dict(attrs)) + + +@pytest.mark.skipif( + not hasattr(litellm, "responses"), reason="Responses API absent at floor" +) +@pytest.mark.parametrize("asynchronous", [False, True]) +@pytest.mark.parametrize("streaming", [False, True]) +@pytest.mark.parametrize( + "flag", + [ + None, + "hide_inputs", + "hide_input_text", + "hide_input_messages", + "hide_outputs", + "hide_output_text", + "hide_output_messages", + "hide_input_images", + ], +) +@pytest.mark.asyncio +async def test_responses_real_sdk(server, capture, asynchronous, streaming, flag): + http, base = server + exporter, provider = capture + inst = LiteLLMInstrumentor() + inst.uninstrument() + inst.instrument( + tracer_provider=provider, config=TraceConfig(**({flag: True} if flag else {})) + ) + image = "data:image/png;base64,c3ludGhldGlj" + params = dict( + model="openai/" + MODEL, + instructions="synthetic-instructions", + input=[ + { + "role": "user", + "content": [ + {"type": "input_text", "text": "synthetic-input"}, + {"type": "input_image", "image_url": image}, + ], + } + ], + tools=[ + {"type": "function", "name": "lookup", "parameters": {"type": "object"}} + ], + api_base=base, + api_key="synthetic-provider-key", + stream=streaming, + num_retries=0, + ) + with provider.get_tracer("test").start_as_current_span("parent") as parent: + result = ( + await litellm.aresponses(**params) + if asynchronous + else litellm.responses(**params) + ) + if streaming: + assert not exporter.get_finished_spans() + if asynchronous: + chunks = [part async for part in result] + await result.aclose() + else: + chunks = list(result) + result.close() + assert chunks[-1].type == "response.completed" + else: + assert result.output[0].content[0].text == "synthetic-output" + assert trace.get_current_span() is parent + spans = exporter.get_finished_spans() + assert len(spans) == 1 + span = spans[0] + assert span.parent.span_id == parent.get_span_context().span_id + assert span.status.status_code is StatusCode.OK + attrs = dict(span.attributes) + assert attrs["gen_ai.provider.name"] == "openai" + assert attrs["gen_ai.response.model"] == RETURNED_MODEL + assert attrs["gen_ai.usage.input_tokens"] == 9 + assert attrs["gen_ai.usage.output_tokens"] == 4 + assert attrs["gen_ai.usage.input_tokens.cache_read"] == 3 + assert attrs["gen_ai.usage.output_tokens.reasoning"] == 2 + assert "synthetic-provider-key" not in span.to_json() + assert "synthetic-echoed-metadata" not in span.to_json() + if flag in ("hide_inputs", "hide_input_text", "hide_input_messages"): + assert "synthetic-input" not in span.to_json() + assert "synthetic-instructions" not in span.to_json() + else: + assert "synthetic-input" in attrs["input.value"] + assert "synthetic-instructions" in attrs["input.value"] + if flag in ("hide_outputs", "hide_output_text", "hide_output_messages"): + assert "synthetic-output" not in span.to_json() + assert '{"x":2}' not in json.dumps(attrs).replace('\\"', '"') + else: + prefix = "gen_ai.output.messages.1.message.tool_calls.0.tool_call." + assert attrs[prefix + "id"] == "call_synthetic" + assert attrs[prefix + "function.name"] == "lookup" + assert attrs[prefix + "function.arguments"] == '{"x":2}' + assert ( + attrs[ + "gen_ai.output.messages.0.message.contents.0.message_content.text" + ] + == "synthetic-output" + ) + if flag == "hide_input_images": + assert image not in span.to_json() + assert len(http.requests) == 1 + assert http.requests[0][1]["input"][0]["content"][1]["image_url"] == image + + +@pytest.mark.skipif( + not hasattr(litellm, "responses"), reason="Responses API absent at floor" +) +@pytest.mark.parametrize("asynchronous", [False, True]) +@pytest.mark.asyncio +async def test_responses_suppression_restoration_and_errors( + server, capture, asynchronous, monkeypatch +): + _, base = server + exporter, provider = capture + inst = LiteLLMInstrumentor() + name = "aresponses" if asynchronous else "responses" + original = inst.original_litellm_funcs[name] + params = dict( + model="openai/" + MODEL, + input="synthetic-input", + api_base=base, + api_key="synthetic-provider-key", + num_retries=0, + ) + token = context.attach(context.set_value(_SUPPRESS_INSTRUMENTATION_KEY, True)) + try: + result = ( + await litellm.aresponses(**params) + if asynchronous + else litellm.responses(**params) + ) + assert result.output[0].content[0].text == "synthetic-output" + finally: + context.detach(token) + assert not exporter.get_finished_spans() + inst.uninstrument() + assert getattr(litellm, name) is original + monkeypatch.setenv("FI_HIDE_ERROR_DETAILS", "true") + inst.instrument(tracer_provider=provider, config=TraceConfig()) + params["api_base"] = base + "/error" + with pytest.raises(Exception): + if asynchronous: + await litellm.aresponses(**params) + else: + litellm.responses(**params) + spans = exporter.get_finished_spans() + assert len(spans) == 1 + assert spans[0].status.status_code is StatusCode.ERROR + assert "synthetic-private-error" not in spans[0].to_json() diff --git a/python/frameworks/litellm/tests/test_stream_extensions.py b/python/frameworks/litellm/tests/test_stream_extensions.py new file mode 100644 index 000000000..46a7f7490 --- /dev/null +++ b/python/frameworks/litellm/tests/test_stream_extensions.py @@ -0,0 +1,381 @@ +"""Deterministic interleaving, usage, routing and stream-abandonment contracts.""" + +import asyncio +import gc +import json +import weakref +from contextlib import contextmanager +from unittest.mock import Mock + +import litellm +import pytest +from fi_instrumentation import TraceConfig +from litellm.types.utils import ModelResponse +from opentelemetry import trace +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +from opentelemetry.trace import StatusCode + +from traceai_litellm import LiteLLMInstrumentor + + +@pytest.fixture +def capture(monkeypatch): + @contextmanager + def install(chunks=None, result=None, config=None, endpoint="completion"): + async def acall(model=None, **kwargs): + async def stream(): + for chunk in chunks: + if isinstance(chunk, BaseException): + raise chunk + yield chunk + + return stream() if chunks is not None else result + + def sync_call(model=None, **kwargs): + if chunks is None: + return result + + def stream(): + for chunk in chunks: + if isinstance(chunk, BaseException): + raise chunk + yield chunk + + return stream() + + monkeypatch.setattr(litellm, endpoint, sync_call, raising=False) + monkeypatch.setattr(litellm, "a" + endpoint, acall, raising=False) + provider = TracerProvider() + exporter = InMemorySpanExporter() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + inst = LiteLLMInstrumentor() + inst.instrument(tracer_provider=provider, config=config or TraceConfig()) + try: + yield exporter, provider + finally: + inst.uninstrument() + provider.shutdown() + + return install + + +async def invoke(asynchronous, *, endpoint="completion", **kwargs): + params = dict(model="openai/gpt-4o-mini", **kwargs) + if asynchronous: + return await getattr(litellm, "a" + endpoint)(**params) + return getattr(litellm, endpoint)(**params) + + +async def consume(stream, asynchronous, partial=False): + if asynchronous: + if partial: + await stream.__anext__() + else: + [chunk async for chunk in stream] + await stream.aclose() + else: + if partial: + next(stream) + else: + list(stream) + stream.close() + + +def attrs(exporter): + spans = exporter.get_finished_spans() + assert len(spans) == 1 + return dict(spans[0].attributes) + + +def tool_chunks(): + chunks = [] + for part in (0, 1): + choices = [] + for choice in (1, 0): + tools = [] + for tool in (1, 0): + function = { + "arguments": '{"value":' if part == 0 else f"{choice * 10 + tool}}}" + } + data = {"index": tool, "function": function} + if part == 0: + data.update(id=f"call-{choice}-{tool}", type="function") + function["name"] = f"tool-{choice}-{tool}" + tools.append(data) + choices.append( + {"index": choice, "delta": {"role": "assistant", "tool_calls": tools}} + ) + chunks.append({"model": "sdk-stream-model", "choices": choices}) + return chunks + + +@pytest.mark.parametrize("asynchronous", [False, True]) +@pytest.mark.parametrize("partial", [False, True]) +@pytest.mark.parametrize( + "flag", [None, "hide_outputs", "hide_output_messages", "hide_output_text"] +) +@pytest.mark.asyncio +async def test_interleaved_tools(capture, asynchronous, partial, flag): + config = TraceConfig(**({flag: True} if flag else {})) + with capture(chunks=tool_chunks(), config=config) as (exporter, _): + stream = await invoke(asynchronous, stream=True) + await consume(stream, asynchronous, partial) + actual = attrs(exporter) + assert actual["gen_ai.response.model"] == "sdk-stream-model" + for choice in (0, 1): + for tool in (0, 1): + prefix = ( + f"gen_ai.output.messages.{choice}.message.tool_calls.{tool}.tool_call." + ) + arguments = '{"value":' + ("" if partial else f"{choice * 10 + tool}}}") + if flag: + assert "value" not in json.dumps(list(actual.values())) + else: + assert actual[prefix + "id"] == f"call-{choice}-{tool}" + assert actual[prefix + "function.name"] == f"tool-{choice}-{tool}" + assert actual[prefix + "function.arguments"] == arguments + + +@pytest.mark.parametrize("asynchronous", [False, True]) +@pytest.mark.parametrize("partial", [False, True]) +@pytest.mark.parametrize("finish", ["discard", "close", "exhaust"]) +@pytest.mark.asyncio +async def test_abandoned_stream_ends_once(capture, asynchronous, partial, finish): + with capture(chunks=tool_chunks()) as (exporter, provider): + with provider.get_tracer("test").start_as_current_span("parent") as parent: + stream = await invoke(asynchronous, stream=True) + span = stream.span + end = Mock(wraps=span.end) + span.end = end + if partial: + if asynchronous: + await stream.__anext__() + else: + next(stream) + if finish == "close": + if asynchronous: + await stream.aclose() + await stream.aclose() + else: + stream.close() + stream.close() + elif finish == "exhaust": + await consume(stream, asynchronous) + ref = weakref.ref(stream) + del stream + gc.collect() + assert ref() is None + assert end.call_count == 1 + actual = attrs(exporter) + assert not span.is_recording() + assert trace.get_current_span() is parent + if partial or finish == "exhaust": + assert ( + actual[ + "gen_ai.output.messages.0.message.tool_calls.0.tool_call.function.name" + ] + == "tool-0-0" + ) + if finish == "discard": + assert ( + exporter.get_finished_spans()[0].status.status_code + is StatusCode.UNSET + ) + + +@pytest.mark.parametrize("asynchronous", [False, True]) +@pytest.mark.parametrize("streaming", [False, True]) +@pytest.mark.parametrize("value", [None, 0, 7]) +@pytest.mark.asyncio +async def test_detailed_usage_missing_zero_and_reported_cost( + capture, asynchronous, streaming, value +): + usage = { + "prompt_tokens": value, + "completion_tokens": value, + "total_tokens": value, + "prompt_tokens_details": {"cached_tokens": value, "audio_tokens": value}, + "completion_tokens_details": {"reasoning_tokens": value, "audio_tokens": value}, + "cache_creation_input_tokens": value, + "cost": value, + } + result = {"model": "actual-model", "choices": [], "usage": usage} + with capture(chunks=[result] if streaming else None, result=result) as ( + exporter, + _, + ): + response = await invoke(asynchronous, stream=streaming) + if streaming: + await consume(response, asynchronous) + actual = attrs(exporter) + for key in ( + "gen_ai.usage.input_tokens", + "gen_ai.usage.output_tokens", + "gen_ai.usage.total_tokens", + "gen_ai.usage.input_tokens.cache_read", + "gen_ai.usage.input_tokens.cache_write", + "gen_ai.usage.input_tokens.audio", + "gen_ai.usage.output_tokens.audio", + "gen_ai.usage.output_tokens.reasoning", + "gen_ai.cost.total", + ): + if value is None: + assert key not in actual + else: + assert actual[key] == value + + +@pytest.mark.parametrize("asynchronous", [False, True]) +@pytest.mark.parametrize("metadata_provider", [None, "azure"]) +@pytest.mark.asyncio +async def test_explicit_routing_and_private_metadata( + capture, asynchronous, metadata_provider +): + result = ModelResponse(model="actual-deployment") + result._hidden_params = { + "api_key": "synthetic-hidden-secret", + "response_cost": 0.003, + } + if metadata_provider: + result._hidden_params["custom_llm_provider"] = metadata_provider + with capture(result=result) as (exporter, _): + await invoke(asynchronous, custom_llm_provider="vertex_ai") + actual = attrs(exporter) + assert actual["gen_ai.provider.name"] == (metadata_provider or "vertex_ai") + assert actual["gen_ai.response.model"] == "actual-deployment" + assert actual["gen_ai.cost.total"] == 0.003 + assert "synthetic-hidden-secret" not in json.dumps(actual) + + +@pytest.mark.parametrize("asynchronous", [False, True]) +@pytest.mark.parametrize("ending", ["close", "error", "cancel", "failed", "discard"]) +@pytest.mark.asyncio +async def test_responses_partial_stream_lifecycle(capture, asynchronous, ending): + events = [ + { + "type": "response.output_item.added", + "output_index": 1, + "item": { + "type": "function_call", + "call_id": "call-synthetic", + "name": "lookup", + "arguments": "", + }, + }, + { + "type": "response.function_call_arguments.delta", + "output_index": 1, + "delta": '{"value":', + }, + ] + error = ( + asyncio.CancelledError() + if ending == "cancel" + else ValueError("synthetic-stream-error") + ) + if ending in ("error", "cancel"): + events.append(error) + elif ending == "failed": + events.append({"type": "response.failed", "response": {"status": "failed"}}) + with capture(chunks=events, endpoint="responses") as (exporter, provider): + with provider.get_tracer("test").start_as_current_span("parent") as parent: + stream = await invoke( + asynchronous, endpoint="responses", input="hello", stream=True + ) + if ending in ("error", "cancel"): + with pytest.raises(type(error)) as caught: + await consume(stream, asynchronous) + assert caught.value is error + elif ending == "failed": + await consume(stream, asynchronous) + else: + for _ in range(2): + if asynchronous: + await stream.__anext__() + else: + next(stream) + if ending == "close": + if asynchronous: + await stream.aclose() + else: + stream.close() + del stream + gc.collect() + assert trace.get_current_span() is parent + actual = attrs(exporter) + assert ( + actual[ + "gen_ai.output.messages.0.message.tool_calls.0.tool_call.function.arguments" + ] + == '{"value":' + ) + status = exporter.get_finished_spans()[0].status.status_code + assert status is ( + StatusCode.ERROR + if ending in ("error", "cancel", "failed") + else StatusCode.UNSET + if ending == "discard" + else StatusCode.OK + ) + + +@pytest.mark.parametrize("asynchronous", [False, True]) +@pytest.mark.asyncio +async def test_bad_telemetry_does_not_change_success(capture, asynchronous): + class UnserializableResponse: + def model_dump(self): + raise ValueError("synthetic-extraction-error") + + result = UnserializableResponse() + with capture(result=result) as (exporter, _): + assert await invoke(asynchronous) is result + assert exporter.get_finished_spans()[0].status.status_code is StatusCode.OK + + +def test_instrumentation_does_not_create_absent_responses_methods(monkeypatch): + for name in ("responses", "aresponses"): + monkeypatch.delattr(litellm, name, raising=False) + provider = TracerProvider() + inst = LiteLLMInstrumentor() + try: + inst.instrument(tracer_provider=provider) + assert not hasattr(litellm, "responses") + assert not hasattr(litellm, "aresponses") + inst.uninstrument() + assert not hasattr(litellm, "responses") + assert not hasattr(litellm, "aresponses") + finally: + if inst.is_instrumented_by_opentelemetry: + inst.uninstrument() + provider.shutdown() + + +@pytest.mark.parametrize("asynchronous", [False, True]) +@pytest.mark.parametrize("streaming", [False, True]) +@pytest.mark.parametrize("private", [False, True]) +@pytest.mark.asyncio +async def test_responses_failed_payload_privacy( + capture, asynchronous, streaming, private, monkeypatch +): + monkeypatch.setenv("FI_HIDE_ERROR_DETAILS", str(private).lower()) + result = { + "object": "response", + "status": "failed", + "output": [], + "error": {"message": "synthetic-private-error", "code": "server_error"}, + } + chunks = [{"type": "response.failed", "response": result}] + with capture( + chunks=chunks if streaming else None, result=result, endpoint="responses" + ) as (exporter, _): + response = await invoke( + asynchronous, endpoint="responses", input="hello", stream=streaming + ) + if streaming: + await consume(response, asynchronous) + actual = attrs(exporter) + assert ("synthetic-private-error" in json.dumps(actual)) is not private + assert result["error"]["message"] == "synthetic-private-error" + assert exporter.get_finished_spans()[0].status.status_code is StatusCode.ERROR diff --git a/python/frameworks/litellm/traceai_litellm/__init__.py b/python/frameworks/litellm/traceai_litellm/__init__.py index cdf325252..30d037cca 100644 --- a/python/frameworks/litellm/traceai_litellm/__init__.py +++ b/python/frameworks/litellm/traceai_litellm/__init__.py @@ -1,5 +1,6 @@ import json import inspect +import weakref from contextlib import contextmanager from enum import Enum from functools import wraps @@ -50,6 +51,7 @@ from opentelemetry.context import _SUPPRESS_INSTRUMENTATION_KEY from opentelemetry.instrumentation.instrumentor import BaseInstrumentor # type: ignore from opentelemetry.util.types import AttributeValue +from traceai_litellm._responses import ResponseEvents, as_dict, response_messages from traceai_litellm.package import _instruments from traceai_litellm.version import __version__ from wrapt import wrap_function_wrapper @@ -57,6 +59,11 @@ _REQUEST_FIELDS = frozenset( { "model", + "custom_llm_provider", + "instructions", + "max_output_tokens", + "reasoning", + "text", "messages", "input", "prompt", @@ -230,7 +237,7 @@ def _instrument_func_type_completion( _set_span_attribute(span, SpanAttributes.INPUT_VALUE, eval_input) for index, input_message in list(enumerate(messages)): - for key, value in _get_attributes_from_message_param(input_message): + for key, value in _message_attributes(input_message, config, output=False): _set_span_attribute( span, f"{SpanAttributes.GEN_AI_INPUT_MESSAGES}.{index}.{key}", value ) @@ -245,6 +252,82 @@ def _instrument_func_type_completion( ) +def _message_attributes(message, config=None, *, output=True): + hide_text = config and (config.hide_output_text if output else config.hide_input_text) + for key, value in _get_attributes_from_message_param(message): + # Released core masks message text, but not tool/function argument strings. + if hide_text and key.endswith(( + ToolCallAttributes.TOOL_CALL_FUNCTION_ARGUMENTS_JSON, + MessageAttributes.MESSAGE_FUNCTION_CALL_ARGUMENTS_JSON, + )): + continue + yield key, value + + +def _instrument_func_type_responses(span, kwargs, config=None): + messages = response_messages(kwargs.get("input")) + if kwargs.get("instructions"): + messages.insert(0, {"role": "system", "content": kwargs["instructions"]}) + request = {k: v for k, v in kwargs.items() if k not in ("input", "instructions")} + request["messages"] = messages + _instrument_func_type_completion(span, request, config) + + +def _set_provider(span, request=None, result=None): + # Read only normalized routing fields. Never serialize hidden metadata, which + # can contain API keys/headers. Do not pass credentials to the resolver. + hidden = as_dict(getattr(result, "_hidden_params", None)) + provider = hidden.get("custom_llm_provider") or getattr(result, "custom_llm_provider", None) + if not provider and request: + provider = request.get("custom_llm_provider") + if not provider and request.get("model"): + _, provider, _, _ = litellm.get_llm_provider(model=request["model"]) + if isinstance(provider, str): + _set_span_attribute(span, SpanAttributes.GEN_AI_PROVIDER_NAME, provider) + + +def _set_result_metadata(span, result): + data = as_dict(result) + _set_span_attribute(span, SpanAttributes.GEN_AI_RESPONSE_MODEL, data.get("model")) + _observe(_set_provider, span, None, result) + _set_usage(span, data.get("usage")) + # LiteLLM may supply a cost computed by its own accounting. Record what it + # reports, but never invoke pricing/cost estimation from instrumentation. + usage = as_dict(data.get("usage")) + if usage.get("cost") is None: + hidden = as_dict(getattr(result, "_hidden_params", None)) + _set_span_attribute(span, SpanAttributes.GEN_AI_COST_TOTAL, hidden.get("response_cost")) + + +def _private_response(data, config): + """Do not leak echoed Responses inputs/metadata through raw output.""" + hide_error = config and getattr(config, "hide_error_details", False) + kind = data.get("type", "") + if kind == "error" and hide_error: + return {"type": "error"} + if data.get("object") == "response": + # Responses snapshots echo instructions, tools and arbitrary metadata. + # Inputs already have their own privacy-filtered attributes. + fields = {"id", "object", "model", "status", "output", "usage", "error", "created_at"} + data = {key: value for key, value in data.items() if key in fields} + if hide_error: + data.pop("error", None) + elif kind.startswith("response."): + data = dict(data) + if hide_error: + data.pop("error", None) + if isinstance(data.get("response"), Mapping): + response = {"object": "response", **data["response"]} + data["response"] = _private_response(response, config) + return data + + +def _set_response_messages(span, messages, config=None): + for index, message in enumerate(messages): + for key, value in _message_attributes(message, config): + _set_span_attribute(span, f"{SpanAttributes.GEN_AI_OUTPUT_MESSAGES}.{index}.{key}", value) + + def _instrument_func_type_embedding( span: trace_api.Span, kwargs: Dict[str, Any], @@ -321,10 +404,8 @@ def _finalize_span(span: trace_api.Span, result: Any, config: Optional[TraceConf # Embedding vectors and private message content must not reappear in the # independent raw response aggregate after core filters semantic attributes. hide_message_output = bool(config and (config.hide_output_text or config.hide_output_messages)) - if not isinstance(result, EmbeddingResponse) and not ( - hide_message_output and isinstance(result, (ModelResponse, ImageResponse)) - ): - _set_raw_output(span, result) + if not isinstance(result, EmbeddingResponse) and not hide_message_output: + _set_raw_output(span, _private_response(as_dict(result), config) or result) if isinstance(result, ModelResponse): choices = result.choices or [] for position, choice in enumerate(choices): @@ -339,7 +420,7 @@ def _finalize_span(span: trace_api.Span, result: Any, config: Optional[TraceConf span, SpanAttributes.OUTPUT_VALUE, choice.message.content ) index = choice.index if choice.index is not None else position - for key, value in _get_attributes_from_message_param(choice.message): + for key, value in _message_attributes(choice.message, config): _set_span_attribute( span, f"{SpanAttributes.GEN_AI_OUTPUT_MESSAGES}.{index}.{key}", value ) @@ -384,7 +465,16 @@ def _finalize_span(span: trace_api.Span, result: Any, config: Optional[TraceConf ) if not hide_message_output: _set_span_attribute(span, SpanAttributes.OUTPUT_VALUE, url) - _set_usage(span, getattr(result, "usage", None)) + elif as_dict(result).get("object") == "response": + _set_response_messages(span, response_messages(as_dict(result).get("output")), config) + _set_result_metadata(span, result) + + +def _response_status(result): + return ( + trace_api.StatusCode.ERROR if as_dict(result).get("status") == "failed" + else trace_api.StatusCode.OK + ) class LiteLLMInstrumentor(BaseInstrumentor): # type: ignore @@ -418,6 +508,9 @@ def _instrument(self, **kwargs: Any) -> None: "aembedding": self._aembedding_wrapper, "image_generation": self._image_generation_wrapper, "aimage_generation": self._aimage_generation_wrapper, + # Guarded below: Responses APIs do not exist at the 1.43.0 floor. + "responses": self._responses_wrapper, + "aresponses": self._aresponses_wrapper, } if Protect is not None: self._original_protect = inspect.getattr_static(Protect, "protect") @@ -473,6 +566,7 @@ def _request(self, name, args, kwargs): "acompletion_with_retries": "completion", "aembedding": "embedding", "aimage_generation": "image_generation", + "aresponses": "responses", }.get(name) for key, param in signature.parameters.items(): if param.kind is inspect.Parameter.VAR_POSITIONAL and key in request: @@ -498,13 +592,15 @@ def _call(self, name, annotate, args, kwargs): name, attributes=dict(get_attributes_from_context()) ) _observe(annotate, span, request, self._config) + _observe(_set_provider, span, request) try: with _provider_context(span): result = original(*args, **kwargs) + _observe(_set_provider, span, None, result) if request.get("stream") and hasattr(result, "__next__"): return StreamingIteratorWrapper(result, span, config=self._config) + span.set_status(_observe(_response_status, result) or trace_api.StatusCode.OK) _observe(_finalize_span, span, result, self._config) - span.set_status(trace_api.StatusCode.OK) except BaseException as error: _record_error(span, error) span.end() @@ -521,13 +617,15 @@ async def _acall(self, name, annotate, args, kwargs): name, attributes=dict(get_attributes_from_context()) ) _observe(annotate, span, request, self._config) + _observe(_set_provider, span, request) try: with _provider_context(span): result = await original(*args, **kwargs) + _observe(_set_provider, span, None, result) if request.get("stream") and hasattr(result, "__anext__"): return AsyncStreamingIteratorWrapper(result, span, config=self._config) + span.set_status(_observe(_response_status, result) or trace_api.StatusCode.OK) _observe(_finalize_span, span, result, self._config) - span.set_status(trace_api.StatusCode.OK) except BaseException as error: _record_error(span, error) span.end() @@ -579,6 +677,12 @@ async def _aimage_generation_wrapper(self, *args: Any, **kwargs: Any) -> Any: "aimage_generation", _instrument_func_type_image_generation, args, kwargs ) + def _responses_wrapper(self, *args: Any, **kwargs: Any) -> Any: + return self._call("responses", _instrument_func_type_responses, args, kwargs) + + async def _aresponses_wrapper(self, *args: Any, **kwargs: Any) -> Any: + return await self._acall("aresponses", _instrument_func_type_responses, args, kwargs) + def _set_wrapper_attr(self, func_wrapper: Any) -> None: func_wrapper.__func__.is_wrapper = True @@ -586,7 +690,7 @@ def _set_wrapper_attr(self, func_wrapper: Any) -> None: def _observe(function, *args): """Telemetry extraction must not change a successful SDK operation.""" try: - function(*args) + return function(*args) except Exception: # Do not log request, response, or exception content (which can include secrets). logger.debug("Could not extract LiteLLM telemetry") @@ -613,17 +717,40 @@ def _record_error(span, error): span.record_exception(error) +def _first_present(*values): + return next((value for value in values if value is not None), None) + + def _set_usage(span, usage): - if hasattr(usage, "model_dump"): - usage = usage.model_dump() - if not isinstance(usage, Mapping): - return - for key, attribute in ( - ("prompt_tokens", SpanAttributes.GEN_AI_USAGE_INPUT_TOKENS), - ("completion_tokens", SpanAttributes.GEN_AI_USAGE_OUTPUT_TOKENS), - ("total_tokens", SpanAttributes.GEN_AI_USAGE_TOTAL_TOKENS), + usage = as_dict(usage) + inputs = as_dict(_first_present( + usage.get("prompt_tokens_details"), usage.get("input_tokens_details") + )) + outputs = as_dict(_first_present( + usage.get("completion_tokens_details"), usage.get("output_tokens_details") + )) + for attribute, value in ( + (SpanAttributes.GEN_AI_USAGE_INPUT_TOKENS, _first_present( + usage.get("prompt_tokens"), usage.get("input_tokens") + )), + (SpanAttributes.GEN_AI_USAGE_OUTPUT_TOKENS, _first_present( + usage.get("completion_tokens"), usage.get("output_tokens") + )), + (SpanAttributes.GEN_AI_USAGE_TOTAL_TOKENS, usage.get("total_tokens")), + (SpanAttributes.GEN_AI_USAGE_INPUT_TOKENS_CACHE_READ, _first_present( + inputs.get("cached_tokens"), inputs.get("cache_read_tokens"), + usage.get("cache_read_input_tokens") + )), + (SpanAttributes.GEN_AI_USAGE_INPUT_TOKENS_CACHE_WRITE, _first_present( + inputs.get("cache_creation_tokens"), inputs.get("cache_write_tokens"), + usage.get("cache_creation_input_tokens") + )), + (SpanAttributes.GEN_AI_USAGE_OUTPUT_TOKENS_REASONING, outputs.get("reasoning_tokens")), + (SpanAttributes.GEN_AI_USAGE_INPUT_TOKENS_AUDIO, inputs.get("audio_tokens")), + (SpanAttributes.GEN_AI_USAGE_OUTPUT_TOKENS_AUDIO, outputs.get("audio_tokens")), + (SpanAttributes.GEN_AI_COST_TOTAL, usage.get("cost")), ): - _set_span_attribute(span, attribute, usage.get(key)) + _set_span_attribute(span, attribute, value) class _StreamState: @@ -634,22 +761,33 @@ def __init__(self, span, span_cm=None, config=None): self.processed_chunks = [] self.content_by_choice = {} self.has_tool_context = False + self.tools_by_choice = {} + self.functions_by_choice = {} + self.responses = ResponseEvents() self._ended = False - self._closed = False def _process(self, chunk): - if hasattr(chunk, "to_dict"): - chunk = chunk.to_dict() - elif hasattr(chunk, "__dict__"): - chunk = chunk.__dict__ - if not isinstance(chunk, Mapping): + _set_result_metadata(self.span, chunk) + chunk = as_dict(chunk) + self.processed_chunks.append(_private_response(chunk, self.config)) + if chunk.get("type", "").startswith("response.") or chunk.get("type") == "error": + response = self.responses.add(chunk) + _set_result_metadata(self.span, response) return - self.processed_chunks.append(chunk) choices = chunk.get("choices") or [] for position, choice in enumerate(choices): index = choice.get("index", position) delta = choice.get("delta") or {} self.has_tool_context |= _has_tool_context(delta) + tools = self.tools_by_choice.setdefault(index, {}) + for tool_position, tool in enumerate(delta.get("tool_calls") or []): + tool_index = tool.get("index", tool_position) + target = tools.setdefault(tool_index, {"function": {}}) + if tool.get("id"): + target["id"] = tool["id"] + _join_function(target["function"], tool.get("function") or {}) + if delta.get("function_call"): + _join_function(self.functions_by_choice.setdefault(index, {}), delta["function_call"]) self.content_by_choice.setdefault(index, "") content = delta.get("content") if isinstance(content, str): @@ -659,12 +797,8 @@ def _process(self, chunk): f"{SpanAttributes.GEN_AI_OUTPUT_MESSAGES}.{index}.{MessageAttributes.MESSAGE_ROLE}", delta.get("role"), ) - _set_usage(self.span, chunk.get("usage")) - _set_span_attribute( - self.span, SpanAttributes.GEN_AI_RESPONSE_MODEL, chunk.get("model") - ) - def _finish(self, error=None): + def _finish(self, error=None, abandoned=False): if self._ended: return self._ended = True @@ -688,12 +822,24 @@ def _finish(self, error=None): f"{SpanAttributes.GEN_AI_OUTPUT_MESSAGES}.{index}.{MessageAttributes.MESSAGE_CONTENT}", text, ) - if error is None: - self.span.set_status(trace_api.StatusCode.OK) - else: + for index, tools in self.tools_by_choice.items(): + # Preserve both choice and tool indices even when interleaved. + for tool_index, tool in tools.items(): + for key, value in _message_attributes({"tool_calls": [tool]}, self.config): + key = key.replace(f"{MessageAttributes.MESSAGE_TOOL_CALLS}.0.", f"{MessageAttributes.MESSAGE_TOOL_CALLS}.{tool_index}.", 1) + _set_span_attribute(self.span, f"{SpanAttributes.GEN_AI_OUTPUT_MESSAGES}.{index}.{key}", value) + for index, function in self.functions_by_choice.items(): + for key, value in _message_attributes({"function_call": function}, self.config): + _set_span_attribute(self.span, f"{SpanAttributes.GEN_AI_OUTPUT_MESSAGES}.{index}.{key}", value) + _set_response_messages(self.span, self.responses.messages(), self.config) + if error is not None: _record_error(self.span, error) + elif self.responses.failed: + self.span.set_status(trace_api.StatusCode.ERROR) + elif not abandoned: + self.span.set_status(trace_api.StatusCode.OK) finally: - if self.span_cm is not None: + if self.span_cm is not None and not abandoned: self.span_cm.__exit__( type(error) if error else None, error, @@ -703,6 +849,36 @@ def _finish(self, error=None): self.span.end() +def _join_function(target, delta): + for field in ("name", "arguments"): + if isinstance(delta.get(field), str): + target[field] = target.get(field, "") + delta[field] + + +class _StreamWrapper: + def __init__(self, span, span_cm=None, config=None): + self.span = span + self._state = _StreamState(span, span_cm, config) + self._closed = False + # The callback owns only the state, not this wrapper or the SDK iterator. + # It cannot keep an unread/partly read stream alive. No async I/O or + # application-context detach is attempted during garbage collection. + self._finalizer = weakref.finalize(self, self._state._finish, None, True) + + @property + def _ended(self): + return self._state._ended + + def _process(self, chunk): + self._state._process(chunk) + + def _finish(self, error=None): + try: + self._state._finish(error) + finally: + self._finalizer.detach() + + def _stream_close_method(stream, *, asynchronous=False): """Use the SDK close API, or its provider stream on older LiteLLM versions.""" seen = set() @@ -718,7 +894,7 @@ def _stream_close_method(stream, *, asynchronous=False): return None -class StreamingIteratorWrapper(_StreamState): +class StreamingIteratorWrapper(_StreamWrapper): def __init__(self, iterator: Iterator[Any], span: trace_api.Span, span_cm=None, config=None): super().__init__(span, span_cm, config) self.iterator = iterator @@ -765,7 +941,7 @@ def __exit__(self, exc_type, exc_value, traceback): self.close() -class AsyncStreamingIteratorWrapper(_StreamState): +class AsyncStreamingIteratorWrapper(_StreamWrapper): def __init__( self, async_iterator: AsyncIterator[Any], span: trace_api.Span, span_cm=None, config=None ): diff --git a/python/frameworks/litellm/traceai_litellm/_responses.py b/python/frameworks/litellm/traceai_litellm/_responses.py new file mode 100644 index 000000000..ab9df5e44 --- /dev/null +++ b/python/frameworks/litellm/traceai_litellm/_responses.py @@ -0,0 +1,126 @@ +"""Normalize Responses items/events without importing version-specific SDK types.""" + +from copy import deepcopy +from typing import Mapping + + +def as_dict(value): + if hasattr(value, "model_dump"): + value = value.model_dump() + elif hasattr(value, "to_dict"): + value = value.to_dict() + return value if isinstance(value, Mapping) else {} + + +def response_messages(items): + """Translate Responses messages/tool linkage to the shared message schema.""" + if isinstance(items, str): + return [{"role": "user", "content": items}] + messages = [] + for item in items or []: + item = as_dict(item) + kind = item.get("type", "message") + if kind == "function_call": + messages.append( + { + "role": "assistant", + "tool_calls": [ + { + "id": item.get("call_id") or item.get("id"), + "function": { + "name": item.get("name"), + "arguments": item.get("arguments"), + }, + } + ], + } + ) + elif kind == "function_call_output": + messages.append( + { + "role": "tool", + "tool_call_id": item.get("call_id"), + "content": item.get("output"), + } + ) + elif kind == "message": + content = item.get("content") + if isinstance(content, list): + blocks = [] + for block in content: + block = as_dict(block) + if block.get("type") in ("input_text", "output_text"): + blocks.append({"type": "text", "text": block.get("text")}) + elif block.get("type") == "input_image": + blocks.append( + { + "type": "image_url", + "image_url": {"url": block.get("image_url")}, + } + ) + else: + blocks.append(dict(block)) + content = blocks + messages.append({"role": item.get("role", "user"), "content": content}) + return messages + + +class ResponseEvents: + """Keep deltas by output/content index; done snapshots replace, never append.""" + + def __init__(self): + self.items = {} + self.failed = False + + def add(self, event): + kind = event.get("type", "") + index = event.get("output_index", 0) + if kind in ("response.output_item.added", "response.output_item.done"): + self.items[index] = deepcopy(dict(as_dict(event.get("item")))) + elif kind in ( + "response.content_part.added", + "response.content_part.done", + "response.output_text.delta", + "response.output_text.done", + ): + item = self.items.setdefault( + index, {"type": "message", "role": "assistant", "content": []} + ) + parts = item.setdefault("content", []) + content_index = event.get("content_index", 0) + while len(parts) <= content_index: + parts.append({"type": "output_text", "text": ""}) + if kind.startswith("response.content_part."): + parts[content_index] = deepcopy(dict(as_dict(event.get("part")))) + elif kind.endswith(".delta"): + parts[content_index]["text"] = parts[content_index].get( + "text", "" + ) + event.get("delta", "") + else: + parts[content_index]["text"] = event.get("text", "") + elif kind in ( + "response.function_call_arguments.delta", + "response.function_call_arguments.done", + ): + item = self.items.setdefault( + index, {"type": "function_call", "arguments": ""} + ) + if kind.endswith(".delta"): + item["arguments"] = item.get("arguments", "") + event.get("delta", "") + else: + item["arguments"] = event.get("arguments", "") + if event.get("name"): + item["name"] = event["name"] + response = as_dict(event.get("response")) + if kind in ("response.completed", "response.incomplete", "response.failed"): + if response.get("output") is not None: + self.items = { + i: deepcopy(dict(as_dict(item))) + for i, item in enumerate(response["output"]) + } + if kind in ("error", "response.failed") or response.get("status") == "failed": + self.failed = True + return response + + def messages(self): + return response_messages([self.items[i] for i in sorted(self.items)]) diff --git a/python/tests/test_framework_litellm.py b/python/tests/test_framework_litellm.py index 2d68877da..78ea3e419 100644 --- a/python/tests/test_framework_litellm.py +++ b/python/tests/test_framework_litellm.py @@ -127,27 +127,22 @@ def test_instrumentation_dependencies(self): assert len(dependencies) >= 1 def test_instrument_basic(self, tracer_provider, config, mock_litellm): - """Test basic instrumentation setup.""" + """Store exactly the supported endpoints present in this SDK version.""" + import litellm + instrumentor = LiteLLMInstrumentor() - - # Mock the original functions before instrumentation - instrumentor.original_litellm_funcs["completion"] = mock_litellm["completion"] - instrumentor.original_litellm_funcs["acompletion"] = mock_litellm["acompletion"] - instrumentor.original_litellm_funcs["completion_with_retries"] = mock_litellm["completion_with_retries"] - instrumentor.original_litellm_funcs["acompletion_with_retries"] = mock_litellm["acompletion_with_retries"] - instrumentor.original_litellm_funcs["embedding"] = mock_litellm["embedding"] - instrumentor.original_litellm_funcs["aembedding"] = mock_litellm["aembedding"] - instrumentor.original_litellm_funcs["image_generation"] = mock_litellm["image_generation"] - instrumentor.original_litellm_funcs["aimage_generation"] = mock_litellm["aimage_generation"] - - instrumentor._instrument(tracer_provider=tracer_provider, config=config) - - # Verify tracer was created - assert hasattr(instrumentor, '_tracer') - assert isinstance(instrumentor._tracer, FITracer) - - # Verify original functions were stored - assert len(instrumentor.original_litellm_funcs) == 8 + instrumentor.original_litellm_funcs.clear() + expected = set(mock_litellm) - {"acompletion_with_retries"} + expected.update(name for name in ("responses", "aresponses") if hasattr(litellm, name)) + originals = {name: getattr(litellm, name) for name in expected} + try: + instrumentor._instrument(tracer_provider=tracer_provider, config=config) + assert isinstance(instrumentor._tracer, FITracer) + assert instrumentor.original_litellm_funcs == originals + finally: + instrumentor._uninstrument() + for name, original in originals.items(): + assert getattr(litellm, name) is original def test_instrument_without_tracer_provider(self, config, mock_litellm): """Test instrumentation without explicit tracer provider.""" From 092839aff5d5270a4d75bc8f70e2b16457da4082 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Wed, 4 Nov 2026 12:29:18 -0800 Subject: [PATCH 10/11] fix(litellm): protect tool credentials and complete text telemetry Project connection-bearing tool settings before serialization, add the supported Responses runtime extra, trace sync and async text completions, and retain reported finish reasons by choice. Cover real HTTP streams, errors, privacy and SDK floor behavior in cloud-tested regressions. Co-Authored-By: Claude Code --- .github/workflows/litellm-python-tests.yml | 4 +- python/frameworks/litellm/README.md | 12 +- python/frameworks/litellm/poetry.lock | 411 ++++++++++++++++- python/frameworks/litellm/pyproject.toml | 6 + .../litellm/tests/test_install_contract.py | 14 + .../tests/test_qualification_regressions.py | 428 ++++++++++++++++++ .../tests/test_text_stream_lifecycle.py | 151 ++++++ .../litellm/traceai_litellm/__init__.py | 129 +++++- python/tests/test_framework_litellm.py | 5 +- 9 files changed, 1153 insertions(+), 7 deletions(-) create mode 100644 python/frameworks/litellm/tests/test_qualification_regressions.py create mode 100644 python/frameworks/litellm/tests/test_text_stream_lifecycle.py diff --git a/.github/workflows/litellm-python-tests.yml b/.github/workflows/litellm-python-tests.yml index c59a24484..1575b3b25 100644 --- a/.github/workflows/litellm-python-tests.yml +++ b/.github/workflows/litellm-python-tests.yml @@ -54,7 +54,9 @@ jobs: 'fi-instrumentation-otel==1.2.2' 'openai==1.40.0' 'httpx==0.27.2' \ pytest pytest-asyncio else - .venv/bin/pip install -e python/frameworks/litellm \ + # Use the same Responses runtime documented for users, not an + # undeclared runner-only MCP dependency. + .venv/bin/pip install -e 'python/frameworks/litellm[responses]' \ --upgrade litellm pytest pytest-asyncio fi .venv/bin/pip check diff --git a/python/frameworks/litellm/README.md b/python/frameworks/litellm/README.md index 947d48577..56f9afa7a 100644 --- a/python/frameworks/litellm/README.md +++ b/python/frameworks/litellm/README.md @@ -11,6 +11,14 @@ This integration provides support for using OpenTelemetry with the Litellm frame pip install traceAI-litellm ``` +For `responses` / `aresponses` on Python 3.10+, install the Responses runtime extra: + +```bash +pip install 'traceAI-litellm[responses]' +``` + +Current LiteLLM versions import `mcp` on the Responses path, even without MCP tools. The extra supplies that upstream runtime dependency; the base installation remains compatible with older LiteLLM versions without Responses support. + ### Set environment variables @@ -38,7 +46,9 @@ trace_provider = register( ``` ### Configure Litellm Instrumentation -Instrument the LiteLLM client to trace `completion` / `acompletion`, `completion_with_retries`, `embedding` / `aembedding`, and `image_generation` / `aimage_generation`. On LiteLLM versions that expose them, `responses` / `aresponses` are also traced. Completion and Responses streams are traced as you consume them; other SDK endpoints are not covered. +Instrument the LiteLLM client to trace `completion` / `acompletion`, `text_completion` / `atext_completion`, `completion_with_retries`, `embedding` / `aembedding`, and `image_generation` / `aimage_generation`. On LiteLLM versions that expose them, `responses` / `aresponses` are also traced (install the Responses extra above). Chat, text completion and Responses streams are traced as you consume them; other SDK endpoints are not covered. + +Reported chat/text completion finish reasons are exported in `gen_ai.response.finish_reasons`, ordered by choice index, and on each indexed output message. No finish reason is invented for an unread or partially consumed stream. Tool telemetry excludes MCP authorization, headers and server URLs; provider requests are unchanged. Function schemas are preserved; other built-in tools currently export only their type. Exhaust streams or explicitly call `close()` / `await aclose()` to finish telemetry and release provider resources promptly. Discarding a stream finalizes its span on garbage collection, but does not guarantee immediate transport cleanup. diff --git a/python/frameworks/litellm/poetry.lock b/python/frameworks/litellm/poetry.lock index 33b20538d..e4f892eed 100644 --- a/python/frameworks/litellm/poetry.lock +++ b/python/frameworks/litellm/poetry.lock @@ -217,6 +217,120 @@ files = [ {file = "certifi-2024.6.2.tar.gz", hash = "sha256:3cd43f1c6fa7dedc5899d69d3ad0398fd018ad1a17fba83ddaf78aa46c747516"}, ] +[[package]] +name = "cffi" +version = "2.1.1" +description = "Foreign Function Interface for Python calling C code." +optional = true +python-versions = ">=3.10" +groups = ["main"] +markers = "python_version >= \"3.10\" and extra == \"responses\" and platform_python_implementation != \"PyPy\"" +files = [ + {file = "cffi-2.1.1-cp310-cp310-macosx_10_15_x86_64.whl", hash = "sha256:baed1e86cc735622097354b9d1281406caf42ff42a886d29faa8e8d1630333be"}, + {file = "cffi-2.1.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ca82be1a1d406ecfe1d25dc16cb33488e5a16bf4438c9fb590484ea29d92478b"}, + {file = "cffi-2.1.1-cp310-cp310-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:42e2f76b9455f5a9a844f770bf3e200ed3da0e15f5df3db9c31fe80b04b3d004"}, + {file = "cffi-2.1.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:5a59cc1c4442bc3d5c703bf720b51138d0bfc173618807c9ee2490a7541dd3d9"}, + {file = "cffi-2.1.1-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:9f8d177621de5cb38ee3e731eda45d421db093ec0739f46a5594babda7987a98"}, + {file = "cffi-2.1.1-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:75f80557d1389eddbd0de2681f6a390a0c5338c31ddaa821381c203fc3fd50d9"}, + {file = "cffi-2.1.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:194cffa889098ced9976c3fc6340305e43f6303657d298da55366907c05c22d6"}, + {file = "cffi-2.1.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:5bb4e7ea95dcd6a014a6fef62e62467d67d8e582326443f3d68e71d6320a9fcf"}, + {file = "cffi-2.1.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:3d22a20b1fb1632cc72c22f95f7b0d2961c3e1c235f245ba4c606c4771035659"}, + {file = "cffi-2.1.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:1dea0e4d7d4f11f619fe8c1d76caf49e24405b4b5743c0e3be16a500ecd930c9"}, + {file = "cffi-2.1.1-cp310-cp310-win32.whl", hash = "sha256:7ce713ace7c0e4520535b42b77eaa742c16dab813978064913e5a3cf82973b41"}, + {file = "cffi-2.1.1-cp310-cp310-win_amd64.whl", hash = "sha256:a48d62ab9d6f4f98c983223a547af44be6ca3691074c31cecced6facd3ba2dc1"}, + {file = "cffi-2.1.1-cp311-cp311-macosx_10_15_x86_64.whl", hash = "sha256:c8d2c9fd1f2d16f780d15127abb050d13d1a76c03a4bd87d7e4980e45e511e12"}, + {file = "cffi-2.1.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:398aff33cee2767e3e781d2554c54bd0dff386bb437581e0d8011fde1a942ec1"}, + {file = "cffi-2.1.1-cp311-cp311-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:154852545011f779917b11c78db2358d095da62a9a172b78ad0a583ee5adc0d0"}, + {file = "cffi-2.1.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:3311ed60d36f83378794e1009ac6258bafbf81f7888b4caa7b35a521e3f95813"}, + {file = "cffi-2.1.1-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:6e192623c49c94421616a5778fba35cf0d5a8d000650c1967ef4448ee5cdd990"}, + {file = "cffi-2.1.1-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:a6e721d4b0e45d5b65e87534470e67b18dcd092c83f68fba09f152b9cbc061af"}, + {file = "cffi-2.1.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:34e261f78cb6ceaaa36f42f2613f4380d94d9c759a9c73c769ee6e0247364632"}, + {file = "cffi-2.1.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:7225e4514edb64eb6740324353e0da0711954fd8d7da4576755b1c6e09b697cd"}, + {file = "cffi-2.1.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:df913725b79db7bcf03448f36b7bf8815363417d5b58deecf9305e3e30f0f21a"}, + {file = "cffi-2.1.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:f5cfbc5fe74540d335175b656c725d74d90e3730c626d92575eea35029d9afaa"}, + {file = "cffi-2.1.1-cp311-cp311-win32.whl", hash = "sha256:f8ec5e643a9a937f64e1999eb9f75d072263751912dc5cd06d3c85f8f44be7c3"}, + {file = "cffi-2.1.1-cp311-cp311-win_amd64.whl", hash = "sha256:42f6930c31dc7f50732c9ae793c2786c7b6b044195967bbdde40bb9be81c4cc0"}, + {file = "cffi-2.1.1-cp311-cp311-win_arm64.whl", hash = "sha256:c7659f22557c5a0bc4855cd635f55edec690cc008a40768527762cb9fb263455"}, + {file = "cffi-2.1.1-cp312-cp312-macosx_10_15_x86_64.whl", hash = "sha256:c8c69575568085ba0b1b10c0249d779a214aea6f6522e949a0fc9fb0fcb449d0"}, + {file = "cffi-2.1.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f81b3b8f3d4e343550fa4baa0e479bba9f2d29ce9c2e9b51d1ce1718d7442fcf"}, + {file = "cffi-2.1.1-cp312-cp312-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:811bd1e21d32de12efca32393a0ab3f5133b54fce9bd44b8bd77ab07da14bf6a"}, + {file = "cffi-2.1.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:68e62fe11f30d5ca8289242866f0a5291402d8529ca2178ab8afc5c9694ae890"}, + {file = "cffi-2.1.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:4a7c934f7360e8cd64fe9efadcbd10c7c6364f531e432b9a4bf5ccbc9e0e8b50"}, + {file = "cffi-2.1.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:3143d81e29e1e20a9ce10901ec369012947876596f75a222235965f2b7ae832e"}, + {file = "cffi-2.1.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c1453022f490d2459a11819d83ad1d586e9ff65a12ac3e705ffebd46d3685dcf"}, + {file = "cffi-2.1.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:208f941bb9d18e768138677f0a6d2ce01f590df56043dda1df1535ac57c88517"}, + {file = "cffi-2.1.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:210019b6c7cf07f081b4c54635c8cf744377001350e29cc0f81c4377b4797735"}, + {file = "cffi-2.1.1-cp312-cp312-win32.whl", hash = "sha256:046bfc24911b37851ee1b51aab8bffe713d89c68c6a057b09484ce9fd5f69b4e"}, + {file = "cffi-2.1.1-cp312-cp312-win_amd64.whl", hash = "sha256:f53e442b08449d42821fa4a4fba000095af9f62742a500f978a9f557ec44339a"}, + {file = "cffi-2.1.1-cp312-cp312-win_arm64.whl", hash = "sha256:7bde5e4cc5c10140859842b9d383af292b22639a4dffb725314baf45968cef80"}, + {file = "cffi-2.1.1-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:b5bdfd1c873d4e093aabc0ca84c4ca6dbc4f752afb5c86f146d9742580c9da2e"}, + {file = "cffi-2.1.1-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:31348097ff5bbe827ccc41795d4dd099d9f0625e7def00ee653c137a490c2a6c"}, + {file = "cffi-2.1.1-cp313-cp313-macosx_10_15_x86_64.whl", hash = "sha256:9d2055050ea716bd38b7f7f1579c275386646b4894c155a3e2f3cd62ed41b7c6"}, + {file = "cffi-2.1.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:19ee6127ee34de7d83ce3d371ebc5ed91addbdcc39f9ab15ce4eb35a4e534971"}, + {file = "cffi-2.1.1-cp313-cp313-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:6a8dddef476fab96d066d578fc88526767b836ab5ab21754e1d5bf3879c31c7c"}, + {file = "cffi-2.1.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f16c709686a78c727bbbf059f92b0bf41c6fc60deec706d2dc19f529175a6125"}, + {file = "cffi-2.1.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:fcd22650c908d7b7da162bbfaab594a1227a15d1643a98c68b122ac642fa2264"}, + {file = "cffi-2.1.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:aa9511c62d14da7aacc9b4bf51f3f697a621e83b2d6919008243c3aad168eea3"}, + {file = "cffi-2.1.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:a931079504ecc49efed7744c476a5c343a92fabf66dec2db95edb1b2fdc770e2"}, + {file = "cffi-2.1.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a2d7755bef5a12ed488f4ef1f1b69ee9191d7396083b755a5d2295f6edb4768b"}, + {file = "cffi-2.1.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:e0bcb7e0f677f543555d2adff3bf19c05f66cdb4796e5ff602442ab2fe3c4ef7"}, + {file = "cffi-2.1.1-cp313-cp313-win32.whl", hash = "sha256:334644fbac4eff73d985a17a91226df55d0f394160c4cfb880e084c8f7161cac"}, + {file = "cffi-2.1.1-cp313-cp313-win_amd64.whl", hash = "sha256:1aa5645c30469b09530c4ebca77ebf8f17618293c58f8549cb1a543a50236e7d"}, + {file = "cffi-2.1.1-cp313-cp313-win_arm64.whl", hash = "sha256:63bbfd5ded17c4840ac07cd8f1c21ba9d9708141f840b324f422f41b207e3973"}, + {file = "cffi-2.1.1-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:7dbb61fe3a7699468030f71bbe5f8a0e326a151daa91beb11a6fc1f980c55e1c"}, + {file = "cffi-2.1.1-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:f24fb43132a4c6b4cb4eb029492919b2db645be6808d738f244fd146c03c32cb"}, + {file = "cffi-2.1.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:d28630f5854ab07ab1fd4aba756de52326c82e6be15d414b12793f1975048b54"}, + {file = "cffi-2.1.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:661c298b4821edebead0c91edd2b00374d67ad7c5a1f7a91d4442633b79d6a72"}, + {file = "cffi-2.1.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:58acb8ab8e295e6c5ea12f888cbb13cf21511ef2a3303a23f4325c29d17fe5c1"}, + {file = "cffi-2.1.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:456a61fa52d579ebf9df2e9552ead5129855dbaff6c1e5a9b1bc408809bdc062"}, + {file = "cffi-2.1.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:a4f00aa42f75d6e4595e8866e748cc1705adc0cddfeb2ca86d0d03993d63ba03"}, + {file = "cffi-2.1.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:b0431303acaea1089ad4b3e9ce4e6518193def1118d4073ca848635ee4ea2e96"}, + {file = "cffi-2.1.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:64faea20f4e2613363a1a9b9c7dd73058f3ecd00133a511e72ad7c511658f527"}, + {file = "cffi-2.1.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5c58fe613dc5e5336357eff555824a314d8e43282600435c8d1cb6a7a2fedd13"}, + {file = "cffi-2.1.1-cp314-cp314-win32.whl", hash = "sha256:1a18a57b58cfb21fc28d72e876acf10eaed67a1ed96226f92af4df681d571c4c"}, + {file = "cffi-2.1.1-cp314-cp314-win_amd64.whl", hash = "sha256:3222ba5d678f80a030e6afbcc33dc1ae5cb45facabb61cee2c7016b8432fde48"}, + {file = "cffi-2.1.1-cp314-cp314-win_arm64.whl", hash = "sha256:ab36d55f9ed2d067327667c2fea18dda018eb628dd6347aa01dda6cf1f5d3836"}, + {file = "cffi-2.1.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:7750c6449dff7864bb9bb27ddfb0267756189201a3afc911d82b3caacd70dfc3"}, + {file = "cffi-2.1.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:0beceaabe56af686895136a2de78db54ecd8e4046b236b8fd6d6cb61389e9bf2"}, + {file = "cffi-2.1.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:49cbc70e6542d4ccccb936558d1064a8012541e78f821f955cff24e357776c94"}, + {file = "cffi-2.1.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:e2d65b31f36619cda3999b78b2aa9632e76b78448e7a56fc4240824200e7c4fc"}, + {file = "cffi-2.1.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:28907ab9bfb6aa13184cfc17c6b8e1023c5ab6fd7076d8c20a35e59fe04f8f29"}, + {file = "cffi-2.1.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:51b31d1c98274844cfd7838ce00bfc27c7423a4dc00fc0772fc3331c2cc90676"}, + {file = "cffi-2.1.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:5e7cecbaadb83884793e05828cee59b210b24583b9c7425d0ba6a754fe22eb4e"}, + {file = "cffi-2.1.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:25792eac27877609e7bb06d42ff88278a6624fff2ba9bbb523c09616b117e80f"}, + {file = "cffi-2.1.1-cp314-cp314t-win32.whl", hash = "sha256:8ef53b2de9bcb9197d31854256575d59dbac0cba72ac627bb291ef5eceb74be4"}, + {file = "cffi-2.1.1-cp314-cp314t-win_amd64.whl", hash = "sha256:616f097f2fe415bc92a247f02e11f634e1f9e9a83d327e3c915c15089c87869e"}, + {file = "cffi-2.1.1-cp314-cp314t-win_arm64.whl", hash = "sha256:ad2c86c495b899d862ea0f4b42891b8713a3bd45dd4105c7fd51c2a72f39f3a5"}, + {file = "cffi-2.1.1-cp315-cp315-ios_13_0_arm64_iphoneos.whl", hash = "sha256:dddad92b554513a31f272570678ba307fb9f618f05e3d4a5eacafff9eae03e1d"}, + {file = "cffi-2.1.1-cp315-cp315-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:da0e573f9f97159390c89d9f1a9e41908b66d408cc5b58d08cf3847d844c531b"}, + {file = "cffi-2.1.1-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:fb92203a88b3d3053034db775110081c49d28be6551923805e039924093761e4"}, + {file = "cffi-2.1.1-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:2ae64be792b8966f2c69538199728b290e34726562896df1e5dc8ffd8d8188e8"}, + {file = "cffi-2.1.1-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:507a24c282e0f42f8ed737cf048572cbf580468da5555764a8331735e9c736b6"}, + {file = "cffi-2.1.1-cp315-cp315-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:246fa40ce8645a614ff682e0b70f37134e460eaf93a775e0cbe3cca585a67a80"}, + {file = "cffi-2.1.1-cp315-cp315-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:471cee653ae88de62096552e6d24ccb4a5adb8c8c9f10b5054d0122c15bf2779"}, + {file = "cffi-2.1.1-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:aeae0e330c9f6acd681f647d46cefd30c29f93e3392882e792e82080c9691399"}, + {file = "cffi-2.1.1-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:42a494cee34437f05546455144f2b5d9ac09b1face62bcfce597d2e521066688"}, + {file = "cffi-2.1.1-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:cc572dace3f60ef98d7b12ff411d20f5362feb31a0439eab0085bbfd349982d7"}, + {file = "cffi-2.1.1-cp315-cp315-win32.whl", hash = "sha256:4f42141fc14250de6dde5ee7ea4432be017252d91f19c5ad043c084cea629cac"}, + {file = "cffi-2.1.1-cp315-cp315-win_amd64.whl", hash = "sha256:e6e8cff14d6fb0be70a09c0bdc58096f501952d04624ebf867e0e56da2df8960"}, + {file = "cffi-2.1.1-cp315-cp315-win_arm64.whl", hash = "sha256:27350daa11d4f10c540e6e89dada4c54feb7256ad03e9a4dc075ebad7ba360d1"}, + {file = "cffi-2.1.1-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:c26608d2222fb1e94487e4a387d85f13eb55d5ed725cb25a0c589ac4ee60e7bc"}, + {file = "cffi-2.1.1-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:4be96343e422f2dfcd12ab5c9f5aebe03f82f737c6bffeca6830b3875cb44aab"}, + {file = "cffi-2.1.1-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:937c0052c05a31ca1daf18de3158eed4dbfcb9cc107adbea227728d647be701e"}, + {file = "cffi-2.1.1-cp315-cp315t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:df423d40ee8654634421812bc3b196da3f9bd7d32929da813f8394c4348a5358"}, + {file = "cffi-2.1.1-cp315-cp315t-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:a730a083190634c65cca36ba5f489531576ebd79bcd5c8e172130f6453127231"}, + {file = "cffi-2.1.1-cp315-cp315t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:363e05fa78e15116c3c32c210ee36884fd6b9afa6d440e47112c3bd511d64cb6"}, + {file = "cffi-2.1.1-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:770de9db11e84213beec501cfcaa013b019820ca881e03344dea5844f7876d94"}, + {file = "cffi-2.1.1-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:7da0c5eff80f0197f3b3d1232ec5a682a9325f4ae9016a78f5f5ca35f9ced1f5"}, + {file = "cffi-2.1.1-cp315-cp315t-win32.whl", hash = "sha256:06c72bb76605a4b0cd0aad6930b69d4baf7dd5d806cfc409b824191099700e66"}, + {file = "cffi-2.1.1-cp315-cp315t-win_amd64.whl", hash = "sha256:d9c275eaacd24aa73f94ffd6de08fc3f932424d8b6c376f4bed7cde376fe7bc3"}, + {file = "cffi-2.1.1-cp315-cp315t-win_arm64.whl", hash = "sha256:d18e5ac0f2f03f4f518d3e23db0f0cad7faa1da8620e9c09461d443bbf6e6692"}, + {file = "cffi-2.1.1.tar.gz", hash = "sha256:dd31f52ea1086513bb9df30f8fcee9b8918323ae067a3d5b78bc826a000712be"}, +] + +[package.dependencies] +pycparser = {version = "*", markers = "implementation_name != \"PyPy\""} + [[package]] name = "charset-normalizer" version = "3.3.2" @@ -345,6 +459,83 @@ files = [ {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, ] +[[package]] +name = "cryptography" +version = "50.0.2" +description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." +optional = true +python-versions = "!=3.9.0,!=3.9.1,>=3.9" +groups = ["main"] +markers = "python_version >= \"3.10\" and extra == \"responses\"" +files = [ + {file = "cryptography-50.0.2-cp311-abi3-macosx_11_0_arm64.whl", hash = "sha256:fa8f5efb344d6908a1ce62f4a24e2e5780f825d6f53f5f50ec5ffacac72936cb"}, + {file = "cryptography-50.0.2-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:79def8d059362e7831389ed3be0ecdf58a89386e1271e35dd9f5af84e81bffd0"}, + {file = "cryptography-50.0.2-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:630ebfea3bf689d075f82316324ff7433dc447fe6bc1bfc76524b74b4a9567d2"}, + {file = "cryptography-50.0.2-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:f9f6143a8c75945eb960d9eb98905a441394abfa24afaae239d514ffb2586480"}, + {file = "cryptography-50.0.2-cp311-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:a582ab2ae1d34f67112cadc86702774c9ea4374df6bca6afe672817203c99134"}, + {file = "cryptography-50.0.2-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:4061c0079120205fb760c58acab6443e217307dcf05e3702cf970e0689972856"}, + {file = "cryptography-50.0.2-cp311-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:ac9ed99d81760c62fe89d5f0815cdfa1ba9a35141cf30f1c2d044f04b4803d2e"}, + {file = "cryptography-50.0.2-cp311-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:87e9ce85beb6b328ba370cc6e6aea483c92617b4c95b1d33a49297eb662bfb04"}, + {file = "cryptography-50.0.2-cp311-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:f265528741e048bce55c3463ed721fb0aa45a5888d8add8cfeccb3035451bbdc"}, + {file = "cryptography-50.0.2-cp311-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:9dab55f57c74c3cad24c323bacbbd04be4705ba6eb0d92e920b1fc4837ed5079"}, + {file = "cryptography-50.0.2-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:25784ce8b9621c90c643efb9e1e2162ab3b0224cae446ad5e70e7fcb1ce18b51"}, + {file = "cryptography-50.0.2-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:85d0d9a31b9098e98534226d5686b47264b95e62ce459dc2e62fdfc809f9fe93"}, + {file = "cryptography-50.0.2-cp311-abi3-win_amd64.whl", hash = "sha256:7afa5a6602a9f29af1f3a2965f831bae7c9d5d597b7cbb716d41ab3b7d89879c"}, + {file = "cryptography-50.0.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:f785f6161f202ab04d8ca194158968798e480ca058943907972da5f12e2881e8"}, + {file = "cryptography-50.0.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:0ecbc5652bdb6fc9eaf89a7d196e20941adfe812f43bc4ca05d9150496821047"}, + {file = "cryptography-50.0.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ab50ee449bf968271e820086f10a33d101dd060370abc10bcd22279be2656539"}, + {file = "cryptography-50.0.2-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:a9f7355e6fab51f6c369b86fb7571cffa05edee2c2121e0380a37fb9ac1cd5c1"}, + {file = "cryptography-50.0.2-cp314-cp314t-manylinux_2_28_ppc64le.whl", hash = "sha256:94e5e9f108ee10471288214d3d233fbfbb492840a8457eb85178d643ddeb32c7"}, + {file = "cryptography-50.0.2-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:241449bf940a5d27309bd317e6f9a2af6932113818bb2b8f5c59ddc7ef16da18"}, + {file = "cryptography-50.0.2-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:d8947001be83df1394050758ce0e745dd74fb134eef0a4b5124208dfc3a68c37"}, + {file = "cryptography-50.0.2-cp314-cp314t-manylinux_2_34_aarch64.whl", hash = "sha256:4a20ce1e5cb4284a86692fdcba7cb8754185c6b2e5c56fcef3751cf451d3cdc2"}, + {file = "cryptography-50.0.2-cp314-cp314t-manylinux_2_34_ppc64le.whl", hash = "sha256:84f964e537f916e2cc85199e5a88742e964939b575ac8598b3f9d6cc416cdaf1"}, + {file = "cryptography-50.0.2-cp314-cp314t-manylinux_2_34_x86_64.whl", hash = "sha256:828d49b0ff5a0e3975865571c5d91dbbdd0d38d8289b249a163e9425413a5e05"}, + {file = "cryptography-50.0.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:deb9fde5c60e437ee4821bc9bc39ff31b42135c27e1dc61ef0a629389c1de62e"}, + {file = "cryptography-50.0.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:8c71ba2cd31fc93748c38e1b613200ff1c2665cbfd5341fe3a61cfde35a1430e"}, + {file = "cryptography-50.0.2-cp314-cp314t-win_amd64.whl", hash = "sha256:78198641e5be9521beea5aa782bb551a58068d10e6eb04c9c680c1b69f2e7d45"}, + {file = "cryptography-50.0.2-cp315-abi3.abi3t-macosx_11_0_arm64.whl", hash = "sha256:edc3342adf8f697fc5f59c887a304356f147b397809440ed64e2fa6af2f50f37"}, + {file = "cryptography-50.0.2-cp315-abi3.abi3t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d370b8d1dfcdf7130178137f6fbee6140774a1acc6cacefc4b42643ec11d0a3a"}, + {file = "cryptography-50.0.2-cp315-abi3.abi3t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f2f9bd7f90c64fe89253f0a2c05e3c4856072660429ce8831b4235bf29403a67"}, + {file = "cryptography-50.0.2-cp315-abi3.abi3t-manylinux_2_28_aarch64.whl", hash = "sha256:e275096ea1e60cc595cda2836fd4a6c725d1125108b868be17f53684d164e2cc"}, + {file = "cryptography-50.0.2-cp315-abi3.abi3t-manylinux_2_28_ppc64le.whl", hash = "sha256:b13478603dcd0a2479ff8e87e2c19a7d525734686fe3c49542472293a204212d"}, + {file = "cryptography-50.0.2-cp315-abi3.abi3t-manylinux_2_28_x86_64.whl", hash = "sha256:58a0c478eeca76fe5e07993c5a0703def34a6dc6a0cda4f5564639b33112ffe7"}, + {file = "cryptography-50.0.2-cp315-abi3.abi3t-manylinux_2_31_armv7l.whl", hash = "sha256:d38cdff612d06fa6a32840d5e1b1f7a27cee4a349aa9085d94a67789d6bfd408"}, + {file = "cryptography-50.0.2-cp315-abi3.abi3t-manylinux_2_34_aarch64.whl", hash = "sha256:fdd28f912fccfec1846a94e2e1e8f9b0012f557f0c46fe4f3eb0d7a87afcf90b"}, + {file = "cryptography-50.0.2-cp315-abi3.abi3t-manylinux_2_34_ppc64le.whl", hash = "sha256:cbc8738fd8526d80f35cb3a40d41f41a2e7030bb3b18b09a6778ef63d291c2fd"}, + {file = "cryptography-50.0.2-cp315-abi3.abi3t-manylinux_2_34_x86_64.whl", hash = "sha256:e105ab60406787da31fccc883fc0f733af1efd78f0136a4599692c4083a73d0c"}, + {file = "cryptography-50.0.2-cp315-abi3.abi3t-musllinux_1_2_aarch64.whl", hash = "sha256:6f8700550aa1474a91e5dc07049c46f98b423b5b1ddd0483e0b51362eeeaf5be"}, + {file = "cryptography-50.0.2-cp315-abi3.abi3t-musllinux_1_2_x86_64.whl", hash = "sha256:c71be1cbfa5cd9a41ee452acf1eccd82b2c05950358b106ec8ceb83411d1a020"}, + {file = "cryptography-50.0.2-cp315-abi3.abi3t-win_amd64.whl", hash = "sha256:c423ab384a46c4dff7217b2ea5ba2e11cffdeab6441acd04cf65a369caf0366c"}, + {file = "cryptography-50.0.2-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:0ec5f09541743261e66e291b4a0cbf0fb2997aeaab6d9e9c740b9dba1b58d1c2"}, + {file = "cryptography-50.0.2-cp39-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:c5e67125c7dca78d199ec4e116aa93dbb83494808ecbb8211a2cb09b1bf41dbd"}, + {file = "cryptography-50.0.2-cp39-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ee247f5c245c9a2fe7c8e2214e295918838e44e00a45a6718451e4004219e767"}, + {file = "cryptography-50.0.2-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:dfe9763530994147d9af1def057a5b9658b00e8f8fe8743d144d1e0911c2e454"}, + {file = "cryptography-50.0.2-cp39-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:58ddb5a8e3179d12f19e4ea34d2d32e9d63a4baa142c875c1eb59f41b7243acd"}, + {file = "cryptography-50.0.2-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:f21e8a22c8605750c7af886bab299a363721264061b4ac0a30efb73cfd58efc5"}, + {file = "cryptography-50.0.2-cp39-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:9c8402a82ea0dc4ceeab793db05f0fafa8ca139ca34fcde5df0f596103c74107"}, + {file = "cryptography-50.0.2-cp39-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:0ddc924c04591c2811ca024d62ecad4f7f6f08af8939c211438f48a16bd23602"}, + {file = "cryptography-50.0.2-cp39-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:a6557e5f38e065ca9fbdaf7cfc7435ecb1d113aa81a022d1b51921ee7432e227"}, + {file = "cryptography-50.0.2-cp39-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:1981f1db4630889b9ef7803fadef12b056f428cb6b85c27ba57b774793b6093c"}, + {file = "cryptography-50.0.2-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:7a8701d6b584d76e909e3d305b7d126b41439876a5aaf76cddc67fc230eafa2e"}, + {file = "cryptography-50.0.2-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:ce47f66801c20ec6c6632453bb5960fe38939e9306970b48b3a5a26de7745d94"}, + {file = "cryptography-50.0.2-cp39-abi3-win_amd64.whl", hash = "sha256:4e81d95e5bafc2d6e34e4bed780e53e4d5b9a2f928573428aa4d35fbec1eb0de"}, + {file = "cryptography-50.0.2-pp311-pypy311_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:92e665960f25fcdc73725b9cec7a3824f279ba97a98653afe9ffac2e43668f67"}, + {file = "cryptography-50.0.2-pp311-pypy311_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:eef4c2f3423810b3070ab391f85436d2f8bbfcb286ac15cbc73190b3563b1f1a"}, + {file = "cryptography-50.0.2-pp311-pypy311_pp73-manylinux_2_34_aarch64.whl", hash = "sha256:7c6d0330c472d96f6a6afe24d80dfdf15176c33096f0a4397ae4c60f3dd3be48"}, + {file = "cryptography-50.0.2-pp311-pypy311_pp73-manylinux_2_34_x86_64.whl", hash = "sha256:1ba34f04897fcdaa73f74145c25f3ec146fbd56593853e88adc2e811303c5f42"}, + {file = "cryptography-50.0.2-pp311-pypy311_pp80-macosx_11_0_arm64.whl", hash = "sha256:3dc4fd8058cea1644971207d530e1a03a184a805ffc8ebdddf0599d78a331b81"}, + {file = "cryptography-50.0.2-pp311-pypy311_pp80-win_amd64.whl", hash = "sha256:7b75de3c8b3be1cdb1052747c929440c3eea46c1bc2cb8a6e3a48388e9b7b452"}, + {file = "cryptography-50.0.2.tar.gz", hash = "sha256:7b46165bb56eb4704e2eaaf86f3c940d19154535d9b0ca7d6d590b04060e00d5"}, +] + +[package.dependencies] +cffi = {version = ">=2.0.0", markers = "platform_python_implementation != \"PyPy\""} +typing-extensions = {version = ">=4.13.2", markers = "python_full_version < \"3.11.0\""} + +[package.extras] +ssh = ["bcrypt (>=3.1.5)"] + [[package]] name = "distro" version = "1.9.0" @@ -673,6 +864,19 @@ http2 = ["h2 (>=3,<5)"] socks = ["socksio (==1.*)"] zstd = ["zstandard (>=0.18.0)"] +[[package]] +name = "httpx-sse" +version = "0.4.3" +description = "Consume Server-Sent Event (SSE) messages with HTTPX." +optional = true +python-versions = ">=3.9" +groups = ["main"] +markers = "python_version >= \"3.10\" and extra == \"responses\"" +files = [ + {file = "httpx_sse-0.4.3-py3-none-any.whl", hash = "sha256:0ac1c9fe3c0afad2e0ebb25a934a59f4c7823b60792691f779fad2c5568830fc"}, + {file = "httpx_sse-0.4.3.tar.gz", hash = "sha256:9b1ed0127459a66014aec3c56bebd93da3c1bc8bb6618c8082039a44889a755d"}, +] + [[package]] name = "huggingface-hub" version = "0.33.2" @@ -991,6 +1195,40 @@ files = [ {file = "markupsafe-3.0.2.tar.gz", hash = "sha256:ee55d3edf80167e48ea11a923c7386f4669df67d7994554387f84e7d8b0a2bf0"}, ] +[[package]] +name = "mcp" +version = "1.30.0" +description = "Model Context Protocol SDK" +optional = true +python-versions = ">=3.10" +groups = ["main"] +markers = "python_version >= \"3.10\" and extra == \"responses\"" +files = [ + {file = "mcp-1.30.0-py3-none-any.whl", hash = "sha256:666edb5009503e1047c9d60346a756f94b261f05cc2625f23d41c728ffc484d0"}, + {file = "mcp-1.30.0.tar.gz", hash = "sha256:445414625fce5c295faa505bb11bacece661ab6f4028d57c935db57820b7a3e4"}, +] + +[package.dependencies] +anyio = ">=4.5" +httpx = ">=0.27.1,<1.0.0" +httpx-sse = ">=0.4" +jsonschema = ">=4.20.0" +pydantic = {version = ">=2.11.0,<3.0.0", markers = "python_version < \"3.14\""} +pydantic-settings = ">=2.5.2" +pyjwt = {version = ">=2.10.1", extras = ["crypto"]} +python-multipart = ">=0.0.9" +pywin32 = {version = ">=310", markers = "sys_platform == \"win32\" and python_version < \"3.14\""} +sse-starlette = ">=1.6.1" +starlette = {version = ">=0.27", markers = "python_version < \"3.14\""} +typing-extensions = ">=4.9.0" +typing-inspection = ">=0.4.1" +uvicorn = {version = ">=0.31.1", markers = "sys_platform != \"emscripten\""} + +[package.extras] +cli = ["python-dotenv (>=1.0.0)", "typer (>=0.16.0)"] +rich = ["rich (>=13.9.4)"] +ws = ["websockets (>=15.0.1)"] + [[package]] name = "multidict" version = "6.6.3" @@ -1401,6 +1639,19 @@ files = [ {file = "protobuf-5.29.5.tar.gz", hash = "sha256:bc1463bafd4b0929216c35f437a8e28731a2b7fe3d98bb77a600efced5a15c84"}, ] +[[package]] +name = "pycparser" +version = "3.11" +description = "C parser in Python" +optional = true +python-versions = ">=3.10" +groups = ["main"] +markers = "python_version >= \"3.10\" and extra == \"responses\" and platform_python_implementation != \"PyPy\" and implementation_name != \"PyPy\"" +files = [ + {file = "pycparser-3.11-py3-none-any.whl", hash = "sha256:51d5a8ba2be0bbe440b99d2112604c95bbbc3c2748a64260186c541e1729cd80"}, + {file = "pycparser-3.11.tar.gz", hash = "sha256:d875f09c3507d00e1aba0eecc6dcadc1352f30fff09dc6bff2f1c2935e97c2bc"}, +] + [[package]] name = "pydantic" version = "2.11.7" @@ -1535,6 +1786,51 @@ files = [ [package.dependencies] typing-extensions = ">=4.6.0,<4.7.0 || >4.7.0" +[[package]] +name = "pydantic-settings" +version = "2.15.0" +description = "Settings management using Pydantic" +optional = true +python-versions = ">=3.10" +groups = ["main"] +markers = "python_version >= \"3.10\" and extra == \"responses\"" +files = [ + {file = "pydantic_settings-2.15.0-py3-none-any.whl", hash = "sha256:0ba092c291c94baceb5eff768aa0d56400a457585bc0175925a5a5510303da42"}, + {file = "pydantic_settings-2.15.0.tar.gz", hash = "sha256:694b793e84f766ba76a90ebdefc01d0a9a045dab0382bee70393da93712ad117"}, +] + +[package.dependencies] +pydantic = ">=2.7.0" +python-dotenv = ">=0.21.0" +typing-inspection = ">=0.4.0" + +[package.extras] +aws-secrets-manager = ["boto3 (>=1.35.0)"] +azure-key-vault = ["azure-identity (>=1.16.0)", "azure-keyvault-secrets (>=4.8.0)"] +gcp-secret-manager = ["google-cloud-secret-manager (>=2.23.1)"] +toml = ["tomli (>=2.0.1)"] +yaml = ["pyyaml (>=6.0.1)"] + +[[package]] +name = "pyjwt" +version = "2.15.1" +description = "JSON Web Token implementation in Python" +optional = true +python-versions = ">=3.9" +groups = ["main"] +markers = "python_version >= \"3.10\" and extra == \"responses\"" +files = [ + {file = "pyjwt-2.15.1-py3-none-any.whl", hash = "sha256:42d59d631f7768a1028a64c7ff581a9bf7519804daf91fc5b6c56e30eec5e193"}, + {file = "pyjwt-2.15.1.tar.gz", hash = "sha256:4f259e80cdfb6b3fc18a7de51fd1ef9ec79652f25019bae68975ca2468a34df8"}, +] + +[package.dependencies] +cryptography = {version = ">=3.4.0", optional = true, markers = "extra == \"crypto\""} +typing_extensions = {version = ">=4.0", markers = "python_version < \"3.11\""} + +[package.extras] +crypto = ["cryptography (>=3.4.0)"] + [[package]] name = "python-dotenv" version = "1.1.1" @@ -1550,6 +1846,51 @@ files = [ [package.extras] cli = ["click (>=5.0)"] +[[package]] +name = "python-multipart" +version = "0.0.32" +description = "A streaming multipart parser for Python" +optional = true +python-versions = ">=3.10" +groups = ["main"] +markers = "python_version >= \"3.10\" and extra == \"responses\"" +files = [ + {file = "python_multipart-0.0.32-py3-none-any.whl", hash = "sha256:ff6d3f776f16878c894e52e107296ffc890e913c611b1a4ec6c44e2821fe2e23"}, + {file = "python_multipart-0.0.32.tar.gz", hash = "sha256:be54b7f3fa167bb83e4fcd936b887b708f4e57fe75911c02aebf53efaf8d938e"}, +] + +[[package]] +name = "pywin32" +version = "312" +description = "Python for Windows Extensions" +optional = true +python-versions = ">=3.9" +groups = ["main"] +markers = "python_version >= \"3.10\" and extra == \"responses\" and sys_platform == \"win32\"" +files = [ + {file = "pywin32-312-cp310-cp310-win32.whl", hash = "sha256:772235332b5d1024c696f11cea1ae4be7930f0a8b894bb43db14e3f435f1ff7e"}, + {file = "pywin32-312-cp310-cp310-win_amd64.whl", hash = "sha256:5dbc35d2b5320dc07f25fa31269cfb767471002b17de5eb067d03da68c7cb2db"}, + {file = "pywin32-312-cp310-cp310-win_arm64.whl", hash = "sha256:3020656e34f1cf7faeb7bccd2b84653a607c6ff0c55ada85e6487d61716deabd"}, + {file = "pywin32-312-cp311-cp311-win32.whl", hash = "sha256:17948aeadbdb091f0ced6ef0841620794e68327b94ee415571c1203594b7215c"}, + {file = "pywin32-312-cp311-cp311-win_amd64.whl", hash = "sha256:d11417d84412f859b722fad0841b3614459ed0047f7542d8362e77884f6b6e8a"}, + {file = "pywin32-312-cp311-cp311-win_arm64.whl", hash = "sha256:b2200a054ca6d6625c4842fc56a4976a4b47f96b73dbe5538c3f813a80359f47"}, + {file = "pywin32-312-cp312-cp312-win32.whl", hash = "sha256:dab4f65ac9c4e48400a2a0530c46c3c579cd5905ecd11b80692373915269208b"}, + {file = "pywin32-312-cp312-cp312-win_amd64.whl", hash = "sha256:b457f6d628a47e8a7346ce22acb7e1a46a4a78b52e1d17e1af56871bd19a93bc"}, + {file = "pywin32-312-cp312-cp312-win_arm64.whl", hash = "sha256:6017c58e12f6809fbb0555b75df144c2922a9ffd18e4b9b5afa863b6c1a9d950"}, + {file = "pywin32-312-cp313-cp313-win32.whl", hash = "sha256:7a27df850933d16a8eabfbaeb73d52b273e2da667f80d70b01a89d1f6828d02c"}, + {file = "pywin32-312-cp313-cp313-win_amd64.whl", hash = "sha256:c53e878d15a1c44788082bfe712a905433473aa38f86375b7cf8b45e3acbaaf9"}, + {file = "pywin32-312-cp313-cp313-win_arm64.whl", hash = "sha256:59aba5d5940842075343a5ddc6b11f1cdf0d1567fe745290359dfbcc7c2eb831"}, + {file = "pywin32-312-cp314-cp314-win32.whl", hash = "sha256:a77a90fbb6881238d2ca9c6fd797b25817f3768fe78d214a90137ff055a75f5b"}, + {file = "pywin32-312-cp314-cp314-win_amd64.whl", hash = "sha256:a4dd3a848290ef724347b19f301045831d8e802fa4464f491b98b1e0a081432e"}, + {file = "pywin32-312-cp314-cp314-win_arm64.whl", hash = "sha256:9fce94568364e0155e6dfb781ac5d95903be8baf28670632beab1b523f300daa"}, + {file = "pywin32-312-cp315-cp315-win32.whl", hash = "sha256:5c1fbe4a937a73ae9297384a3da38518cbc694c68ad8a809b2e19acd350f03ed"}, + {file = "pywin32-312-cp315-cp315-win_amd64.whl", hash = "sha256:c2f03a0f73f804a13c2735b99392b0cd426bb4f2c4d0178e5ac966a0f21618d5"}, + {file = "pywin32-312-cp315-cp315-win_arm64.whl", hash = "sha256:a8597d28f267b39074aef51fa593530082b39cbe5a074226096857b1fed2dfb9"}, + {file = "pywin32-312-cp39-cp39-win32.whl", hash = "sha256:d620900033cc7531e50727c3c8333091df5dd3ffe6d68cdca38c03f5821408d5"}, + {file = "pywin32-312-cp39-cp39-win_amd64.whl", hash = "sha256:dc90147579a905b8635e1b0ec6514967dcb07e6e0d9c42f1477feef14cac23bb"}, + {file = "pywin32-312-cp39-cp39-win_arm64.whl", hash = "sha256:02ebca0f0242b75292e218065004310d6a477407c09fa449bfe4f6022bc0c0fc"}, +] + [[package]] name = "pyyaml" version = "6.0.2" @@ -1922,6 +2263,50 @@ files = [ {file = "sniffio-1.3.1.tar.gz", hash = "sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc"}, ] +[[package]] +name = "sse-starlette" +version = "3.5.1" +description = "SSE plugin for Starlette" +optional = true +python-versions = ">=3.10" +groups = ["main"] +markers = "python_version >= \"3.10\" and extra == \"responses\"" +files = [ + {file = "sse_starlette-3.5.1-py3-none-any.whl", hash = "sha256:a6cdcfec47b952911b1d99a7b8056429b5984039b875bf6575174220364b551f"}, + {file = "sse_starlette-3.5.1.tar.gz", hash = "sha256:dbb1461cd53c1e284537c3ddcf92f54f24178519a3f1e335e49e5283993db0f7"}, +] + +[package.dependencies] +anyio = ">=4.7.0" +starlette = ">=0.49.1" + +[package.extras] +daphne = ["daphne (>=4.2.0)"] +examples = ["fastapi (>=0.115.12)", "pydantic (>=2)", "uvicorn (>=0.34.0)"] +examples-db = ["aiosqlite (>=0.21.0)", "sqlalchemy[asyncio] (>=2.0.41)"] +granian = ["granian (>=2.3.1)"] +uvicorn = ["uvicorn (>=0.34.0)"] + +[[package]] +name = "starlette" +version = "1.7.0" +description = "The little ASGI library that shines." +optional = true +python-versions = ">=3.10" +groups = ["main"] +markers = "python_version >= \"3.10\" and extra == \"responses\"" +files = [ + {file = "starlette-1.7.0-py3-none-any.whl", hash = "sha256:67f8e99895493dd2911a03f11314af6ceebeae4e704bb9f43dfc6a9db151c93e"}, + {file = "starlette-1.7.0.tar.gz", hash = "sha256:c79f74ea63cff761804fbbfb182f1e0b440c2d07b164d24700c5a1bab5d6ff5d"}, +] + +[package.dependencies] +anyio = ">=4.0.0,<5" +typing-extensions = {version = ">=4.10.0", markers = "python_version < \"3.13\""} + +[package.extras] +full = ["httpx (>=0.27.0,<0.29.0)", "httpx2 (>=2.0.0)", "itsdangerous", "jinja2", "opentelemetry-api", "python-multipart (>=0.0.18)", "pyyaml"] + [[package]] name = "tenacity" version = "9.1.2" @@ -2086,6 +2471,27 @@ h2 = ["h2 (>=4,<5)"] socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] zstd = ["zstandard (>=0.18.0)"] +[[package]] +name = "uvicorn" +version = "0.54.0" +description = "The lightning-fast ASGI server." +optional = true +python-versions = ">=3.10" +groups = ["main"] +markers = "python_version >= \"3.10\" and extra == \"responses\" and sys_platform != \"emscripten\"" +files = [ + {file = "uvicorn-0.54.0-py3-none-any.whl", hash = "sha256:505bdb0f318731d45f1f712071fc781a8981f6847a31c902c9f5e652d4f67faf"}, + {file = "uvicorn-0.54.0.tar.gz", hash = "sha256:a2e33cbfaa0306f8e6b0c13e0cb89d7d7a2da3e62b90c66e18c33d9807b28620"}, +] + +[package.dependencies] +click = ">=7.0" +h11 = ">=0.8" +typing-extensions = {version = ">=4.0", markers = "python_version < \"3.11\""} + +[package.extras] +standard = ["httptools (>=0.8.0)", "python-dotenv (>=0.13)", "pyyaml (>=5.1)", "uvloop (>=0.15.1) ; sys_platform != \"win32\" and sys_platform != \"cygwin\" and platform_python_implementation != \"PyPy\"", "watchfiles (>=0.20)", "websockets (>=13.0)"] + [[package]] name = "wrapt" version = "1.17.2" @@ -2314,7 +2720,10 @@ enabler = ["pytest-enabler (>=2.2)"] test = ["big-O", "jaraco.functools", "jaraco.itertools", "jaraco.test", "more_itertools", "pytest (>=6,!=8.1.*)", "pytest-ignore-flaky"] type = ["pytest-mypy"] +[extras] +responses = ["mcp"] + [metadata] lock-version = "2.1" python-versions = ">3.9, <3.14" -content-hash = "1c9c37790807d6f7e6de6462eb0d25a17a72af60d42e48e98523bf900dfb4ff5" +content-hash = "dcb9ecb75a09726f6214bda62a205f4020e88c3710307972d1e0be4d2b402f29" diff --git a/python/frameworks/litellm/pyproject.toml b/python/frameworks/litellm/pyproject.toml index 2d33fc305..9a7f68b06 100644 --- a/python/frameworks/litellm/pyproject.toml +++ b/python/frameworks/litellm/pyproject.toml @@ -33,6 +33,12 @@ litellm = ">=1.43.0" fi-instrumentation-otel = ">=1.2.2" # LiteLLM imports tenacity lazily for its optional retry helpers. tenacity = ">=8.2.3" +# Current LiteLLM Responses imports MCP even for requests without MCP tools. +# Keep this optional so the historical LiteLLM/OpenAI floor remains installable. +mcp = {version = ">=1.0,<3", optional = true, python = ">=3.10"} + +[tool.poetry.extras] +responses = ["mcp"] [tool.poetry.urls] Homepage = "https://futureagi.com" diff --git a/python/frameworks/litellm/tests/test_install_contract.py b/python/frameworks/litellm/tests/test_install_contract.py index 19782dace..67780146f 100644 --- a/python/frameworks/litellm/tests/test_install_contract.py +++ b/python/frameworks/litellm/tests/test_install_contract.py @@ -31,6 +31,20 @@ def test_retry_helper_works_with_runtime_dependencies(): assert response.choices[0].message.content == "hello" +def test_responses_extra_declares_the_upstream_runtime_dependency(): + dependencies = [Requirement(value) for value in requires("traceAI-litellm")] + mcp = next(dep for dep in dependencies if dep.name == "mcp") + assert mcp.marker.evaluate({"extra": "responses"}) + assert not mcp.marker.evaluate({"extra": ""}) + assert Version("2.3.0") in mcp.specifier + if hasattr(litellm, "responses"): + # Latest CI installs the documented extra in a clean environment. This + # assertion and the real HTTP Responses tests prevent runner-only fixes. + assert Version(version("mcp")) in mcp.specifier + readme = (Path(__file__).parents[1] / "README.md").read_text() + assert "pip install 'traceAI-litellm[responses]'" in readme + + def test_readme_separates_provider_and_telemetry_credentials(): readme = (Path(__file__).parents[1] / "README.md").read_text() setup = readme.split("## Quickstart")[0] diff --git a/python/frameworks/litellm/tests/test_qualification_regressions.py b/python/frameworks/litellm/tests/test_qualification_regressions.py new file mode 100644 index 000000000..de67560f0 --- /dev/null +++ b/python/frameworks/litellm/tests/test_qualification_regressions.py @@ -0,0 +1,428 @@ +"""Real SDK/loopback regressions for tool privacy, text endpoints and finish reasons.""" + +import copy +import json +import threading +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from importlib.metadata import version + +import litellm +import pytest +from fi_instrumentation import TraceConfig +from fi_instrumentation.fi_types import SpanAttributes +from opentelemetry import context +from opentelemetry.context import _SUPPRESS_INSTRUMENTATION_KEY +from opentelemetry.trace import StatusCode +from traceai_litellm import LiteLLMInstrumentor + +from test_http_regressions import capture, server # noqa: F401 + +FINISH = SpanAttributes.GEN_AI_RESPONSE_FINISH_REASONS + + +@pytest.mark.skipif( + not hasattr(litellm, "responses"), reason="Responses absent at floor" +) +@pytest.mark.parametrize("asynchronous", [False, True]) +@pytest.mark.parametrize("streaming", [False, True]) +@pytest.mark.asyncio +async def test_mcp_credentials_never_exported(server, capture, asynchronous, streaming): + http, base = server + exporter, _ = capture + tool = { + "type": "mcp", + "server_label": "synthetic-server", + "server_url": "https://synthetic.invalid/mcp?token=synthetic-url-token", + "authorization": "synthetic-mcp-authorization", + "headers": { + "Authorization": "synthetic-header-auth", + "X-Api-Key": "synthetic-header-key", + }, + "require_approval": "never", + } + before = copy.deepcopy(tool) + params = dict( + model="openai/gpt-4o-mini", + input="hello", + tools=[tool], + api_key="synthetic-provider-key", + api_base=base, + stream=streaming, + num_retries=0, + ) + result = ( + await litellm.aresponses(**params) + if asynchronous + else litellm.responses(**params) + ) + if streaming: + chunks = [chunk async for chunk in result] if asynchronous else list(result) + assert chunks[-1].type == "response.completed" + else: + assert result.output[0].content[0].text == "synthetic-output" + assert tool == before + sent = http.requests[-1][1]["tools"][0] + assert sent["authorization"] == before["authorization"] + assert sent["server_url"] == before["server_url"] + assert sent["headers"] == before["headers"] + spans = exporter.get_finished_spans() + assert len(spans) == 1 + serialized = spans[0].to_json() + for credential in ( + before["authorization"], + *before["headers"].values(), + "synthetic-url-token", + ): + assert credential not in serialized + tools = json.loads(spans[0].attributes["gen_ai.request.parameters"])["tools"] + assert tools[0]["type"] == "mcp" + assert tools[0]["server_label"] == "synthetic-server" + + +class CompletionHandler(BaseHTTPRequestHandler): + def log_message(self, *args): + pass + + def do_POST(self): + body = json.loads(self.rfile.read(int(self.headers["Content-Length"]))) + self.server.requests.append((self.path, body)) + if self.server.fail: + self.send_response(400) + self.send_header("Content-Type", "application/json") + self.end_headers() + self.wfile.write( + json.dumps( + { + "error": { + "message": "synthetic-failure", + "type": "invalid_request_error", + } + } + ).encode() + ) + return + text = not self.path.endswith("/chat/completions") + base = { + "id": "cmpl_synthetic", + "object": "text_completion" if text else "chat.completion", + "created": 1, + "model": "gpt-3.5-turbo-instruct" if text else "gpt-4o-mini", + } + choices = self.server.choice_reasons + self.send_response(200) + if body.get("stream"): + self.send_header("Content-Type", "text/event-stream") + self.end_headers() + if not text: + base["object"] = "chat.completion.chunk" + chunks = [] + for part in ("synthetic-", "output"): + chunks.append( + { + **base, + "choices": [ + { + "index": index, + "finish_reason": None, + **( + {"text": part} + if text + else { + "delta": {"role": "assistant", "content": part} + } + ), + } + for index, _ in choices + ], + } + ) + chunks.append( + { + **base, + "choices": [ + { + "index": index, + "finish_reason": reason, + **({"text": ""} if text else {"delta": {}}), + } + for index, reason in choices + ], + } + ) + chunks.append( + { + **base, + "choices": [], + "usage": { + "prompt_tokens": 2, + "completion_tokens": 3, + "total_tokens": 5, + }, + } + ) + for chunk in chunks: + self.wfile.write(("data: " + json.dumps(chunk) + "\n\n").encode()) + self.wfile.write(b"data: [DONE]\n\n") + else: + self.send_header("Content-Type", "application/json") + self.end_headers() + self.wfile.write( + json.dumps( + { + **base, + "choices": [ + { + "index": index, + "finish_reason": reason, + **( + {"text": "synthetic-output", "logprobs": None} + if text + else { + "message": { + "role": "assistant", + "content": "synthetic-output", + } + } + ), + } + for index, reason in choices + ], + "usage": { + "prompt_tokens": 2, + "completion_tokens": 3, + "total_tokens": 5, + }, + } + ).encode() + ) + + +@pytest.fixture +def completion_server(): + http = ThreadingHTTPServer(("127.0.0.1", 0), CompletionHandler) + http.requests = [] + http.choice_reasons = [(0, "stop")] + http.fail = False + thread = threading.Thread(target=http.serve_forever, daemon=True) + thread.start() + yield http, "http://127.0.0.1:%d/v1" % http.server_port + http.shutdown() + http.server_close() + thread.join() + + +async def complete( + base, asynchronous=False, streaming=False, text=True, consume=True, **kwargs +): + params = dict( + api_base=base, + api_key="synthetic-provider-key", + num_retries=0, + stream=streaming, + **kwargs, + ) + if streaming: + params.setdefault("stream_options", {"include_usage": True}) + if text: + params.update( + model="text-completion-openai/gpt-3.5-turbo-instruct", + prompt="synthetic-prompt", + ) + result = ( + await litellm.atext_completion(**params) + if asynchronous + else litellm.text_completion(**params) + ) + else: + params.update( + model="openai/gpt-4o-mini", messages=[{"role": "user", "content": "hello"}] + ) + result = ( + await litellm.acompletion(**params) + if asynchronous + else litellm.completion(**params) + ) + if streaming and consume: + return [chunk async for chunk in result] if asynchronous else list(result) + return result + + +@pytest.mark.parametrize("asynchronous", [False, True]) +@pytest.mark.parametrize("streaming", [False, True]) +@pytest.mark.asyncio +async def test_text_completion_real_sdk( + completion_server, capture, asynchronous, streaming +): + http, base = completion_server + exporter, _ = capture + result = await complete(base, asynchronous, streaming) + assert result + assert http.requests[-1][0].endswith("/completions") + # The floor SDK normalizes the provider prompt to a singleton batch. + expected_prompt = ( + ["synthetic-prompt"] if version("litellm") == "1.43.0" else "synthetic-prompt" + ) + assert http.requests[-1][1]["prompt"] == expected_prompt + spans = exporter.get_finished_spans() + assert len(spans) == 1 + assert spans[0].name == ("atext_completion" if asynchronous else "text_completion") + attrs = spans[0].attributes + assert attrs["input.value"] == "synthetic-prompt" + assert attrs["output.value"] == "synthetic-output" + assert attrs["gen_ai.output.messages.0.message.content"] == "synthetic-output" + assert attrs["gen_ai.output.messages.0.message.role"] == "assistant" + totals = [] + for chunk in result if streaming else [result]: + data = chunk if isinstance(chunk, dict) else chunk.model_dump() + usage = data.get("usage") + if hasattr(usage, "model_dump"): + usage = usage.model_dump() + if usage and usage.get("total_tokens") is not None: + totals.append(usage["total_tokens"]) + if not totals: + # The floor's async text stream exposes no usage chunk at all. + assert version("litellm") == "1.43.0" and streaming and asynchronous + assert "gen_ai.usage.total_tokens" not in attrs + else: + assert attrs["gen_ai.usage.total_tokens"] == totals[-1] + if not streaming or version("litellm") != "1.43.0": + assert totals[-1] == 5 + assert "synthetic-provider-key" not in spans[0].to_json() + + +@pytest.mark.parametrize("asynchronous", [False, True]) +@pytest.mark.parametrize("streaming", [False, True]) +@pytest.mark.parametrize("text", [False, True]) +@pytest.mark.parametrize("reason", ["stop", "length", "content_filter", "tool_calls"]) +@pytest.mark.asyncio +async def test_reported_finish_reasons( + completion_server, capture, asynchronous, streaming, text, reason +): + http, base = completion_server + http.choice_reasons = [(0, reason)] + exporter, _ = capture + await complete(base, asynchronous, streaming, text) + spans = exporter.get_finished_spans() + assert len(spans) == 1 + assert spans[0].attributes.get(FINISH) == (reason,) + assert ( + spans[0].attributes.get("gen_ai.output.messages.0.message.finish_reason") + == reason + ) + + +@pytest.mark.parametrize("asynchronous", [False, True]) +@pytest.mark.parametrize("streaming", [False, True]) +@pytest.mark.parametrize("text", [False, True]) +@pytest.mark.asyncio +async def test_finish_reasons_keep_choice_identity( + completion_server, capture, asynchronous, streaming, text +): + http, base = completion_server + http.choice_reasons = [(1, "length"), (0, "stop")] + exporter, _ = capture + result = await complete(base, asynchronous, streaming, text, n=2) + # LiteLLM itself can renumber non-streamed chat choices and collapse terminal + # stream choices. Uninstrumented HTTP controls reproduce this. Record exactly + # what the SDK exposes, not an invented reconstruction of the wire response. + # Sparse/interleaved identity is independently pinned with controlled chunks. + reported = {} + for chunk in result if streaming else [result]: + data = chunk if isinstance(chunk, dict) else chunk.model_dump() + for choice in data["choices"]: + if choice.get("finish_reason") is not None: + reported[choice["index"]] = choice["finish_reason"] + assert reported + attrs = exporter.get_finished_spans()[0].attributes + assert attrs.get(FINISH) == tuple(reported[index] for index in sorted(reported)) + for index, reason in reported.items(): + assert ( + attrs.get(f"gen_ai.output.messages.{index}.message.finish_reason") == reason + ) + + +@pytest.mark.parametrize("asynchronous", [False, True]) +@pytest.mark.parametrize("text", [False, True]) +@pytest.mark.asyncio +async def test_no_invented_reason_on_early_close( + completion_server, capture, asynchronous, text +): + _, base = completion_server + exporter, _ = capture + stream = await complete(base, asynchronous, True, text, consume=False) + if asynchronous: + await stream.__anext__() + await stream.aclose() + else: + next(stream) + stream.close() + spans = exporter.get_finished_spans() + assert len(spans) == 1 + assert FINISH not in spans[0].attributes + + +@pytest.mark.parametrize("asynchronous", [False, True]) +@pytest.mark.parametrize("streaming", [False, True]) +@pytest.mark.asyncio +async def test_text_error_and_suppression( + completion_server, capture, asynchronous, streaming +): + http, base = completion_server + exporter, _ = capture + token = context.attach(context.set_value(_SUPPRESS_INSTRUMENTATION_KEY, True)) + try: + assert await complete(base, asynchronous, streaming) + finally: + context.detach(token) + assert not exporter.get_finished_spans() + http.fail = True + swallowed_by_floor = ( + version("litellm") == "1.43.0" and streaming and not asynchronous + ) + if swallowed_by_floor: + # 1.43.0 catches a sync text-stream HTTP error inside the SDK and yields + # a None sentinel. Prove this with instrumentation fully removed; do not + # change SDK behavior or fabricate an exception/finish reason in tracing. + inst = LiteLLMInstrumentor() + inst.uninstrument() + control = await complete(base, asynchronous, streaming) + assert control == [None] + assert not exporter.get_finished_spans() + inst.instrument(tracer_provider=capture[1], config=TraceConfig()) + assert await complete(base, asynchronous, streaming) == control + else: + with pytest.raises(Exception): + await complete(base, asynchronous, streaming) + spans = exporter.get_finished_spans() + assert len(spans) == 1 + expected_status = StatusCode.OK if swallowed_by_floor else StatusCode.ERROR + assert spans[0].status.status_code is expected_status + assert FINISH not in spans[0].attributes + + +@pytest.mark.parametrize( + "flag", + [ + "hide_inputs", + "hide_input_text", + "hide_input_messages", + "hide_outputs", + "hide_output_text", + "hide_output_messages", + ], +) +@pytest.mark.parametrize("asynchronous", [False, True]) +@pytest.mark.parametrize("streaming", [False, True]) +@pytest.mark.asyncio +async def test_text_privacy(completion_server, capture, flag, asynchronous, streaming): + _, base = completion_server + exporter, provider = capture + inst = LiteLLMInstrumentor() + inst.uninstrument() + inst.instrument(tracer_provider=provider, config=TraceConfig(**{flag: True})) + await complete(base, asynchronous, streaming) + spans = exporter.get_finished_spans() + assert len(spans) == 1 + assert ("synthetic-prompt" if "input" in flag else "synthetic-output") not in spans[ + 0 + ].to_json() diff --git a/python/frameworks/litellm/tests/test_text_stream_lifecycle.py b/python/frameworks/litellm/tests/test_text_stream_lifecycle.py new file mode 100644 index 000000000..e5ebbacbc --- /dev/null +++ b/python/frameworks/litellm/tests/test_text_stream_lifecycle.py @@ -0,0 +1,151 @@ +"""Text API delegation and partially consumed streams retain shared lifecycle guarantees.""" + +import json + +import litellm +import pytest +from opentelemetry.trace import StatusCode + +from test_stream_extensions import capture, consume, invoke # noqa: F401 + + +@pytest.mark.parametrize("asynchronous", [False, True]) +@pytest.mark.asyncio +async def test_text_stream_failure(capture, asynchronous): + failure = RuntimeError("synthetic-midstream-failure") + chunks = [ + { + "object": "text_completion", + "choices": [{"index": 0, "text": "partial", "finish_reason": None}], + }, + failure, + ] + with capture(chunks=chunks, endpoint="text_completion") as (exporter, _): + stream = await invoke( + asynchronous, endpoint="text_completion", prompt="hello", stream=True + ) + with pytest.raises(RuntimeError) as caught: + await consume(stream, asynchronous) + assert caught.value is failure + spans = exporter.get_finished_spans() + assert len(spans) == 1 + assert spans[0].status.status_code is StatusCode.ERROR + assert spans[0].attributes["output.value"] == "partial" + assert "gen_ai.response.finish_reasons" not in spans[0].attributes + + +@pytest.mark.parametrize("asynchronous", [False, True]) +@pytest.mark.parametrize("streaming", [False, True]) +@pytest.mark.asyncio +async def test_sparse_interleaved_finish_reasons(capture, asynchronous, streaming): + chunks = [ + { + "choices": [ + {"index": 3, "delta": {"content": "third"}, "finish_reason": None} + ] + }, + { + "choices": [ + {"index": 1, "delta": {"content": "first"}, "finish_reason": "length"} + ] + }, + {"choices": [{"index": 3, "delta": {}, "finish_reason": "stop"}]}, + {"choices": [{"index": 1, "delta": {}, "finish_reason": None}]}, + {"choices": [], "usage": {"total_tokens": 7}}, + ] + result = { + "choices": [ + {"index": 3, "text": "third", "finish_reason": "stop"}, + {"index": 1, "text": "first", "finish_reason": "length"}, + ] + } + with capture(chunks=chunks if streaming else None, result=result) as (exporter, _): + response = await invoke(asynchronous, stream=streaming) + if streaming: + await consume(response, asynchronous) + attrs = exporter.get_finished_spans()[0].attributes + assert attrs["gen_ai.response.finish_reasons"] == ("length", "stop") + assert attrs["gen_ai.output.messages.1.message.finish_reason"] == "length" + assert attrs["gen_ai.output.messages.3.message.finish_reason"] == "stop" + assert "gen_ai.output.messages.0.message.finish_reason" not in attrs + + +@pytest.mark.parametrize("asynchronous", [False, True]) +@pytest.mark.asyncio +async def test_function_schema_is_not_treated_as_connection_credentials( + capture, asynchronous +): + function = { + "name": "authorize", + "parameters": { + "type": "object", + "properties": { + "authorization": {"type": "string"}, + "headers": {"type": "object"}, + }, + }, + } + tools = [{"type": "function", "function": function}] + with capture(result={"choices": []}) as (exporter, _): + await invoke(asynchronous, tools=tools) + params = json.loads( + exporter.get_finished_spans()[0].attributes["gen_ai.request.parameters"] + ) + assert params["tools"] == tools + + +@pytest.mark.parametrize("asynchronous", [False, True]) +@pytest.mark.asyncio +async def test_unknown_builtin_connection_fields_are_not_exported( + capture, asynchronous +): + tools = [ + { + "type": "future_builtin", + "headers": {"x-provider-token": "synthetic-secret"}, + "connection": {"authorization": "synthetic-secret"}, + } + ] + with capture(result={"choices": []}) as (exporter, _): + await invoke(asynchronous, tools=tools) + span = exporter.get_finished_spans()[0] + assert "synthetic-secret" not in span.to_json() + params = json.loads(span.attributes["gen_ai.request.parameters"]) + assert params["tools"] == [{"type": "future_builtin"}] + assert tools[0]["headers"]["x-provider-token"] == "synthetic-secret" + + +@pytest.mark.parametrize("asynchronous", [False, True]) +@pytest.mark.asyncio +async def test_text_delegation_emits_one_span(capture, monkeypatch, asynchronous): + result = { + "object": "text_completion", + "choices": [{"index": 0, "text": "hello", "finish_reason": "stop"}], + } + + def delegated(prompt, model=None, **kwargs): + return litellm.completion( + model=model, messages=[{"role": "user", "content": prompt}] + ) + + async def adelegated(*args, **kwargs): + return litellm.text_completion(*args, **kwargs) + + monkeypatch.setattr(litellm, "text_completion", delegated) + monkeypatch.setattr(litellm, "atext_completion", adelegated) + original = litellm.text_completion + with capture(result=result) as (exporter, _): + if asynchronous: + response = await litellm.atext_completion( + "positional prompt", model="openai/gpt-4o-mini" + ) + else: + response = litellm.text_completion( + "positional prompt", model="openai/gpt-4o-mini" + ) + assert response is result + spans = exporter.get_finished_spans() + assert len(spans) == 1 + assert spans[0].name == ("atext_completion" if asynchronous else "text_completion") + assert spans[0].attributes["input.value"] == "positional prompt" + assert litellm.text_completion is original diff --git a/python/frameworks/litellm/traceai_litellm/__init__.py b/python/frameworks/litellm/traceai_litellm/__init__.py index 30d037cca..c5ecad860 100644 --- a/python/frameworks/litellm/traceai_litellm/__init__.py +++ b/python/frameworks/litellm/traceai_litellm/__init__.py @@ -192,6 +192,46 @@ def _get_attributes_from_image( yield f"{ImageAttributes.IMAGE_URL}", url +def _telemetry_tool(tool): + """Project tool definitions, never connection configuration, into telemetry. + + MCP authorization/headers and server URLs (which can embed credentials) must + reach the provider unchanged but must not reach an exporter. Unknown built-in + tools are type-only until their configuration has a safe projection. Function + parameter schemas are application data, not connection headers. + """ + tool = as_dict(tool) + kind = tool.get("type") + fields = {"type"} + if kind == "function": + fields.update(("name", "description", "parameters", "strict")) + elif kind == "mcp": + fields.update( + ("server_label", "server_description", "allowed_tools", "require_approval") + ) + result = {key: value for key, value in tool.items() if key in fields} + if kind == "function" and "function" in tool: + result["function"] = { + key: value for key, value in as_dict(tool["function"]).items() + if key in ("name", "description", "parameters", "strict") + } + return result + + +def _telemetry_parameters(parameters): + parameters = dict(parameters) + if parameters.get("tools") is not None: + tools = parameters["tools"] + # Do not consume a caller-owned iterator while preparing telemetry. + parameters["tools"] = ( + [_telemetry_tool(tool) for tool in tools] + if isinstance(tools, (list, tuple)) else [] + ) + if isinstance(parameters.get("tool_choice"), Mapping): + parameters["tool_choice"] = _telemetry_tool(parameters["tool_choice"]) + return parameters + + def _instrument_func_type_completion( span: trace_api.Span, kwargs: Dict[str, Any], @@ -242,9 +282,9 @@ def _instrument_func_type_completion( span, f"{SpanAttributes.GEN_AI_INPUT_MESSAGES}.{index}.{key}", value ) - invocation_params = { + invocation_params = _telemetry_parameters({ k: v for k, v in kwargs.items() if k not in ["model", "messages"] - } + }) _set_span_attribute( span, SpanAttributes.GEN_AI_REQUEST_PARAMETERS, @@ -264,6 +304,26 @@ def _message_attributes(message, config=None, *, output=True): yield key, value +def _instrument_func_type_text_completion(span, kwargs, config=None): + # Prompt is content, not a generation parameter: never leave an unmasked + # duplicate in request.parameters. Normalize strings/batches to messages. + request = {key: value for key, value in kwargs.items() if key != "prompt"} + prompt = kwargs.get("prompt") + prompts = ( + prompt + if isinstance(prompt, list) and all(isinstance(p, str) for p in prompt) + else [prompt] + ) + request["messages"] = [ + { + "role": "user", + "content": value if isinstance(value, str) else safe_json_dumps(value), + } + for value in prompts if value is not None + ] + _instrument_func_type_completion(span, request, config) + + def _instrument_func_type_responses(span, kwargs, config=None): messages = response_messages(kwargs.get("input")) if kwargs.get("instructions"): @@ -400,6 +460,36 @@ def _instrument_func_type_image_generation( _set_span_attribute(span, SpanAttributes.INPUT_VALUE, str(prompt)) +def _finish_reasons(choices): + reasons = {} + for position, choice in enumerate(choices or []): + choice = as_dict(choice) + index = choice.get("index") + if index is None: + index = position + reason = choice.get("finish_reason") + if isinstance(index, int) and isinstance(reason, str) and reason: + reasons[index] = reason + return reasons + + +def _set_finish_reasons(span, reasons): + if not reasons: + return + _set_span_attribute( + span, SpanAttributes.GEN_AI_RESPONSE_FINISH_REASONS, + [reasons[index] for index in sorted(reasons)], + ) + # The standard array is in choice-index order. Indexed attributes also retain + # identity for sparse/interleaved choices, without inventing missing reasons. + for index, reason in reasons.items(): + _set_span_attribute( + span, + f"{SpanAttributes.GEN_AI_OUTPUT_MESSAGES}.{index}.message.finish_reason", + reason, + ) + + def _finalize_span(span: trace_api.Span, result: Any, config: Optional[TraceConfig] = None) -> None: # Embedding vectors and private message content must not reappear in the # independent raw response aggregate after core filters semantic attributes. @@ -465,8 +555,21 @@ def _finalize_span(span: trace_api.Span, result: Any, config: Optional[TraceConf ) if not hide_message_output: _set_span_attribute(span, SpanAttributes.OUTPUT_VALUE, url) + elif as_dict(result).get("object") == "text_completion": + choices = as_dict(result).get("choices") or [] + for position, choice in enumerate(choices): + choice = as_dict(choice) + index = choice.get("index") + if index is None: + index = position + message = {"role": "assistant", "content": choice.get("text")} + if len(choices) == 1 and not hide_message_output: + _set_span_attribute(span, SpanAttributes.OUTPUT_VALUE, choice.get("text")) + for key, value in _message_attributes(message, config): + _set_span_attribute(span, f"{SpanAttributes.GEN_AI_OUTPUT_MESSAGES}.{index}.{key}", value) elif as_dict(result).get("object") == "response": _set_response_messages(span, response_messages(as_dict(result).get("output")), config) + _set_finish_reasons(span, _finish_reasons(as_dict(result).get("choices"))) _set_result_metadata(span, result) @@ -501,6 +604,8 @@ def _instrument(self, **kwargs: Any) -> None: functions_to_instrument = { "completion": self._completion_wrapper, "acompletion": self._acompletion_wrapper, + "text_completion": self._text_completion_wrapper, + "atext_completion": self._atext_completion_wrapper, "completion_with_retries": self._completion_with_retries_wrapper, # Bug report filed on GitHub for acompletion_with_retries: https://github.com/BerriAI/litellm/issues/4908 # "acompletion_with_retries": self._acompletion_with_retries_wrapper, @@ -562,6 +667,7 @@ def _request(self, name, args, kwargs): # Async endpoints and retry helpers may expose only *args/**kwargs. endpoint = { "acompletion": "completion", + "atext_completion": "text_completion", "completion_with_retries": "completion", "acompletion_with_retries": "completion", "aembedding": "embedding", @@ -581,7 +687,9 @@ def _request(self, name, args, kwargs): request = dict(kwargs) # Capture generation inputs/settings, not arbitrary SDK control objects, # credentials, HTTP headers, callbacks, or provider-specific secrets. - return {key: value for key, value in request.items() if key in _REQUEST_FIELDS} + return _telemetry_parameters({ + key: value for key, value in request.items() if key in _REQUEST_FIELDS + }) def _call(self, name, annotate, args, kwargs): original = self.original_litellm_funcs[name] @@ -677,6 +785,16 @@ async def _aimage_generation_wrapper(self, *args: Any, **kwargs: Any) -> Any: "aimage_generation", _instrument_func_type_image_generation, args, kwargs ) + def _text_completion_wrapper(self, *args: Any, **kwargs: Any) -> Any: + return self._call( + "text_completion", _instrument_func_type_text_completion, args, kwargs + ) + + async def _atext_completion_wrapper(self, *args: Any, **kwargs: Any) -> Any: + return await self._acall( + "atext_completion", _instrument_func_type_text_completion, args, kwargs + ) + def _responses_wrapper(self, *args: Any, **kwargs: Any) -> Any: return self._call("responses", _instrument_func_type_responses, args, kwargs) @@ -760,6 +878,7 @@ def __init__(self, span, span_cm=None, config=None): self.config = config self.processed_chunks = [] self.content_by_choice = {} + self.finish_reasons_by_choice = {} self.has_tool_context = False self.tools_by_choice = {} self.functions_by_choice = {} @@ -775,9 +894,12 @@ def _process(self, chunk): _set_result_metadata(self.span, response) return choices = chunk.get("choices") or [] + self.finish_reasons_by_choice.update(_finish_reasons(choices)) for position, choice in enumerate(choices): index = choice.get("index", position) delta = choice.get("delta") or {} + if isinstance(choice.get("text"), str): + delta = {"role": "assistant", "content": choice["text"]} self.has_tool_context |= _has_tool_context(delta) tools = self.tools_by_choice.setdefault(index, {}) for tool_position, tool in enumerate(delta.get("tool_calls") or []): @@ -832,6 +954,7 @@ def _finish(self, error=None, abandoned=False): for key, value in _message_attributes({"function_call": function}, self.config): _set_span_attribute(self.span, f"{SpanAttributes.GEN_AI_OUTPUT_MESSAGES}.{index}.{key}", value) _set_response_messages(self.span, self.responses.messages(), self.config) + _set_finish_reasons(self.span, self.finish_reasons_by_choice) if error is not None: _record_error(self.span, error) elif self.responses.failed: diff --git a/python/tests/test_framework_litellm.py b/python/tests/test_framework_litellm.py index 78ea3e419..ca3b0e905 100644 --- a/python/tests/test_framework_litellm.py +++ b/python/tests/test_framework_litellm.py @@ -133,7 +133,10 @@ def test_instrument_basic(self, tracer_provider, config, mock_litellm): instrumentor = LiteLLMInstrumentor() instrumentor.original_litellm_funcs.clear() expected = set(mock_litellm) - {"acompletion_with_retries"} - expected.update(name for name in ("responses", "aresponses") if hasattr(litellm, name)) + expected.update( + name for name in ("responses", "aresponses", "text_completion", "atext_completion") + if hasattr(litellm, name) + ) originals = {name: getattr(litellm, name) for name in expected} try: instrumentor._instrument(tracer_provider=tracer_provider, config=config) From 662940f73a6138bd7a49c556eeead7d7bafbca1a Mon Sep 17 00:00:00 2001 From: Jacob Cavazos Date: Thu, 5 Nov 2026 18:55:47 -0600 Subject: [PATCH 11/11] fix(litellm): emit input.value as typed ChatInput for per-item masking MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Source core now fail-closes input.value to __REDACTED__ under hide_input_images when the value is an opaque string, so the sanitized conversation JSON was being dropped entirely. Emit the filtered messages through the structured ChatInput API instead (same as fireworks), which lets core mask per content item: images and tool-call payloads are redacted while text, tool history, and non-text blocks survive. Also teach core _content the input_audio part kind — speech blobs redact their data under hide_input_text but are preserved under hide_input_images, matching the test contract that non-image content survives image hiding. Unknown part types still fail closed. Generated with Devin Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../instrumentation/_chat_input.py | 18 +++++++++ .../litellm/tests/test_privacy_and_choices.py | 26 ++++++++++--- .../litellm/tests/test_provider_contract.py | 2 +- .../tests/test_qualification_regressions.py | 2 +- .../tests/test_text_stream_lifecycle.py | 3 +- .../litellm/traceai_litellm/__init__.py | 37 +++++++++++++------ python/tests/test_framework_litellm.py | 12 +++++- 7 files changed, 78 insertions(+), 22 deletions(-) diff --git a/python/fi_instrumentation/instrumentation/_chat_input.py b/python/fi_instrumentation/instrumentation/_chat_input.py index 24f65c2f5..94e60b915 100644 --- a/python/fi_instrumentation/instrumentation/_chat_input.py +++ b/python/fi_instrumentation/instrumentation/_chat_input.py @@ -98,6 +98,24 @@ def _content(value: Any, config: "TraceConfig") -> Any: if detail is not None: reference["detail"] = detail result.append({"type": kind, "image_url": reference}) + elif kind == "input_audio": + audio = part.get("input_audio") + if not _object(audio): + raise ValueError("invalid audio content") + data, fmt = audio.get("data"), audio.get("format") + if type(data) is not str or type(fmt) is not str: + raise ValueError("invalid audio content") + # Speech carries the same semantic content as text; the binary blob + # is masked when input text is private, not under the image flag. + result.append( + { + "type": kind, + "input_audio": { + "data": _REDACTED if config.hide_input_text else data, + "format": fmt, + }, + } + ) else: # Never trust a caller-supplied modality label we do not understand. result.append(_REDACTED) diff --git a/python/frameworks/litellm/tests/test_privacy_and_choices.py b/python/frameworks/litellm/tests/test_privacy_and_choices.py index 9cb9e9e14..1cd520546 100644 --- a/python/frameworks/litellm/tests/test_privacy_and_choices.py +++ b/python/frameworks/litellm/tests/test_privacy_and_choices.py @@ -115,8 +115,17 @@ async def test_hide_input_images_with_tool_history( assert marker not in actual["input.value"] assert (marker in json.dumps(actual)) is not hide if with_tool: - conversation = json.loads(actual["input.value"]) - assert conversation[1:] == messages[1:] + conversation = json.loads(actual["input.value"])["messages"] + # Content-bearing keys (tool call arguments can embed image blobs) are + # redacted by core whenever images or text are hidden. + expected = copy.deepcopy(messages[1:]) + if hide: + for message in expected: + for key in ("name", "tool_call_id", "refusal", "tool_calls", + "function_call", "audio"): + if key in message: + message[key] = "__REDACTED__" + assert conversation[1:] == expected assert "{{gen_ai.input.images.0}}" in json.dumps(conversation[0]) if hide: assert "gen_ai.input.images" not in actual @@ -242,7 +251,7 @@ def test_base64_image_size_limit(with_tool, limit, image_kind): if image_kind == "oversized": assert marker not in json.dumps(actual) if with_tool: - assert json.loads(actual["input.value"])[1:] == messages[1:] + assert json.loads(actual["input.value"])["messages"][1:] == messages[1:] @pytest.mark.parametrize("streaming", [False, True]) @@ -420,8 +429,15 @@ def test_image_sanitization_preserves_other_tool_conversation_content(): api_key="synthetic-key", ) actual = attrs(exporter) - conversation = json.loads(actual["input.value"]) + conversation = json.loads(actual["input.value"])["messages"] assert audio in conversation[0]["content"] - assert conversation[1:] == original[1:] + # Content-bearing keys are redacted by core while images are hidden. + expected = copy.deepcopy(original[1:]) + for message in expected: + for key in ("name", "tool_call_id", "refusal", "tool_calls", + "function_call", "audio"): + if key in message: + message[key] = "__REDACTED__" + assert conversation[1:] == expected assert marker not in json.dumps(actual) assert messages == original diff --git a/python/frameworks/litellm/tests/test_provider_contract.py b/python/frameworks/litellm/tests/test_provider_contract.py index 3341477b4..f0152acd9 100644 --- a/python/frameworks/litellm/tests/test_provider_contract.py +++ b/python/frameworks/litellm/tests/test_provider_contract.py @@ -252,7 +252,7 @@ def test_tool_calls_preserved(capture, monkeypatch): assert litellm.completion(model="gpt-4o-mini", messages=messages) is response assert messages == original_messages attrs = attributes(exporter) - assert json.loads(attrs[SpanAttributes.INPUT_VALUE]) == messages + assert json.loads(attrs[SpanAttributes.INPUT_VALUE])["messages"] == messages output = json.loads(attrs[SpanAttributes.OUTPUT_VALUE])["choices"][0]["message"] assert output["content"] == "Calling a tool" assert output["tool_calls"][0]["function"]["name"] == "lookup_out" diff --git a/python/frameworks/litellm/tests/test_qualification_regressions.py b/python/frameworks/litellm/tests/test_qualification_regressions.py index de67560f0..47d122a72 100644 --- a/python/frameworks/litellm/tests/test_qualification_regressions.py +++ b/python/frameworks/litellm/tests/test_qualification_regressions.py @@ -267,7 +267,7 @@ async def test_text_completion_real_sdk( assert len(spans) == 1 assert spans[0].name == ("atext_completion" if asynchronous else "text_completion") attrs = spans[0].attributes - assert attrs["input.value"] == "synthetic-prompt" + assert json.loads(attrs["input.value"])["messages"][0]["content"] == "synthetic-prompt" assert attrs["output.value"] == "synthetic-output" assert attrs["gen_ai.output.messages.0.message.content"] == "synthetic-output" assert attrs["gen_ai.output.messages.0.message.role"] == "assistant" diff --git a/python/frameworks/litellm/tests/test_text_stream_lifecycle.py b/python/frameworks/litellm/tests/test_text_stream_lifecycle.py index e5ebbacbc..e7cc41078 100644 --- a/python/frameworks/litellm/tests/test_text_stream_lifecycle.py +++ b/python/frameworks/litellm/tests/test_text_stream_lifecycle.py @@ -147,5 +147,6 @@ async def adelegated(*args, **kwargs): spans = exporter.get_finished_spans() assert len(spans) == 1 assert spans[0].name == ("atext_completion" if asynchronous else "text_completion") - assert spans[0].attributes["input.value"] == "positional prompt" + input_messages = json.loads(spans[0].attributes["input.value"])["messages"] + assert input_messages[0]["content"] == "positional prompt" assert litellm.text_completion is original diff --git a/python/frameworks/litellm/traceai_litellm/__init__.py b/python/frameworks/litellm/traceai_litellm/__init__.py index c5ecad860..e0fd4be8c 100644 --- a/python/frameworks/litellm/traceai_litellm/__init__.py +++ b/python/frameworks/litellm/traceai_litellm/__init__.py @@ -34,6 +34,12 @@ get_attributes_from_context, safe_json_dumps, ) +try: + from fi_instrumentation.instrumentation.config import ChatInput +except ImportError: + # The declared published 1.2.2 floor predates the structured-input API and + # the opaque-input guard. Retain its existing path, not a validation fallback. + ChatInput = None from fi_instrumentation.fi_types import ( EmbeddingAttributes, FiSpanKindValues, @@ -257,24 +263,31 @@ def _instrument_func_type_completion( # aggregate. Omit it when either message-content privacy flag applies. hide_message_input = bool(config and (config.hide_input_text or config.hide_input_messages)) if not hide_message_input and (filtered_messages := process_messages.get("filtered_messages")): + # Emit the sanitized conversation as a typed ChatInput so core masks + # it per item instead of fail-closing the whole opaque aggregate. _set_span_attribute( - span, SpanAttributes.INPUT_VALUE, json.dumps(filtered_messages) + span, + SpanAttributes.INPUT_VALUE, + ChatInput({"model": kwargs.get("model"), "messages": filtered_messages}) + if ChatInput is not None + else json.dumps(filtered_messages), ) if input_images := process_messages.get("input_images"): _set_span_attribute( span, SpanAttributes.INPUT_IMAGES, json.dumps(input_images) ) - if not hide_message_input and any( - _has_tool_context(message) or _has_non_text_content(message) - for message in messages - ): - # Preserve tool linkage and non-text blocks (such as input_audio) - # without copying image bytes into input.value. - _set_span_attribute( - span, SpanAttributes.INPUT_VALUE, safe_json_dumps(filtered_messages) - ) - elif not hide_message_input and (eval_input := process_messages.get("eval_input")): - _set_span_attribute(span, SpanAttributes.INPUT_VALUE, eval_input) + # Opaque aggregate fallbacks for cores without the structured ChatInput + # API; the typed path above already carries the sanitized conversation. + if not hide_message_input and ChatInput is None: + if any( + _has_tool_context(message) or _has_non_text_content(message) + for message in messages + ): + _set_span_attribute( + span, SpanAttributes.INPUT_VALUE, safe_json_dumps(filtered_messages) + ) + elif eval_input := process_messages.get("eval_input"): + _set_span_attribute(span, SpanAttributes.INPUT_VALUE, eval_input) for index, input_message in list(enumerate(messages)): for key, value in _message_attributes(input_message, config, output=False): diff --git a/python/tests/test_framework_litellm.py b/python/tests/test_framework_litellm.py index ca3b0e905..d48e8a8de 100644 --- a/python/tests/test_framework_litellm.py +++ b/python/tests/test_framework_litellm.py @@ -908,7 +908,15 @@ def _final_input_value(self, mock_span): if c.args[0] == SpanAttributes.INPUT_VALUE ] assert values, "INPUT_VALUE was never set" - return values[-1] + final = values[-1] + # The span receives a typed ChatInput; resolve it the way core's mask does. + try: + from fi_instrumentation.instrumentation.config import ChatInput, mask_chat_input + except ImportError: + ChatInput = None + if ChatInput is not None and type(final) is ChatInput: + final = mask_chat_input(final, TraceConfig()) + return final def test_completion_keeps_all_message_texts(self): mock_span = MagicMock() @@ -950,7 +958,7 @@ def test_completion_multimodal_text_and_image_placeholder(self): }, ) attrs = {c.args[0]: c.args[1] for c in mock_span.set_attribute.call_args_list} - final = attrs[SpanAttributes.INPUT_VALUE] + final = self._final_input_value(mock_span) assert "describe this" in final assert "in detail" in final assert f"{{{SpanAttributes.INPUT_IMAGES}.0}}" in final