From 31ed5e57744e722c05483b47f34bb80a3963e00f Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Sun, 1 Nov 2026 21:50:34 -0800 Subject: [PATCH 1/6] fix(vllm): protect private content and preserve request lifecycles Exclude transport credentials, mask opaque copies under granular privacy flags, preserve legacy stream cleanup and iterable messages, and finish cancelled legacy/shared requests. Leave shared explicit close to existing PR #411. Co-Authored-By: Claude Code --- .github/workflows/ollama-provider-tests.yml | 12 +- .../instrumentation/config.py | 22 +- .../openai/traceai_openai/_request.py | 4 +- .../openai/traceai_openai/_stream.py | 4 +- .../openai/traceai_openai/_with_span.py | 2 +- .../vllm/tests/test_legacy_lifecycle.py | 363 +++++++++++++++ .../vllm/tests/test_recall_privacy.py | 436 ++++++++++++++++++ .../frameworks/vllm/traceai_vllm/_wrappers.py | 395 +++++++++++----- python/tests/test_granular_privacy_copies.py | 70 +++ 9 files changed, 1189 insertions(+), 119 deletions(-) create mode 100644 python/frameworks/vllm/tests/test_legacy_lifecycle.py create mode 100644 python/frameworks/vllm/tests/test_recall_privacy.py create mode 100644 python/tests/test_granular_privacy_copies.py diff --git a/.github/workflows/ollama-provider-tests.yml b/.github/workflows/ollama-provider-tests.yml index 348445463..7049e3ca3 100644 --- a/.github/workflows/ollama-provider-tests.yml +++ b/.github/workflows/ollama-provider-tests.yml @@ -7,6 +7,8 @@ on: - 'python/frameworks/openai/**' - 'python/frameworks/vllm/**' - 'python/fi_instrumentation/**' + - 'python/tests/test_granular_privacy_copies.py' + - 'python/tests/test_structured_content_privacy.py' - 'python/pyproject.toml' - '.github/workflows/ollama-provider-tests.yml' push: @@ -15,6 +17,8 @@ on: - 'python/frameworks/openai/**' - 'python/frameworks/vllm/**' - 'python/fi_instrumentation/**' + - 'python/tests/test_granular_privacy_copies.py' + - 'python/tests/test_structured_content_privacy.py' - 'python/pyproject.toml' - '.github/workflows/ollama-provider-tests.yml' workflow_dispatch: @@ -30,7 +34,7 @@ jobs: strategy: fail-fast: false matrix: - openai: ['1.69.0', '3.42.0'] + openai: ['1.69.0', '3.42.1'] steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: @@ -57,6 +61,9 @@ jobs: --allow-hosts=127.0.0.1 --allow-unix-socket "$1" -m pytest "$2/python/frameworks/vllm/tests" \ -q -o addopts="" -p no:cacheprovider --disable-socket --allow-unix-socket + "$1" -m pytest "$2/python/tests/test_granular_privacy_copies.py" \ + "$2/python/tests/test_structured_content_privacy.py" \ + -q -o addopts="" -p no:cacheprovider --disable-socket --allow-unix-socket ' _ "$(command -v python)" "$PWD" legacy: @@ -81,4 +88,7 @@ jobs: ip link set lo up "$1" -m pytest "$2/python/frameworks/vllm/tests" \ -q -o addopts="" -p no:cacheprovider --disable-socket --allow-unix-socket + "$1" -m pytest "$2/python/tests/test_granular_privacy_copies.py" \ + "$2/python/tests/test_structured_content_privacy.py" \ + -q -o addopts="" -p no:cacheprovider --disable-socket --allow-unix-socket ' _ "$(command -v python)" "$PWD" diff --git a/python/fi_instrumentation/instrumentation/config.py b/python/fi_instrumentation/instrumentation/config.py index 1df7ea1f7..09bbdec33 100644 --- a/python/fi_instrumentation/instrumentation/config.py +++ b/python/fi_instrumentation/instrumentation/config.py @@ -368,6 +368,24 @@ def mask( and key == SpanAttributes.GEN_AI_REQUEST_PARAMETERS ): return None + # Opaque aggregate copies cannot be filtered by message-part type. Hide + # the whole copy when any contained input/output category is private; + # typed message attributes below still preserve the other categories. + elif ( + self.hide_input_messages or self.hide_input_text or self.hide_input_images + ) and _has_content_prefix( + key, + ( + SpanAttributes.INPUT_VALUE, + SpanAttributes.RAW_INPUT, + SpanAttributes.GEN_AI_REQUEST_PARAMETERS, + ), + ): + value = REDACTED_VALUE + elif ( + self.hide_output_messages or self.hide_output_text + ) and _has_content_prefix(key, (SpanAttributes.OUTPUT_VALUE, SpanAttributes.RAW_OUTPUT)): + value = REDACTED_VALUE elif self.hide_inputs and _has_content_prefix(key, (SpanAttributes.INPUT_VALUE,)): value = REDACTED_VALUE elif self.hide_inputs and key == SpanAttributes.INPUT_MIME_TYPE: @@ -432,7 +450,9 @@ def mask( ) and _is_embedding_text_key(key): # Embedding input text is input content for all three flags. value = REDACTED_VALUE - elif self.hide_input_images and _has_content_prefix(key, (SpanAttributes.INPUT_IMAGES,)): + elif (self.hide_input_images or self.hide_input_messages) and _has_content_prefix( + key, (SpanAttributes.INPUT_IMAGES,) + ): return None elif ( self.hide_input_images diff --git a/python/frameworks/openai/traceai_openai/_request.py b/python/frameworks/openai/traceai_openai/_request.py index c6e8cf426..ddfecf72a 100644 --- a/python/frameworks/openai/traceai_openai/_request.py +++ b/python/frameworks/openai/traceai_openai/_request.py @@ -340,7 +340,7 @@ def __call__( cast_to=cast_to, request_parameters=request_parameters, ) - except Exception as e: + except BaseException as e: status = trace_api.Status( status_code=trace_api.StatusCode.ERROR, description=f"{type(e).__name__}: {e}", @@ -406,7 +406,7 @@ async def __call__( add_io_to_span_attributes( with_span, None, response, is_streaming=self._is_streaming(response) ) - except Exception as exception: + except BaseException as exception: with_span.record_exception(exception) status = trace_api.Status( status_code=trace_api.StatusCode.ERROR, diff --git a/python/frameworks/openai/traceai_openai/_stream.py b/python/frameworks/openai/traceai_openai/_stream.py index a3a60042e..5975a7671 100644 --- a/python/frameworks/openai/traceai_openai/_stream.py +++ b/python/frameworks/openai/traceai_openai/_stream.py @@ -85,7 +85,7 @@ def __next__(self) -> Any: status = trace_api.Status(status_code=trace_api.StatusCode.OK) self._finish_tracing(status=status) raise - except Exception as exception: + except BaseException as exception: if not self._self_is_finished: status = trace_api.Status( status_code=trace_api.StatusCode.ERROR, @@ -175,7 +175,7 @@ async def __anext__(self): status = trace_api.Status(status_code=trace_api.StatusCode.OK) self._finish_tracing(status=status) raise - except Exception as exception: + except BaseException as exception: if not self._self_is_finished: status = trace_api.Status( status_code=trace_api.StatusCode.ERROR, diff --git a/python/frameworks/openai/traceai_openai/_with_span.py b/python/frameworks/openai/traceai_openai/_with_span.py index 5f5419ad0..923a31af8 100644 --- a/python/frameworks/openai/traceai_openai/_with_span.py +++ b/python/frameworks/openai/traceai_openai/_with_span.py @@ -49,7 +49,7 @@ def __init__( def is_finished(self) -> bool: return self._is_finished - def record_exception(self, exception: Exception) -> None: + def record_exception(self, exception: BaseException) -> None: if self._is_finished: return try: diff --git a/python/frameworks/vllm/tests/test_legacy_lifecycle.py b/python/frameworks/vllm/tests/test_legacy_lifecycle.py new file mode 100644 index 000000000..f988628bd --- /dev/null +++ b/python/frameworks/vllm/tests/test_legacy_lifecycle.py @@ -0,0 +1,363 @@ +"""Legacy span lifecycle and privacy with the real OpenAI SDK and no network.""" + +import asyncio +import json + +import openai +import pytest +from fi_instrumentation import TraceConfig +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +from opentelemetry.trace import StatusCode +from traceai_vllm import VLLMInstrumentor + +if int(openai.__version__.split(".")[0]) >= 3: + import httpx2 as httpx +else: + import httpx + +OPTIONS = { + "base_url": "http://localhost:8000/v1", + "api_key": "synthetic-not-a-real-key", + "max_retries": 0, +} +PARAMS = { + "model": "test-vllm-model", + "messages": [{"role": "user", "content": "Hello"}], +} +COMPLETION = { + "id": "c1", + "object": "chat.completion", + "created": 1, + "model": "test-vllm-model", + "choices": [ + { + "index": 0, + "finish_reason": "stop", + "message": {"role": "assistant", "content": "Synthetic output"}, + } + ], +} + + +def _chunk(content): + return { + "id": "c1", + "object": "chat.completion.chunk", + "created": 1, + "model": "test-vllm-model", + "choices": [ + { + "index": 0, + "delta": {"role": "assistant", "content": content}, + "finish_reason": None, + } + ], + } + + +def _sse(parts): + body = "".join(f"data: {json.dumps(_chunk(part))}\n\n" for part in parts) + return body + "data: [DONE]\n\n" + + +def _stream_response(request): + return httpx.Response( + 200, + headers={"content-type": "text/event-stream"}, + content=_sse(["part0", "part1", "part2"]), + ) + + +class _Abort(BaseException): + """A non-Exception interruption, like KeyboardInterrupt or SystemExit.""" + + +@pytest.fixture +def exporter(): + provider = TracerProvider() + exporter = InMemorySpanExporter() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + instrumentor = VLLMInstrumentor() + with pytest.warns(DeprecationWarning): + instrumentor.instrument(tracer_provider=provider, config=TraceConfig()) + try: + yield exporter + finally: + instrumentor.uninstrument() + provider.shutdown() + + +def _sync_client(handler): + return openai.OpenAI( + **OPTIONS, + http_client=httpx.Client( + transport=httpx.MockTransport(handler), trust_env=False + ), + ) + + +def _async_client(handler): + return openai.AsyncOpenAI( + **OPTIONS, + http_client=httpx.AsyncClient( + transport=httpx.MockTransport(handler), trust_env=False + ), + ) + + +def _only_span(exporter): + spans = exporter.get_finished_spans() + assert len(spans) == 1, [(s.name, s.status.status_code) for s in spans] + return spans[0] + + +def _assert_error(span, exception_type): + assert span.status.status_code == StatusCode.ERROR + assert span.attributes["error.type"].split(".")[-1] == exception_type + recorded = [ + e.attributes["exception.type"] for e in span.events if e.name == "exception" + ] + assert [name.split(".")[-1] for name in recorded] == [exception_type] + + +def _assert_streamed_output(span): + assert span.status.status_code == StatusCode.OK + assert ( + span.attributes["gen_ai.output.messages.0.message.content"] == "part0part1part2" + ) + + +@pytest.mark.parametrize("use_async", [False, True]) +def test_transport_options_are_not_exported(exporter, use_async): + seen = {} + + def handle(request): + seen["authorization"] = request.headers.get("authorization") + seen["query"] = request.url.params.get("key") + seen["body"] = json.loads(request.content) + return httpx.Response(200, json=COMPLETION) + + request = { + **PARAMS, + "extra_headers": {"Authorization": "Bearer SYNTHETIC-HEADER-TOKEN"}, + "extra_query": {"key": "SYNTHETIC-QUERY-TOKEN"}, + "extra_body": {"top_k": 5}, + "timeout": 5.0, + } + + async def run_async(): + async with _async_client(handle) as client: + await client.chat.completions.create(**request) + + if use_async: + asyncio.run(run_async()) + else: + with _sync_client(handle) as client: + client.chat.completions.create(**request) + + # The SDK still sends every option; only the exported span omits them. + assert seen["authorization"] == "Bearer SYNTHETIC-HEADER-TOKEN" + assert seen["query"] == "SYNTHETIC-QUERY-TOKEN" + assert seen["body"]["top_k"] == 5 + span = _only_span(exporter) + exported = json.dumps(dict(span.attributes)) + assert "SYNTHETIC-HEADER-TOKEN" not in exported + assert "SYNTHETIC-QUERY-TOKEN" not in exported + recorded = json.loads(span.attributes["input.value"]) + assert set(recorded) == {"model", "messages", "extra_body"} + # Request-body parameters such as vLLM sampling options remain visible. + assert recorded["extra_body"] == {"top_k": 5} + + +def test_async_call_cancelled_by_timeout_is_exported_as_error(exporter): + async def slow(request): + await asyncio.sleep(5) + return httpx.Response(200, json=COMPLETION) + + async def run(): + async with _async_client(slow) as client: + with pytest.raises((asyncio.TimeoutError, TimeoutError)): + await asyncio.wait_for(client.chat.completions.create(**PARAMS), 0.2) + + asyncio.run(run()) + _assert_error(_only_span(exporter), "CancelledError") + + +def test_sync_call_interrupted_is_exported_as_error(exporter): + def interrupt(request): + raise _Abort("synthetic interrupt") + + with _sync_client(interrupt) as client: + with pytest.raises(_Abort): + client.chat.completions.create(**PARAMS) + _assert_error(_only_span(exporter), "_Abort") + + +def test_async_stream_cancelled_midway_is_exported_as_error(exporter): + class SlowStream(httpx.AsyncByteStream): + async def __aiter__(self): + yield f"data: {json.dumps(_chunk('first'))}\n\n".encode() + await asyncio.sleep(5) + yield b"data: [DONE]\n\n" + + async def slow_stream(request): + return httpx.Response( + 200, headers={"content-type": "text/event-stream"}, stream=SlowStream() + ) + + async def run(): + async with _async_client(slow_stream) as client: + stream = await client.chat.completions.create(**PARAMS, stream=True) + + async def consume(): + async for _ in stream: + pass + + with pytest.raises((asyncio.TimeoutError, TimeoutError)): + await asyncio.wait_for(consume(), 0.3) + + asyncio.run(run()) + _assert_error(_only_span(exporter), "CancelledError") + + +def test_sync_stream_failure_midway_is_exported_as_error(exporter): + class BrokenStream(httpx.SyncByteStream): + def __iter__(self): + yield f"data: {json.dumps(_chunk('first'))}\n\n".encode() + raise httpx.ReadError("synthetic disconnect") + + def broken(request): + return httpx.Response( + 200, headers={"content-type": "text/event-stream"}, stream=BrokenStream() + ) + + with _sync_client(broken) as client: + stream = client.chat.completions.create(**PARAMS, stream=True) + # Newer SDKs wrap transport read errors in APIConnectionError. + with pytest.raises((httpx.ReadError, openai.APIConnectionError)) as raised: + for _ in stream: + pass + _assert_error(_only_span(exporter), type(raised.value).__name__) + + +def test_sync_stream_keeps_sdk_stream_api(exporter): + with _sync_client(_stream_response) as client: + stream = client.chat.completions.create(**PARAMS, stream=True) + assert isinstance(stream, openai.Stream) + assert stream.response.status_code == 200 + assert exporter.get_finished_spans() == () + assert [c.choices[0].delta.content for c in stream] == [ + "part0", + "part1", + "part2", + ] + _assert_streamed_output(_only_span(exporter)) + + +@pytest.mark.skipif( + not hasattr(openai.Stream, "__enter__"), + reason="this SDK's Stream is not a context manager", +) +def test_sync_stream_context_manager(exporter): + with _sync_client(_stream_response) as client: + with client.chat.completions.create(**PARAMS, stream=True) as stream: + assert isinstance(stream, openai.Stream) + assert len(list(stream)) == 3 + _assert_streamed_output(_only_span(exporter)) + + +@pytest.mark.skipif( + not hasattr(openai.Stream, "__enter__"), + reason="this SDK's Stream is not a context manager", +) +def test_sync_stream_context_manager_exited_early(exporter): + with _sync_client(_stream_response) as client: + with client.chat.completions.create(**PARAMS, stream=True) as stream: + next(stream) + span = _only_span(exporter) + assert span.status.status_code == StatusCode.OK + assert span.attributes["gen_ai.output.messages.0.message.content"] == "part0" + + +def test_sync_stream_closed_early(exporter): + with _sync_client(_stream_response) as client: + stream = client.chat.completions.create(**PARAMS, stream=True) + for _ in stream: + break + stream.close() + span = _only_span(exporter) + assert span.status.status_code == StatusCode.OK + assert span.attributes["gen_ai.output.messages.0.message.content"] == "part0" + + +def test_async_stream_keeps_sdk_stream_api(exporter): + async def run(): + async with _async_client(_stream_response) as client: + stream = await client.chat.completions.create(**PARAMS, stream=True) + assert isinstance(stream, openai.AsyncStream) + assert stream.response.status_code == 200 + assert [c.choices[0].delta.content async for c in stream] == [ + "part0", + "part1", + "part2", + ] + + asyncio.run(run()) + _assert_streamed_output(_only_span(exporter)) + + +@pytest.mark.skipif( + not hasattr(openai.AsyncStream, "__aenter__"), + reason="this SDK's AsyncStream is not a context manager", +) +def test_async_stream_context_manager(exporter): + async def run(): + async with _async_client(_stream_response) as client: + async with await client.chat.completions.create( + **PARAMS, stream=True + ) as stream: + assert isinstance(stream, openai.AsyncStream) + assert len([c async for c in stream]) == 3 + + asyncio.run(run()) + _assert_streamed_output(_only_span(exporter)) + + +def test_async_stream_closed_early(exporter): + async def run(): + async with _async_client(_stream_response) as client: + stream = await client.chat.completions.create(**PARAMS, stream=True) + async for _ in stream: + break + await stream.close() + + asyncio.run(run()) + span = _only_span(exporter) + assert span.status.status_code == StatusCode.OK + assert span.attributes["gen_ai.output.messages.0.message.content"] == "part0" + + +def test_raw_response_stream_is_returned_unchanged(exporter): + with _sync_client(_stream_response) as client: + raw = client.chat.completions.with_raw_response.create(**PARAMS, stream=True) + assert raw.http_response.status_code == 200 + assert len(list(raw.parse())) == 3 + span = _only_span(exporter) + assert span.status.status_code == StatusCode.OK + assert "synthetic-not-a-real-key" not in json.dumps(dict(span.attributes)) + + +def test_raw_response_does_not_export_client_internals(exporter): + def handle(request): + return httpx.Response(200, json=COMPLETION) + + with _sync_client(handle) as client: + raw = client.chat.completions.with_raw_response.create( + **PARAMS, extra_headers={"Authorization": "Bearer SYNTHETIC-HEADER-TOKEN"} + ) + assert raw.parse().choices[0].message.content == "Synthetic output" + exported = json.dumps(dict(_only_span(exporter).attributes)) + assert "SYNTHETIC-HEADER-TOKEN" not in exported + assert "synthetic-not-a-real-key" not in exported diff --git a/python/frameworks/vllm/tests/test_recall_privacy.py b/python/frameworks/vllm/tests/test_recall_privacy.py new file mode 100644 index 000000000..0b1b867ed --- /dev/null +++ b/python/frameworks/vllm/tests/test_recall_privacy.py @@ -0,0 +1,436 @@ +"""Retained and shared vLLM routes: privacy and cancellation at SDK boundaries.""" + +import asyncio +import importlib.util +import json +from contextlib import contextmanager + +import openai +import pytest +from fi_instrumentation import TraceConfig +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +from opentelemetry.trace import StatusCode +from traceai_vllm import VLLMInstrumentor + +if int(openai.__version__.split(".")[0]) >= 3: + import httpx2 as httpx +else: + import httpx + +SDK_VERSION = tuple(int(part) for part in openai.__version__.split(".")[:2]) +ROUTES = ["legacy"] + ( + ["shared"] + if SDK_VERSION >= (1, 69) and importlib.util.find_spec("traceai_openai") is not None + else [] +) +INPUT = "SYNTHETIC-PRIVATE-PROMPT" +OUTPUT = "SYNTHETIC-PRIVATE-ANSWER" +IMAGE = "https://example.invalid/SYNTHETIC-PRIVATE-IMAGE.png" +OPTIONS = { + "base_url": "http://localhost:8000/v1", + "api_key": "synthetic-key", + "max_retries": 0, +} +PARAMS = {"model": "test-vllm-model", "messages": [{"role": "user", "content": INPUT}]} +COMPLETION = { + "id": "c1", + "object": "chat.completion", + "created": 1, + "model": "test-vllm-model", + "choices": [ + { + "index": 0, + "finish_reason": "stop", + "message": {"role": "assistant", "content": OUTPUT}, + } + ], + "usage": {"prompt_tokens": 2, "completion_tokens": 3, "total_tokens": 5}, +} + + +def chunk(content): + return { + "id": "c1", + "object": "chat.completion.chunk", + "created": 1, + "model": "test-vllm-model", + "choices": [ + { + "index": 0, + "delta": {"role": "assistant", "content": content}, + "finish_reason": None, + } + ], + } + + +@contextmanager +def instrumented(route, config): + provider = TracerProvider() + exporter = InMemorySpanExporter() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + if route == "legacy": + instrumentor = VLLMInstrumentor() + with pytest.warns(DeprecationWarning): + instrumentor.instrument(tracer_provider=provider, config=config) + else: + from traceai_openai import OpenAIInstrumentor + + instrumentor = OpenAIInstrumentor() + instrumentor.instrument(tracer_provider=provider, config=config) + try: + yield exporter + finally: + instrumentor.uninstrument() + provider.shutdown() + + +def client_for(route, handler, use_async=False): + if use_async: + client = openai.AsyncOpenAI( + **OPTIONS, + http_client=httpx.AsyncClient( + transport=httpx.MockTransport(handler), trust_env=False + ), + ) + else: + client = openai.OpenAI( + **OPTIONS, + http_client=httpx.Client( + transport=httpx.MockTransport(handler), trust_env=False + ), + ) + if route == "shared": + from traceai_openai import set_client_provider + + set_client_provider(client, provider="vllm") + return client + + +def only_span(exporter): + spans = exporter.get_finished_spans() + assert len(spans) == 1 + return spans[0] + + +def serialized(span): + return json.dumps( + { + "attributes": dict(span.attributes), + "events": [dict(e.attributes) for e in span.events], + "status": span.status.description, + } + ) + + +@pytest.mark.parametrize("route", ROUTES) +@pytest.mark.parametrize("use_async", [False, True]) +@pytest.mark.parametrize("streaming", [False, True]) +@pytest.mark.parametrize( + "flag,sentinel", + [ + ("hide_input_text", INPUT), + ("hide_input_messages", INPUT), + ("hide_output_text", OUTPUT), + ("hide_output_messages", OUTPUT), + ("hide_input_images", IMAGE), + ], +) +def test_granular_privacy_covers_every_copy( + route, use_async, streaming, flag, sentinel, monkeypatch +): + # Exercise the documented environment flags, not just direct config construction. + monkeypatch.setenv("FI_" + flag.upper(), "true") + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": INPUT}, + {"type": "image_url", "image_url": {"url": IMAGE}}, + ], + } + ] + seen = [] + + def handler(request): + seen.append(json.loads(request.content)) + if streaming: + body = f"data: {json.dumps(chunk(OUTPUT))}\n\ndata: [DONE]\n\n" + return httpx.Response( + 200, headers={"content-type": "text/event-stream"}, content=body + ) + return httpx.Response(200, json=COMPLETION) + + async def run_async(): + async with client_for(route, handler, True) as client: + response = await client.chat.completions.create( + **{**PARAMS, "messages": messages}, stream=streaming + ) + if streaming: + assert [c.choices[0].delta.content async for c in response] == [OUTPUT] + else: + assert response.choices[0].message.content == OUTPUT + + with instrumented(route, TraceConfig()) as exporter: + if use_async: + asyncio.run(run_async()) + else: + with client_for(route, handler) as client: + response = client.chat.completions.create( + **{**PARAMS, "messages": messages}, stream=streaming + ) + if streaming: + assert [c.choices[0].delta.content for c in response] == [OUTPUT] + else: + assert response.choices[0].message.content == OUTPUT + span = only_span(exporter) + wire = serialized(span) + assert sentinel not in wire + assert span.attributes["gen_ai.provider.name"] == "vllm" + assert span.status.status_code == StatusCode.OK + if flag.startswith("hide_input"): + assert OUTPUT in wire + else: + assert INPUT in wire + if flag == "hide_input_images": + assert INPUT in wire # typed text stays useful when only images are hidden + if flag == "hide_input_text": + assert IMAGE in wire # typed images are independent of text capture + if flag == "hide_input_messages": + assert IMAGE not in wire # separate image copies are messages too + assert seen[0]["messages"] == messages # masking must not modify the request + + +@pytest.mark.parametrize("route", ROUTES) +@pytest.mark.parametrize("streaming", [False, True]) +@pytest.mark.parametrize("hide_error_details", [False, True]) +def test_task_cancellation_finishes_once_and_preserves_error( + route, streaming, hide_error_details +): + async def run(exporter): + entered = asyncio.Event() + + class SlowStream(httpx.AsyncByteStream): + async def __aiter__(self): + yield f"data: {json.dumps(chunk(OUTPUT))}\n\n".encode() + entered.set() + await asyncio.Event().wait() + + async def handler(request): + if streaming: + return httpx.Response( + 200, + headers={"content-type": "text/event-stream"}, + stream=SlowStream(), + ) + entered.set() + await asyncio.Event().wait() + + async with client_for(route, handler, True) as client: + stream = None + + cancellation_args = [] + + async def request(): + nonlocal stream + try: + response = await client.chat.completions.create( + **PARAMS, stream=streaming + ) + if streaming: + stream = response + async for _ in stream: + pass + except asyncio.CancelledError as error: + cancellation_args.append(error.args) + raise + + task = asyncio.create_task(request()) + await asyncio.wait_for(entered.wait(), 2) + task.cancel("SYNTHETIC-CANCELLATION-DETAIL") + with pytest.raises(asyncio.CancelledError): + await task + # Python 3.10 may replace the error when awaiting a cancelled task; + # assert preservation at the actual application call boundary. + assert cancellation_args == [("SYNTHETIC-CANCELLATION-DETAIL",)] + if stream is not None: + await stream.response.aclose() + span = only_span(exporter) + assert span.status.status_code == StatusCode.ERROR + assert span.attributes["error.type"].endswith("CancelledError") + events = [e for e in span.events if e.name == "exception"] + assert len(events) == 1 + if hide_error_details: + assert "SYNTHETIC-CANCELLATION-DETAIL" not in serialized(span) + else: + assert "SYNTHETIC-CANCELLATION-DETAIL" in serialized(span) + + with instrumented( + route, TraceConfig(hide_error_details=hide_error_details) + ) as exporter: + asyncio.run(run(exporter)) + + +@pytest.mark.skipif( + SDK_VERSION < (1, 69), reason="SDK 1.0 declares List, not Iterable messages" +) +@pytest.mark.parametrize("use_async", [False, True]) +@pytest.mark.parametrize("iterable_container", [False, True]) +def test_legacy_generator_messages_reach_transport(use_async, iterable_container): + seen = [] + yielded = [] + + def messages(): + for message in PARAMS["messages"]: + yielded.append(message) + yield message + + def handler(request): + seen.append(json.loads(request.content)["messages"]) + return httpx.Response(200, json=COMPLETION) + + iterator = messages() + + class SingleUseIterable: + def __iter__(self): + return iterator + + request = { + **PARAMS, + "messages": SingleUseIterable() if iterable_container else iterator, + } + + async def run(): + async with client_for("legacy", handler, True) as client: + await client.chat.completions.create(**request) + + with instrumented("legacy", TraceConfig()) as exporter: + if use_async: + asyncio.run(run()) + else: + with client_for("legacy", handler) as client: + client.chat.completions.create(**request) + assert seen == [PARAMS["messages"]] + assert yielded == PARAMS["messages"] + assert INPUT in serialized(only_span(exporter)) + + +@pytest.mark.parametrize("use_async", [False, True]) +@pytest.mark.parametrize("consume_first", [False, True]) +def test_legacy_explicit_close_releases_response_and_finishes_once( + use_async, consume_first +): + closed = [] + + class SyncBody(httpx.SyncByteStream): + def __iter__(self): + yield f"data: {json.dumps(chunk(OUTPUT))}\n\n".encode() + yield b"data: [DONE]\n\n" + + def close(self): + closed.append(True) + + class AsyncBody(httpx.AsyncByteStream): + async def __aiter__(self): + yield f"data: {json.dumps(chunk(OUTPUT))}\n\n".encode() + yield b"data: [DONE]\n\n" + + async def aclose(self): + closed.append(True) + + def handler(request): + return httpx.Response( + 200, + headers={"content-type": "text/event-stream"}, + stream=AsyncBody() if use_async else SyncBody(), + ) + + async def run(): + async with client_for("legacy", handler, True) as client: + stream = await client.chat.completions.create(**PARAMS, stream=True) + if consume_first: + await stream.__anext__() + await stream.close() + await stream.close() + assert stream.response.is_closed + + with instrumented("legacy", TraceConfig()) as exporter: + if use_async: + asyncio.run(run()) + else: + with client_for("legacy", handler) as client: + stream = client.chat.completions.create(**PARAMS, stream=True) + if consume_first: + next(stream) + stream.close() + stream.close() + assert stream.response.is_closed + assert closed == [True] + assert only_span(exporter).status.status_code == StatusCode.OK + + +@pytest.mark.skipif( + not hasattr(openai.Stream, "__enter__"), + reason="SDK 1.0 streams do not implement context management", +) +@pytest.mark.parametrize("use_async", [False, True]) +@pytest.mark.parametrize("consume_first", [False, True]) +@pytest.mark.parametrize("exception_type", [None, RuntimeError, asyncio.CancelledError]) +def test_legacy_context_exit_finishes_once(use_async, consume_first, exception_type): + def handler(request): + body = f"data: {json.dumps(chunk(OUTPUT))}\n\ndata: [DONE]\n\n".encode() + return httpx.Response( + 200, + headers={"content-type": "text/event-stream"}, + stream=httpx.ByteStream(body), + ) + + def exit_body(): + if exception_type is not None: + raise exception_type("SYNTHETIC-CONSUMER-ERROR") + + async def run_async(): + async with client_for("legacy", handler, True) as client: + stream = await client.chat.completions.create(**PARAMS, stream=True) + try: + async with stream as entered: + assert entered is stream + if consume_first: + await stream.__anext__() + exit_body() + finally: + assert stream.response.is_closed + await stream.close() + + def run_sync(): + with client_for("legacy", handler) as client: + stream = client.chat.completions.create(**PARAMS, stream=True) + try: + with stream as entered: + assert entered is stream + if consume_first: + next(stream) + exit_body() + finally: + assert stream.response.is_closed + stream.close() + + with instrumented("legacy", TraceConfig(hide_error_details=True)) as exporter: + + def run(): + if use_async: + asyncio.run(run_async()) + else: + run_sync() + + if exception_type is not None: + with pytest.raises(exception_type, match="SYNTHETIC-CONSUMER-ERROR"): + run() + else: + run() + span = only_span(exporter) + expected = StatusCode.ERROR if exception_type is not None else StatusCode.OK + assert span.status.status_code == expected + assert "SYNTHETIC-CONSUMER-ERROR" not in serialized(span) diff --git a/python/frameworks/vllm/traceai_vllm/_wrappers.py b/python/frameworks/vllm/traceai_vllm/_wrappers.py index 7fe061aee..1f10743b8 100644 --- a/python/frameworks/vllm/traceai_vllm/_wrappers.py +++ b/python/frameworks/vllm/traceai_vllm/_wrappers.py @@ -3,18 +3,30 @@ import logging import sys from abc import ABC +from collections.abc import Iterable from contextlib import contextmanager -from typing import Any, Dict, Iterator, List, Tuple +from typing import Any, Dict, Iterator, List, Optional, Tuple from fi_instrumentation import safe_json_dumps -from fi_instrumentation.fi_types import FiSpanKindValues, SpanAttributes, MessageAttributes +from fi_instrumentation.fi_types import ( + FiSpanKindValues, + ImageAttributes, + MessageAttributes, + MessageContentAttributes, + SpanAttributes, +) from opentelemetry import trace as trace_api from opentelemetry.trace import INVALID_SPAN, Span, Status, StatusCode from opentelemetry.util.types import AttributeValue +from wrapt import ObjectProxy logger = logging.getLogger(__name__) logger.addHandler(logging.NullHandler()) +# SDK transport options: request headers and query strings can carry +# credentials, and timeouts are client settings rather than request content. +_TRANSPORT_OPTIONS = frozenset({"extra_headers", "extra_query", "timeout"}) + def _shared_instrumentor_enabled() -> bool: """Defer to the shared request-level instrumentor while it is enabled. @@ -110,26 +122,53 @@ def _get_input_attributes( invocation_params["use_beam_search"] = kwargs["use_beam_search"] if invocation_params: - attributes[SpanAttributes.GEN_AI_REQUEST_PARAMETERS] = safe_json_dumps(invocation_params) + attributes[SpanAttributes.GEN_AI_REQUEST_PARAMETERS] = safe_json_dumps( + invocation_params + ) - # Add input messages + # Preserve typed message parts: stringifying a multimodal list would put + # image URLs in message.content, bypassing the image-specific hide policy. for i, message in enumerate(messages): prefix = f"{SpanAttributes.GEN_AI_INPUT_MESSAGES}.{i}" - if isinstance(message, dict): - if "role" in message: - attributes[f"{prefix}.{MessageAttributes.MESSAGE_ROLE}"] = message["role"] - if "content" in message: - attributes[f"{prefix}.{MessageAttributes.MESSAGE_CONTENT}"] = str(message["content"]) if message["content"] else "" - else: - # Handle message objects - if hasattr(message, "role"): - attributes[f"{prefix}.{MessageAttributes.MESSAGE_ROLE}"] = message.role - if hasattr(message, "content"): - attributes[f"{prefix}.{MessageAttributes.MESSAGE_CONTENT}"] = str(message.content) if message.content else "" + role = ( + message.get("role") + if isinstance(message, dict) + else getattr(message, "role", None) + ) + content = ( + message.get("content") + if isinstance(message, dict) + else getattr(message, "content", None) + ) + if role is not None: + attributes[f"{prefix}.{MessageAttributes.MESSAGE_ROLE}"] = role + if isinstance(content, str): + attributes[f"{prefix}.{MessageAttributes.MESSAGE_CONTENT}"] = content + elif isinstance(content, (list, tuple)): + for j, part in enumerate(content): + if not isinstance(part, dict): + continue + part_prefix = f"{prefix}.{MessageAttributes.MESSAGE_CONTENTS}.{j}" + if part.get("type") == "text" and isinstance(part.get("text"), str): + attributes[ + f"{part_prefix}.{MessageContentAttributes.MESSAGE_CONTENT_TEXT}" + ] = part["text"] + elif part.get("type") == "image_url": + image = part.get("image_url") + if isinstance(image, dict) and isinstance(image.get("url"), str): + attributes[ + f"{part_prefix}.{MessageContentAttributes.MESSAGE_CONTENT_IMAGE}.{ImageAttributes.IMAGE_URL}" + ] = image["url"] # Add raw input raw_input = {"model": model, "messages": messages} - raw_input.update({k: v for k, v in kwargs.items() if v is not None}) + raw_input.update( + { + k: v + for k, v in kwargs.items() + if v is not None and k not in _TRANSPORT_OPTIONS + } + ) attributes[SpanAttributes.INPUT_VALUE] = safe_json_dumps(raw_input) return attributes @@ -153,7 +192,9 @@ def _get_output_attributes(response: Any) -> Dict[str, AttributeValue]: if hasattr(message, "role"): attributes[f"{prefix}.{MessageAttributes.MESSAGE_ROLE}"] = message.role if hasattr(message, "content") and message.content: - attributes[f"{prefix}.{MessageAttributes.MESSAGE_CONTENT}"] = message.content + attributes[f"{prefix}.{MessageAttributes.MESSAGE_CONTENT}"] = ( + message.content + ) attributes[SpanAttributes.OUTPUT_VALUE] = message.content # Get usage metrics @@ -162,16 +203,19 @@ def _get_output_attributes(response: Any) -> Dict[str, AttributeValue]: if hasattr(usage, "prompt_tokens"): attributes[SpanAttributes.GEN_AI_USAGE_INPUT_TOKENS] = usage.prompt_tokens if hasattr(usage, "completion_tokens"): - attributes[SpanAttributes.GEN_AI_USAGE_OUTPUT_TOKENS] = usage.completion_tokens + attributes[SpanAttributes.GEN_AI_USAGE_OUTPUT_TOKENS] = ( + usage.completion_tokens + ) if hasattr(usage, "total_tokens"): attributes[SpanAttributes.GEN_AI_USAGE_TOTAL_TOKENS] = usage.total_tokens - # Add raw output + # Add raw output. Only serialize response models: other objects, such as + # with_raw_response wrappers, hold the client and its request headers. try: if hasattr(response, "model_dump"): - attributes[SpanAttributes.OUTPUT_VALUE] = safe_json_dumps(response.model_dump()) - elif hasattr(response, "__dict__"): - attributes[SpanAttributes.OUTPUT_VALUE] = safe_json_dumps(response.__dict__) + attributes[SpanAttributes.OUTPUT_VALUE] = safe_json_dumps( + response.model_dump() + ) except Exception: pass @@ -198,23 +242,194 @@ def _get_stream_output_attributes( if hasattr(last_chunk, "usage") and last_chunk.usage: usage = last_chunk.usage if hasattr(usage, "prompt_tokens"): - attributes[SpanAttributes.GEN_AI_USAGE_INPUT_TOKENS] = usage.prompt_tokens + attributes[SpanAttributes.GEN_AI_USAGE_INPUT_TOKENS] = ( + usage.prompt_tokens + ) if hasattr(usage, "completion_tokens"): - attributes[SpanAttributes.GEN_AI_USAGE_OUTPUT_TOKENS] = usage.completion_tokens + attributes[SpanAttributes.GEN_AI_USAGE_OUTPUT_TOKENS] = ( + usage.completion_tokens + ) if hasattr(usage, "total_tokens"): - attributes[SpanAttributes.GEN_AI_USAGE_TOTAL_TOKENS] = usage.total_tokens + attributes[SpanAttributes.GEN_AI_USAGE_TOTAL_TOKENS] = ( + usage.total_tokens + ) # Add raw output try: - attributes[SpanAttributes.OUTPUT_VALUE] = safe_json_dumps([ - c.model_dump() if hasattr(c, "model_dump") else str(c) for c in chunks - ]) + attributes[SpanAttributes.OUTPUT_VALUE] = safe_json_dumps( + [c.model_dump() if hasattr(c, "model_dump") else str(c) for c in chunks] + ) except Exception: pass return attributes +def _error_status(error: BaseException) -> Status: + return Status(StatusCode.ERROR, f"{type(error).__name__}: {error}") + + +def _end_with_error(span: Span, error: BaseException) -> None: + span.record_exception(error) + span.set_status(_error_status(error)) + span.end() + + +class _StreamSpan: + """Accumulate chunks and end the span exactly once.""" + + __slots__ = ("span", "chunks", "content", "finished") + + def __init__(self, span: Span) -> None: + self.span = span + self.chunks: List[Any] = [] + self.content = "" + self.finished = False + + def process(self, chunk: Any) -> None: + try: + self.chunks.append(chunk) + if hasattr(chunk, "choices") and chunk.choices: + choice = chunk.choices[0] + if hasattr(choice, "delta") and hasattr(choice.delta, "content"): + if choice.delta.content: + self.content += choice.delta.content + except Exception: + logger.exception("Failed to accumulate vLLM stream chunk") + + def finish(self, error: Optional[BaseException] = None) -> None: + if self.finished: + return + self.finished = True + try: + self.span.set_attributes( + _get_stream_output_attributes(self.chunks, self.content) + ) + except Exception: + logger.exception("Failed to set vLLM stream attributes") + if error is None: + self.span.set_status(Status(StatusCode.OK)) + self.span.end() + else: + _end_with_error(self.span, error) + + +class _Stream(ObjectProxy): # type: ignore[misc] + """Proxy the SDK Stream so its API stays intact while chunks are traced.""" + + __slots__ = ("_self_tracking",) + + def __init__(self, stream: Any, span: Span) -> None: + super().__init__(stream) + self._self_tracking = _StreamSpan(span) + + def __iter__(self) -> "_Stream": + return self + + def __next__(self) -> Any: + try: + chunk = self.__wrapped__.__next__() + except StopIteration: + self._self_tracking.finish() + raise + except BaseException as error: + self._self_tracking.finish(error) + raise + self._self_tracking.process(chunk) + return chunk + + def __enter__(self) -> "_Stream": + # ObjectProxy would return the SDK stream and bypass this iterator. + self.__wrapped__.__enter__() + return self + + def __exit__(self, exc_type: Any, exc: Any, tb: Any) -> Any: + try: + return self.__wrapped__.__exit__(exc_type, exc, tb) + except BaseException as close_error: + exc = close_error + raise + finally: + self._self_tracking.finish(exc) + + def close(self) -> None: + try: + close = getattr(self.__wrapped__, "close", None) + if close is not None: + close() + else: + # OpenAI 1.0 streams expose the response but not close(). + self.__wrapped__.response.close() + except BaseException as error: + self._self_tracking.finish(error) + raise + self._self_tracking.finish() + + +class _AsyncStream(ObjectProxy): # type: ignore[misc] + """Proxy the SDK AsyncStream so its API stays intact while chunks are traced.""" + + __slots__ = ("_self_tracking",) + + def __init__(self, stream: Any, span: Span) -> None: + super().__init__(stream) + self._self_tracking = _StreamSpan(span) + + def __aiter__(self) -> "_AsyncStream": + return self + + async def __anext__(self) -> Any: + try: + chunk = await self.__wrapped__.__anext__() + except StopAsyncIteration: + self._self_tracking.finish() + raise + except BaseException as error: + self._self_tracking.finish(error) + raise + self._self_tracking.process(chunk) + return chunk + + async def __aenter__(self) -> "_AsyncStream": + await self.__wrapped__.__aenter__() + return self + + async def __aexit__(self, exc_type: Any, exc: Any, tb: Any) -> Any: + try: + return await self.__wrapped__.__aexit__(exc_type, exc, tb) + except BaseException as close_error: + exc = close_error + raise + finally: + self._self_tracking.finish(exc) + + async def close(self) -> None: + try: + close = getattr(self.__wrapped__, "close", None) + if close is not None: + await close() + else: + await self.__wrapped__.response.aclose() + except BaseException as error: + self._self_tracking.finish(error) + raise + self._self_tracking.finish() + + +def _finish_response(span: Span, response: Any) -> None: + """End the span for a non-stream result. + + A with_raw_response stream also lands here: it is returned unchanged so the + SDK raw-response API keeps working, and its chunks are not recorded. + """ + try: + span.set_attributes(_get_output_attributes(response)) + except Exception: + logger.exception("Failed to set vLLM response attributes") + span.set_status(Status(StatusCode.OK)) + span.end() + + class _CompletionsWrapper(_WithTracer): """Wrapper for synchronous chat completions.""" @@ -232,58 +447,36 @@ def __call__( # Extract parameters model = kwargs.get("model", args[0] if args else "unknown") messages = kwargs.get("messages", args[1] if len(args) > 1 else []) + if isinstance(messages, Iterable) and not isinstance( + messages, (list, tuple, str, bytes) + ): + # Trace and send the same materialized input, never exhaust the + # caller's single-use iterable solely for instrumentation. + messages = list(messages) + if "messages" in kwargs: + kwargs = {**kwargs, "messages": messages} + else: + args = (args[0], messages, *args[2:]) stream = kwargs.get("stream", False) # Get input attributes - exclude model, messages, stream from kwargs - extra_kwargs = {k: v for k, v in kwargs.items() if k not in ("model", "messages", "stream")} + extra_kwargs = { + k: v for k, v in kwargs.items() if k not in ("model", "messages", "stream") + } input_attrs = _get_input_attributes(model, messages, **extra_kwargs) with self._start_span("vLLM Chat Completions", input_attrs) as span: try: response = wrapped(*args, **kwargs) - - if stream: - return self._wrap_stream(response, span) - - # Non-streaming response - output_attrs = _get_output_attributes(response) - span.set_attributes(output_attrs) - span.set_status(Status(StatusCode.OK)) - span.end() - return response - - except Exception as e: - span.record_exception(e) - span.set_status(Status(StatusCode.ERROR, str(e))) - span.end() + except BaseException as e: + # Includes KeyboardInterrupt and other non-Exception interrupts. + _end_with_error(span, e) raise - def _wrap_stream(self, stream: Any, span: Span) -> Any: - """Wrap a streaming response to capture all chunks.""" - chunks = [] - full_content = "" - - try: - for chunk in stream: - chunks.append(chunk) - if hasattr(chunk, "choices") and chunk.choices: - choice = chunk.choices[0] - if hasattr(choice, "delta") and hasattr(choice.delta, "content"): - if choice.delta.content: - full_content += choice.delta.content - yield chunk - - output_attrs = _get_stream_output_attributes(chunks, full_content) - span.set_attributes(output_attrs) - span.set_status(Status(StatusCode.OK)) - - except Exception as e: - span.record_exception(e) - span.set_status(Status(StatusCode.ERROR, str(e))) - raise - - finally: - span.end() + if stream and hasattr(response, "__next__"): + return _Stream(response, span) + _finish_response(span, response) + return response class _AsyncCompletionsWrapper(_WithTracer): @@ -303,55 +496,33 @@ async def __call__( # Extract parameters model = kwargs.get("model", args[0] if args else "unknown") messages = kwargs.get("messages", args[1] if len(args) > 1 else []) + if isinstance(messages, Iterable) and not isinstance( + messages, (list, tuple, str, bytes) + ): + # Trace and send the same materialized input, never exhaust the + # caller's single-use iterable solely for instrumentation. + messages = list(messages) + if "messages" in kwargs: + kwargs = {**kwargs, "messages": messages} + else: + args = (args[0], messages, *args[2:]) stream = kwargs.get("stream", False) # Get input attributes - exclude model, messages, stream from kwargs - extra_kwargs = {k: v for k, v in kwargs.items() if k not in ("model", "messages", "stream")} + extra_kwargs = { + k: v for k, v in kwargs.items() if k not in ("model", "messages", "stream") + } input_attrs = _get_input_attributes(model, messages, **extra_kwargs) with self._start_span("vLLM Chat Completions", input_attrs) as span: try: response = await wrapped(*args, **kwargs) - - if stream: - return self._wrap_async_stream(response, span) - - # Non-streaming response - output_attrs = _get_output_attributes(response) - span.set_attributes(output_attrs) - span.set_status(Status(StatusCode.OK)) - span.end() - return response - - except Exception as e: - span.record_exception(e) - span.set_status(Status(StatusCode.ERROR, str(e))) - span.end() + except BaseException as e: + # Includes asyncio.CancelledError from timeouts and task cancellation. + _end_with_error(span, e) raise - async def _wrap_async_stream(self, stream: Any, span: Span) -> Any: - """Wrap an async streaming response to capture all chunks.""" - chunks = [] - full_content = "" - - try: - async for chunk in stream: - chunks.append(chunk) - if hasattr(chunk, "choices") and chunk.choices: - choice = chunk.choices[0] - if hasattr(choice, "delta") and hasattr(choice.delta, "content"): - if choice.delta.content: - full_content += choice.delta.content - yield chunk - - output_attrs = _get_stream_output_attributes(chunks, full_content) - span.set_attributes(output_attrs) - span.set_status(Status(StatusCode.OK)) - - except Exception as e: - span.record_exception(e) - span.set_status(Status(StatusCode.ERROR, str(e))) - raise - - finally: - span.end() + if stream and hasattr(response, "__anext__"): + return _AsyncStream(response, span) + _finish_response(span, response) + return response diff --git a/python/tests/test_granular_privacy_copies.py b/python/tests/test_granular_privacy_copies.py new file mode 100644 index 000000000..be42bf2b8 --- /dev/null +++ b/python/tests/test_granular_privacy_copies.py @@ -0,0 +1,70 @@ +"""Granular message policies also cover opaque duplicate representations.""" + +import pytest +from fi_instrumentation.fi_types import SpanAttributes +from fi_instrumentation.instrumentation.config import REDACTED_VALUE, TraceConfig + + +@pytest.mark.parametrize( + "flag", ["hide_input_text", "hide_input_messages", "hide_input_images"] +) +@pytest.mark.parametrize( + "key", + [ + SpanAttributes.INPUT_VALUE, + SpanAttributes.RAW_INPUT, + SpanAttributes.GEN_AI_REQUEST_PARAMETERS, + ], +) +@pytest.mark.parametrize("suffix", ["", ".messages.0.content"]) +def test_input_aggregate_is_hidden_without_evaluating_it(flag, key, suffix): + config = TraceConfig(**{flag: True}) + + def sensitive(): + pytest.fail("Hidden aggregate must not be evaluated") + + assert config.mask(key + suffix, sensitive) == REDACTED_VALUE + assert config.mask(SpanAttributes.OUTPUT_VALUE, "answer") == "answer" + assert config.mask(SpanAttributes.GEN_AI_REQUEST_MODEL, "model") == "model" + assert config.mask(SpanAttributes.GEN_AI_USAGE_INPUT_TOKENS, 3) == 3 + + +@pytest.mark.parametrize("flag", ["hide_output_text", "hide_output_messages"]) +@pytest.mark.parametrize( + "key", [SpanAttributes.OUTPUT_VALUE, SpanAttributes.RAW_OUTPUT] +) +@pytest.mark.parametrize("suffix", ["", ".choices.0.message.content"]) +def test_output_aggregate_is_hidden_without_evaluating_it(flag, key, suffix): + config = TraceConfig(**{flag: True}) + + def sensitive(): + pytest.fail("Hidden aggregate must not be evaluated") + + assert config.mask(key + suffix, sensitive) == REDACTED_VALUE + assert config.mask(SpanAttributes.INPUT_VALUE, "prompt") == "prompt" + assert config.mask(SpanAttributes.GEN_AI_USAGE_OUTPUT_TOKENS, 3) == 3 + + +def test_hide_input_messages_covers_separate_image_copy(): + assert ( + TraceConfig(hide_input_messages=True).mask( + SpanAttributes.INPUT_IMAGES, "secret-image" + ) + is None + ) + + +def test_default_capture_and_similar_metadata_keys_stay_visible(): + for key in ( + SpanAttributes.INPUT_VALUE, + SpanAttributes.OUTPUT_VALUE, + SpanAttributes.RAW_INPUT, + SpanAttributes.RAW_OUTPUT, + ): + assert TraceConfig().mask(key, "content") == "content" + assert ( + TraceConfig(hide_input_text=True, hide_output_text=True).mask( + key + "_size", 3 + ) + == 3 + ) From b077381af1210b6931f7ad64b3b290766238be5d Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Sun, 1 Nov 2026 22:02:22 -0800 Subject: [PATCH 2/6] test(privacy): enforce masking of sibling raw content copies Remove DeepSeek's exception for output.value from granular privacy assertions and align Discovery Engine's opaque input expectation with the central mask. Co-Authored-By: Claude Code --- .../frameworks/deepseek/tests/test_privacy_sdk.py | 14 +++++--------- .../tests/test_discoveryengine_config.py | 7 ++++--- 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/python/frameworks/deepseek/tests/test_privacy_sdk.py b/python/frameworks/deepseek/tests/test_privacy_sdk.py index 56c395497..bab21a4d6 100644 --- a/python/frameworks/deepseek/tests/test_privacy_sdk.py +++ b/python/frameworks/deepseek/tests/test_privacy_sdk.py @@ -122,8 +122,7 @@ def respond(request): tool_key = "deepseek.tool_calls.0.function.arguments" content_key = "gen_ai.output.messages.0.message.content" if hide_content: - if policy == "hide_outputs": - assert attributes["output.value"] == REDACTED_VALUE + assert attributes["output.value"] == REDACTED_VALUE if policy == "hide_output_text": for key in (*reasoning_keys, tool_key, content_key): assert attributes[key] == REDACTED_VALUE @@ -131,19 +130,16 @@ def respond(request): assert not any(key.startswith("gen_ai.output.messages") for key in attributes) for key in (*reasoning_keys, tool_key): assert key not in attributes - # Text/message policies intentionally do not hide the separate raw - # output.value field. Everything else must be free of private output. - masked_attributes = dict(attributes) - if policy != "hide_outputs": - masked_attributes.pop("output.value") + # Granular policies must also hide opaque raw copies: excluding + # output.value here would let the same private answer leak again. for private in (answer, reasoning, arguments, "PRIVATE_TOOL_ARGUMENTS_6823"): - assert private not in json.dumps(masked_attributes) + assert private not in json.dumps(attributes) else: for key in reasoning_keys: assert attributes[key] == reasoning assert attributes[tool_key] == arguments assert attributes[content_key] == answer - if not hide_content or policy != "hide_outputs": + if not hide_content: payload = json.loads(attributes["output.value"]) output = payload if streaming else payload["choices"][0]["message"] assert output["content"] == answer diff --git a/python/frameworks/discoveryengine/tests/test_discoveryengine_config.py b/python/frameworks/discoveryengine/tests/test_discoveryengine_config.py index e8c551d7d..a295507d9 100644 --- a/python/frameworks/discoveryengine/tests/test_discoveryengine_config.py +++ b/python/frameworks/discoveryengine/tests/test_discoveryengine_config.py @@ -210,8 +210,8 @@ def test_hide_outputs_changes_nothing_because_no_output_text_is_recorded(fake): @pytest.mark.parametrize("source", ["config", "env"]) @pytest.mark.parametrize("flag", OTHER_CONTENT_HIDE_FLAGS) def test_other_content_hide_flags_follow_core_error_policy(fake, monkeypatch, flag, source): - # These flags do not hide a retrieval query, but native core derives error - # privacy from all content-hide flags, not only hide_inputs. No opt-out. + # Granular input policies also hide the opaque input.value copy. Output + # policies leave it visible; all content-hide flags derive error privacy. plain = _denied(fake, {}) options = {} if source == "config": @@ -219,7 +219,8 @@ def test_other_content_hide_flags_follow_core_error_policy(fake, monkeypatch, fl else: monkeypatch.setenv("FI_" + flag.upper(), "true") span = _denied(fake, options) - assert attrs(span)["input.value"] == DENIED_QUERY + expected_query = REDACTED_VALUE if flag in ("hide_input_messages", "hide_input_text") else DENIED_QUERY + assert attrs(span)["input.value"] == expected_query assert attrs(span)["discoveryengine.error.code"] == 403 assert attrs(span)["discoveryengine.error.status"] == "PERMISSION_DENIED" assert span.status.status_code is StatusCode.ERROR From 809b6bab430325c217c477a04ce71cdbd5b6198d Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Sun, 1 Nov 2026 22:54:52 -0800 Subject: [PATCH 3/6] fix(vllm): enforce privacy core and finalize async cleanup aliases Require fi-instrumentation-otel 1.2.3 for OpenAI and vLLM, test built wheels, and preserve messages while hiding every legacy invocation-parameter copy. Finalize aclose on both routes without duplicating PR #411's shared close methods. Co-Authored-By: Claude Code --- .github/workflows/ollama-provider-tests.yml | 8 +- .github/workflows/python-tests.yml | 13 +- .github/workflows/vercel-gateway-tests.yml | 4 +- .release-please-manifest.json | 2 +- .../instrumentation/version.py | 2 +- python/frameworks/openai/pyproject.toml | 2 +- .../openai/traceai_openai/_stream.py | 13 ++ python/frameworks/vllm/pyproject.toml | 2 +- .../tests/test_core_privacy_dependency.py | 25 ++++ .../vllm/tests/test_recall_privacy.py | 125 ++++++++++++++++++ .../frameworks/vllm/traceai_vllm/__init__.py | 8 +- .../frameworks/vllm/traceai_vllm/_wrappers.py | 31 ++++- python/pyproject.toml | 2 +- 13 files changed, 218 insertions(+), 19 deletions(-) create mode 100644 python/frameworks/vllm/tests/test_core_privacy_dependency.py diff --git a/.github/workflows/ollama-provider-tests.yml b/.github/workflows/ollama-provider-tests.yml index 7049e3ca3..ce54466de 100644 --- a/.github/workflows/ollama-provider-tests.yml +++ b/.github/workflows/ollama-provider-tests.yml @@ -42,11 +42,11 @@ jobs: - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: '3.11' - - name: Install SDK and test dependencies + - name: Install built wheels and SDK test dependencies env: OPENAI_VERSION: ${{ matrix.openai }} run: >- - python -m pip install -e python -e python/frameworks/openai -e python/frameworks/vllm + python -m pip install ./python ./python/frameworks/openai ./python/frameworks/vllm "openai==$OPENAI_VERSION" 'pytest==8.4.2' 'pytest-asyncio==0.24.0' 'pytest-socket==0.7.0' - name: Test attribution over localhost OTLP env: @@ -77,9 +77,9 @@ jobs: - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: '3.10' - - name: Install legacy package without the shared instrumentor + - name: Install legacy wheels without the shared instrumentor run: >- - python -m pip install -e python -e python/frameworks/vllm + python -m pip install ./python ./python/frameworks/vllm 'openai==1.0.0' 'httpx==0.27.2' 'pytest==8.4.2' 'pytest-asyncio==0.24.0' 'pytest-socket==0.7.0' - name: Test legacy SDK with outbound network disabled diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml index c708aa2de..af540b6fe 100644 --- a/.github/workflows/python-tests.yml +++ b/.github/workflows/python-tests.yml @@ -176,7 +176,7 @@ jobs: - name: Install the package and its dev dependencies (poetry) id: install_poetry - if: matrix.package.backend == 'poetry' + if: matrix.package.backend == 'poetry' && matrix.package.name != 'openai' && matrix.package.name != 'vllm' working-directory: ${{ matrix.package.dir }} run: | pip install "poetry==$POETRY_VERSION" @@ -193,6 +193,17 @@ jobs: fi done + - name: Install privacy-dependent OpenAI packages with the local core + if: matrix.package.name == 'openai' || matrix.package.name == 'vllm' + working-directory: ${{ matrix.package.dir }} + run: | + # These packages require the core release prepared in the same PR. + # Resolve built wheels together, never an older core from PyPI/poetry.lock. + python -m venv .venv + .venv/bin/pip install "$GITHUB_WORKSPACE/python" . \ + pytest==8.4.2 pytest-asyncio==0.24.0 pytest-mock==3.14.0 + .venv/bin/pip check + - name: Install the package and its dev dependencies (hatch) id: install_hatch if: matrix.package.backend == 'hatch' diff --git a/.github/workflows/vercel-gateway-tests.yml b/.github/workflows/vercel-gateway-tests.yml index c3fc4e940..ec32b8ea8 100644 --- a/.github/workflows/vercel-gateway-tests.yml +++ b/.github/workflows/vercel-gateway-tests.yml @@ -34,10 +34,10 @@ jobs: - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: '3.13' - - name: Install the SDK and published tracing core + - name: Install the SDK and tracing wheels env: SDK_VERSION: ${{ matrix.sdk }} - run: python -m pip install "openai==$SDK_VERSION" 'fi-instrumentation-otel==1.2.2' pytest pytest-socket + run: python -m pip install "openai==$SDK_VERSION" ./python ./python/frameworks/openai pytest pytest-socket - name: Run SDK-to-OTLP contracts with non-local sockets blocked env: PYTHONPATH: python/frameworks/openai diff --git a/.release-please-manifest.json b/.release-please-manifest.json index d394b6567..d178c8099 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -125,7 +125,7 @@ "go/lambda": "0.1.0", "go/openai": "0.1.0", "csharp/src/FIInstrumentation": "1.0.0", - "python": "1.2.2", + "python": "1.2.3", "typescript/packages/traceai_elevenlabs": "0.1.0", "go/bedrock": "0.1.0" } diff --git a/python/fi_instrumentation/instrumentation/version.py b/python/fi_instrumentation/instrumentation/version.py index 48c9c4fc4..aa52eaa3f 100644 --- a/python/fi_instrumentation/instrumentation/version.py +++ b/python/fi_instrumentation/instrumentation/version.py @@ -1 +1 @@ -__version__ = "1.2.2" # x-release-please-version +__version__ = "1.2.3" # x-release-please-version diff --git a/python/frameworks/openai/pyproject.toml b/python/frameworks/openai/pyproject.toml index 3a189f628..6fed50223 100644 --- a/python/frameworks/openai/pyproject.toml +++ b/python/frameworks/openai/pyproject.toml @@ -28,7 +28,7 @@ classifiers = [ [tool.poetry.dependencies] python = ">3.9,<3.14" -fi-instrumentation-otel = ">=0.1.11" +fi-instrumentation-otel = ">=1.2.3" openai = ">=1.69.0" [tool.poetry.urls] diff --git a/python/frameworks/openai/traceai_openai/_stream.py b/python/frameworks/openai/traceai_openai/_stream.py index 5975a7671..fa1863b67 100644 --- a/python/frameworks/openai/traceai_openai/_stream.py +++ b/python/frameworks/openai/traceai_openai/_stream.py @@ -164,6 +164,19 @@ async def __aexit__(self, exc_type: Any, exc: Any, tb: Any) -> Any: finally: _finish_on_exit(self, exc) + async def aclose(self) -> None: + # Intercept the SDK alias: delegating its bound method bypasses tracing. + # Finish here as well as in close(), so alias cleanup is independently + # safe even when the wrapped SDK owns close(). Finalization is idempotent. + error = None + try: + await self.close() + except BaseException as exc: + error = exc + raise + finally: + _finish_on_exit(self, error) + def __aiter__(self): return self diff --git a/python/frameworks/vllm/pyproject.toml b/python/frameworks/vllm/pyproject.toml index 738c677fa..16c7cd143 100644 --- a/python/frameworks/vllm/pyproject.toml +++ b/python/frameworks/vllm/pyproject.toml @@ -28,7 +28,7 @@ classifiers = [ [tool.poetry.dependencies] python = ">3.9,<3.14" -fi-instrumentation-otel = ">=0.1.11" +fi-instrumentation-otel = ">=1.2.3" openai = ">=1.0.0" [tool.poetry.group.dev.dependencies] diff --git a/python/frameworks/vllm/tests/test_core_privacy_dependency.py b/python/frameworks/vllm/tests/test_core_privacy_dependency.py new file mode 100644 index 000000000..cf5bda322 --- /dev/null +++ b/python/frameworks/vllm/tests/test_core_privacy_dependency.py @@ -0,0 +1,25 @@ +"""Installed metadata must require the core that protects duplicate content.""" + +from importlib.metadata import PackageNotFoundError, requires, version + +import pytest +from packaging.requirements import Requirement +from packaging.version import Version + + +@pytest.mark.parametrize("package", ["traceAI-vllm", "traceAI-openai"]) +def test_instrumentor_requires_privacy_fixed_core(package): + try: + dependencies = requires(package) + except PackageNotFoundError: + pytest.skip( + "Shared instrumentor is intentionally absent at the legacy SDK floor" + ) + core = next( + Requirement(value) + for value in dependencies + if Requirement(value).name == "fi-instrumentation-otel" + ) + assert Version("1.2.2") not in core.specifier + assert Version("1.2.3") in core.specifier + assert Version(version("fi-instrumentation-otel")) in core.specifier diff --git a/python/frameworks/vllm/tests/test_recall_privacy.py b/python/frameworks/vllm/tests/test_recall_privacy.py index 0b1b867ed..f0502056b 100644 --- a/python/frameworks/vllm/tests/test_recall_privacy.py +++ b/python/frameworks/vllm/tests/test_recall_privacy.py @@ -434,3 +434,128 @@ def run(): expected = StatusCode.ERROR if exception_type is not None else StatusCode.OK assert span.status.status_code == expected assert "SYNTHETIC-CONSUMER-ERROR" not in serialized(span) + + +@pytest.mark.parametrize("route", ROUTES) +@pytest.mark.parametrize("consume_first", [False, True]) +@pytest.mark.parametrize("cleanup", ["direct", "contextlib"]) +def test_aclose_finishes_once_and_releases_response(route, consume_first, cleanup): + from contextlib import aclosing + + async def run(exporter): + closed = [] + responses = [] + + class Body(httpx.AsyncByteStream): + async def __aiter__(self): + yield f"data: {json.dumps(chunk(OUTPUT))}\n\n".encode() + yield b"data: [DONE]\n\n" + + async def aclose(self): + closed.append(True) + + async def handler(request): + response = httpx.Response( + 200, headers={"content-type": "text/event-stream"}, stream=Body() + ) + responses.append(response) + return response + + async with client_for(route, handler, True) as client: + stream = await client.chat.completions.create(**PARAMS, stream=True) + assert isinstance(stream, openai.AsyncStream) + if cleanup == "contextlib": + async with aclosing(stream) as entered: + assert entered is stream + if consume_first: + assert (await entered.__anext__()).choices[ + 0 + ].delta.content == OUTPUT + else: + if consume_first: + assert (await stream.__anext__()).choices[0].delta.content == OUTPUT + await stream.aclose() + await stream.aclose() + only_span(exporter) # aclose itself must finalize, without a later close(). + await stream.close() + assert responses[0].is_closed + assert closed == [True] + span = only_span(exporter) + assert span.status.status_code == StatusCode.OK + if consume_first: + assert OUTPUT in serialized(span) + + with instrumented(route, TraceConfig()) as exporter: + asyncio.run(run(exporter)) + + +@pytest.mark.parametrize("route", ROUTES) +@pytest.mark.parametrize("use_async", [False, True]) +@pytest.mark.parametrize("streaming", [False, True]) +@pytest.mark.parametrize("policy", ["environment", "explicit", "override"]) +def test_invocation_privacy_covers_raw_copy( + route, use_async, streaming, policy, monkeypatch +): + secret_stop = "SYNTHETIC-PRIVATE-STOP-PARAMETER" + secret_extra = "SYNTHETIC-PRIVATE-EXTRA-PARAMETER" + monkeypatch.setenv("FI_HIDE_LLM_INVOCATION_PARAMETERS", "true") + config = ( + TraceConfig() + if policy == "environment" + else TraceConfig(hide_llm_invocation_parameters=policy == "explicit") + ) + hidden = policy != "override" + seen = [] + + def handler(request): + seen.append(json.loads(request.content)) + if streaming: + return httpx.Response( + 200, + headers={"content-type": "text/event-stream"}, + content=f"data: {json.dumps(chunk(OUTPUT))}\n\ndata: [DONE]\n\n", + ) + return httpx.Response(200, json=COMPLETION) + + kwargs = dict( + **PARAMS, + stop=[secret_stop], + temperature=0.7, + extra_body={"custom_parameter": secret_extra}, + stream=streaming, + ) + + async def run_async(): + async with client_for(route, handler, True) as client: + result = await client.chat.completions.create(**kwargs) + if streaming: + assert [c async for c in result] + + with instrumented(route, config) as exporter: + if use_async: + asyncio.run(run_async()) + else: + with client_for(route, handler) as client: + result = client.chat.completions.create(**kwargs) + if streaming: + assert list(result) + span = only_span(exporter) + assert seen[0]["stop"] == [secret_stop] + assert seen[0]["temperature"] == 0.7 + assert seen[0]["custom_parameter"] == secret_extra + assert seen[0]["messages"] == PARAMS["messages"] + assert INPUT in serialized(span) + assert OUTPUT in serialized(span) + if hidden: + assert "gen_ai.request.parameters" not in span.attributes + assert secret_stop not in serialized(span) + assert secret_extra not in serialized(span) + if route == "legacy": + raw = json.loads(span.attributes["input.value"]) + assert raw == {"model": PARAMS["model"], "messages": PARAMS["messages"]} + else: + assert secret_stop in span.attributes["gen_ai.request.parameters"] + if route == "legacy": + raw = json.loads(span.attributes["input.value"]) + assert raw["temperature"] == 0.7 + assert raw["extra_body"] == {"custom_parameter": secret_extra} diff --git a/python/frameworks/vllm/traceai_vllm/__init__.py b/python/frameworks/vllm/traceai_vllm/__init__.py index 94413cd86..e69b943df 100644 --- a/python/frameworks/vllm/traceai_vllm/__init__.py +++ b/python/frameworks/vllm/traceai_vllm/__init__.py @@ -93,7 +93,9 @@ def _instrument(self, **kwargs: Any) -> None: wrap_function_wrapper( module="openai.resources.chat", name="Completions.create", - wrapper=_CompletionsWrapper(tracer=self._tracer, vllm_base_urls=self._vllm_base_urls), + wrapper=_CompletionsWrapper( + tracer=self._tracer, vllm_base_urls=self._vllm_base_urls, config=config + ), ) # Wrap asynchronous completions @@ -101,7 +103,9 @@ def _instrument(self, **kwargs: Any) -> None: wrap_function_wrapper( module="openai.resources.chat", name="AsyncCompletions.create", - wrapper=_AsyncCompletionsWrapper(tracer=self._tracer, vllm_base_urls=self._vllm_base_urls), + wrapper=_AsyncCompletionsWrapper( + tracer=self._tracer, vllm_base_urls=self._vllm_base_urls, config=config + ), ) def _uninstrument(self, **kwargs: Any) -> None: diff --git a/python/frameworks/vllm/traceai_vllm/_wrappers.py b/python/frameworks/vllm/traceai_vllm/_wrappers.py index 1f10743b8..738e14581 100644 --- a/python/frameworks/vllm/traceai_vllm/_wrappers.py +++ b/python/frameworks/vllm/traceai_vllm/_wrappers.py @@ -7,7 +7,7 @@ from contextlib import contextmanager from typing import Any, Dict, Iterator, List, Optional, Tuple -from fi_instrumentation import safe_json_dumps +from fi_instrumentation import TraceConfig, safe_json_dumps from fi_instrumentation.fi_types import ( FiSpanKindValues, ImageAttributes, @@ -48,11 +48,13 @@ def __init__( tracer: trace_api.Tracer, vllm_base_urls: List[str], *args: Any, + config: Optional[TraceConfig] = None, **kwargs: Any, ) -> None: super().__init__(*args, **kwargs) self._tracer = tracer self._vllm_base_urls = vllm_base_urls + self._config = config if config is not None else TraceConfig() def _is_vllm_client(self, instance: Any) -> bool: """Check if the client instance is configured for vLLM.""" @@ -92,6 +94,8 @@ def _start_span( def _get_input_attributes( model: str, messages: list, + *, + hide_invocation_parameters: bool = False, **kwargs: Any, ) -> Dict[str, AttributeValue]: """Extract input attributes from request parameters.""" @@ -160,13 +164,16 @@ def _get_input_attributes( f"{part_prefix}.{MessageContentAttributes.MESSAGE_CONTENT_IMAGE}.{ImageAttributes.IMAGE_URL}" ] = image["url"] - # Add raw input + # Keep messages visible, but do not duplicate private invocation settings + # (including provider-specific extra_body fields) into the raw input. raw_input = {"model": model, "messages": messages} raw_input.update( { k: v for k, v in kwargs.items() - if v is not None and k not in _TRANSPORT_OPTIONS + if v is not None + and k not in _TRANSPORT_OPTIONS + and not hide_invocation_parameters } ) attributes[SpanAttributes.INPUT_VALUE] = safe_json_dumps(raw_input) @@ -415,6 +422,10 @@ async def close(self) -> None: raise self._self_tracking.finish() + async def aclose(self) -> None: + # ObjectProxy otherwise delegates the bound SDK alias past our close(). + await self.close() + def _finish_response(span: Span, response: Any) -> None: """End the span for a non-stream result. @@ -463,7 +474,12 @@ def __call__( extra_kwargs = { k: v for k, v in kwargs.items() if k not in ("model", "messages", "stream") } - input_attrs = _get_input_attributes(model, messages, **extra_kwargs) + input_attrs = _get_input_attributes( + model, + messages, + hide_invocation_parameters=self._config.hide_llm_invocation_parameters, + **extra_kwargs, + ) with self._start_span("vLLM Chat Completions", input_attrs) as span: try: @@ -512,7 +528,12 @@ async def __call__( extra_kwargs = { k: v for k, v in kwargs.items() if k not in ("model", "messages", "stream") } - input_attrs = _get_input_attributes(model, messages, **extra_kwargs) + input_attrs = _get_input_attributes( + model, + messages, + hide_invocation_parameters=self._config.hide_llm_invocation_parameters, + **extra_kwargs, + ) with self._start_span("vLLM Chat Completions", input_attrs) as span: try: diff --git a/python/pyproject.toml b/python/pyproject.toml index 0559ffd47..224802eb1 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "fi-instrumentation-otel" -version = "1.2.2" +version = "1.2.3" description = "OpenTelemetry tracing for AI applications: tracer setup, span helpers, exporters, and semantic conventions for Future AGI." authors = ["Future AGI "] readme = "README.md" From cbe0a285470a28399dbf300ee3277cdedb72faa6 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Sun, 1 Nov 2026 22:58:39 -0800 Subject: [PATCH 4/6] test(openai): validate privacy release wheels across dependent CI Keep fail-closed installation guards for the local minimum-core path, build core alongside Parallel distributions, and patch the actual httpx2 transport in SDK 3.x framework tests. Co-Authored-By: Claude Code --- .github/scripts/test_python_test_plan.py | 13 ++++++++++++- .github/workflows/parallel-python-tests.yml | 5 +++-- .github/workflows/python-tests.yml | 4 ++-- python/tests/test_framework_openai.py | 4 +++- 4 files changed, 20 insertions(+), 6 deletions(-) diff --git a/.github/scripts/test_python_test_plan.py b/.github/scripts/test_python_test_plan.py index 0f52ca91a..db7765dc7 100644 --- a/.github/scripts/test_python_test_plan.py +++ b/.github/scripts/test_python_test_plan.py @@ -642,9 +642,15 @@ def test_every_run_step_uses_bash_with_pipefail(self): ( "install_poetry", "Install the package and its dev dependencies (poetry)", - "if: matrix.package.backend == 'poetry'", + "if: matrix.package.backend == 'poetry' && matrix.package.name != 'openai' && matrix.package.name != 'vllm'", "poetry install --no-interaction", ), + ( + "install_local_core", + "Install privacy-dependent OpenAI packages with the local core", + "if: matrix.package.name == 'openai' || matrix.package.name == 'vllm'", + '.venv/bin/pip install "$GITHUB_WORKSPACE/python" . pytest==8.4.2 pytest-asyncio==0.24.0 pytest-mock==3.14.0', + ), ( "install_hatch", "Install the package and its dev dependencies (hatch)", @@ -677,6 +683,11 @@ def test_every_run_step_uses_bash_with_pipefail(self): "fi", "done", ], + "install_local_core": [ + "python -m venv .venv", + '.venv/bin/pip install "$GITHUB_WORKSPACE/python" . pytest==8.4.2 pytest-asyncio==0.24.0 pytest-mock==3.14.0', + ".venv/bin/pip check", + ], "install_hatch": [ "python -m venv .venv", '.venv/bin/pip install -e ".[dev]"', diff --git a/.github/workflows/parallel-python-tests.yml b/.github/workflows/parallel-python-tests.yml index 553945dd8..29be5b9ee 100644 --- a/.github/workflows/parallel-python-tests.yml +++ b/.github/workflows/parallel-python-tests.yml @@ -57,12 +57,13 @@ jobs: run: | python -m venv .parallel-test-venv .parallel-test-venv/bin/pip install build + .parallel-test-venv/bin/python -m build "--$DISTRIBUTION" python .parallel-test-venv/bin/python -m build "--$DISTRIBUTION" python/frameworks/parallel .parallel-test-venv/bin/python -m build "--$DISTRIBUTION" python/frameworks/openai if [ "$DISTRIBUTION" = wheel ]; then - packages=(python/frameworks/{parallel,openai}/dist/*.whl) + packages=(python/dist/*.whl python/frameworks/{parallel,openai}/dist/*.whl) else - packages=(python/frameworks/{parallel,openai}/dist/*.tar.gz) + packages=(python/dist/*.tar.gz python/frameworks/{parallel,openai}/dist/*.tar.gz) fi .parallel-test-venv/bin/pip install "${packages[@]}" \ "parallel-web==$PARALLEL_VERSION" "openai==$OPENAI_VERSION" pytest pytest-asyncio diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml index af540b6fe..66363b62f 100644 --- a/.github/workflows/python-tests.yml +++ b/.github/workflows/python-tests.yml @@ -194,14 +194,14 @@ jobs: done - name: Install privacy-dependent OpenAI packages with the local core + id: install_local_core if: matrix.package.name == 'openai' || matrix.package.name == 'vllm' working-directory: ${{ matrix.package.dir }} run: | # These packages require the core release prepared in the same PR. # Resolve built wheels together, never an older core from PyPI/poetry.lock. python -m venv .venv - .venv/bin/pip install "$GITHUB_WORKSPACE/python" . \ - pytest==8.4.2 pytest-asyncio==0.24.0 pytest-mock==3.14.0 + .venv/bin/pip install "$GITHUB_WORKSPACE/python" . pytest==8.4.2 pytest-asyncio==0.24.0 pytest-mock==3.14.0 .venv/bin/pip check - name: Install the package and its dev dependencies (hatch) diff --git a/python/tests/test_framework_openai.py b/python/tests/test_framework_openai.py index 5917a54f1..53e574f59 100644 --- a/python/tests/test_framework_openai.py +++ b/python/tests/test_framework_openai.py @@ -58,7 +58,9 @@ def setup_trace_provider(self): @pytest.fixture def mock_openai_requests(self): """Mock OpenAI HTTP requests to avoid real API calls.""" - with patch('httpx.Client.send') as mock_request: + # OpenAI 3.x defaults to httpx2; patch the transport the SDK uses. + transport = "httpx2" if int(openai.__version__.split(".")[0]) >= 3 else "httpx" + with patch(f"{transport}.Client.send") as mock_request: # Set up mock response for chat completions mock_response = MagicMock() mock_response.status_code = 200 From 83bbe3c228bfe28c34ae93dba62df699248cb602 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Sun, 1 Nov 2026 23:02:35 -0800 Subject: [PATCH 5/6] test(openai): retain legacy HTTPX transport fixtures in latest CI OpenAI 3.x no longer installs classic httpx transitively; tests still exercise both supported transports. Co-Authored-By: Claude Code --- .github/scripts/test_python_test_plan.py | 4 ++-- .github/workflows/python-tests.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/scripts/test_python_test_plan.py b/.github/scripts/test_python_test_plan.py index db7765dc7..9762028a4 100644 --- a/.github/scripts/test_python_test_plan.py +++ b/.github/scripts/test_python_test_plan.py @@ -649,7 +649,7 @@ def test_every_run_step_uses_bash_with_pipefail(self): "install_local_core", "Install privacy-dependent OpenAI packages with the local core", "if: matrix.package.name == 'openai' || matrix.package.name == 'vllm'", - '.venv/bin/pip install "$GITHUB_WORKSPACE/python" . pytest==8.4.2 pytest-asyncio==0.24.0 pytest-mock==3.14.0', + '.venv/bin/pip install "$GITHUB_WORKSPACE/python" . pytest==8.4.2 pytest-asyncio==0.24.0 pytest-mock==3.14.0 httpx==0.27.2', ), ( "install_hatch", @@ -685,7 +685,7 @@ def test_every_run_step_uses_bash_with_pipefail(self): ], "install_local_core": [ "python -m venv .venv", - '.venv/bin/pip install "$GITHUB_WORKSPACE/python" . pytest==8.4.2 pytest-asyncio==0.24.0 pytest-mock==3.14.0', + '.venv/bin/pip install "$GITHUB_WORKSPACE/python" . pytest==8.4.2 pytest-asyncio==0.24.0 pytest-mock==3.14.0 httpx==0.27.2', ".venv/bin/pip check", ], "install_hatch": [ diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml index 66363b62f..bdb8293d9 100644 --- a/.github/workflows/python-tests.yml +++ b/.github/workflows/python-tests.yml @@ -201,7 +201,7 @@ jobs: # These packages require the core release prepared in the same PR. # Resolve built wheels together, never an older core from PyPI/poetry.lock. python -m venv .venv - .venv/bin/pip install "$GITHUB_WORKSPACE/python" . pytest==8.4.2 pytest-asyncio==0.24.0 pytest-mock==3.14.0 + .venv/bin/pip install "$GITHUB_WORKSPACE/python" . pytest==8.4.2 pytest-asyncio==0.24.0 pytest-mock==3.14.0 httpx==0.27.2 .venv/bin/pip check - name: Install the package and its dev dependencies (hatch) From dbcf699c76f6a7365814d8a0b1a5d31f8aa557cf Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Sun, 1 Nov 2026 23:44:00 -0800 Subject: [PATCH 6/6] fix(vllm): finalize spans when abandoned streams are disposed Restore generator-finally tracing behavior through an idempotent tracking-state finalizer. Keep transport cleanup independent and cover sync/async disposal, output privacy, retained SDK transports, and finalizer failures at floor/latest SDK versions. Co-Authored-By: Claude Code --- .../vllm/tests/test_recall_privacy.py | 115 ++++++++++++++++++ .../vllm/tests/test_stream_span_disposal.py | 39 ++++++ .../frameworks/vllm/traceai_vllm/_wrappers.py | 11 ++ 3 files changed, 165 insertions(+) create mode 100644 python/frameworks/vllm/tests/test_stream_span_disposal.py diff --git a/python/frameworks/vllm/tests/test_recall_privacy.py b/python/frameworks/vllm/tests/test_recall_privacy.py index f0502056b..9cf2106c5 100644 --- a/python/frameworks/vllm/tests/test_recall_privacy.py +++ b/python/frameworks/vllm/tests/test_recall_privacy.py @@ -1,6 +1,7 @@ """Retained and shared vLLM routes: privacy and cancellation at SDK boundaries.""" import asyncio +import gc import importlib.util import json from contextlib import contextmanager @@ -559,3 +560,117 @@ async def run_async(): raw = json.loads(span.attributes["input.value"]) assert raw["temperature"] == 0.7 assert raw["extra_body"] == {"custom_parameter": secret_extra} + + +@pytest.mark.parametrize("use_async", [False, True]) +@pytest.mark.parametrize("consumer_exit", ["break", "return"]) +@pytest.mark.parametrize("hide_output_text", [False, True]) +def test_legacy_partial_stream_disposal_keeps_span( + use_async, consumer_exit, hide_output_text +): + def handler(request): + return httpx.Response( + 200, + headers={"content-type": "text/event-stream"}, + content=f"data: {json.dumps(chunk(OUTPUT))}\n\ndata: [DONE]\n\n", + ) + + def run_sync(): + with client_for("legacy", handler) as client: + for value in client.chat.completions.create(**PARAMS, stream=True): + assert value.choices[0].delta.content == OUTPUT + if consumer_exit == "return": + return + break + + async def run_async(): + async with client_for("legacy", handler, True) as client: + async for value in await client.chat.completions.create( + **PARAMS, stream=True + ): + assert value.choices[0].delta.content == OUTPUT + if consumer_exit == "return": + return + break + + with instrumented( + "legacy", TraceConfig(hide_output_text=hide_output_text) + ) as exporter: + if use_async: + asyncio.run(run_async()) # Includes async-generator shutdown. + else: + run_sync() + gc.collect() + span = only_span(exporter) + assert span.status.status_code == StatusCode.OK + assert span.attributes["gen_ai.provider.name"] == "vllm" + assert (OUTPUT in serialized(span)) is not hide_output_text + assert INPUT in serialized(span) + assert only_span(exporter) is span # Provider shutdown cannot duplicate it. + + +@pytest.mark.parametrize("use_async", [False, True]) +@pytest.mark.parametrize("consume_first", [False, True]) +def test_legacy_disposal_finalizes_without_closing_transport(use_async, consume_first): + closed = [] + + class SyncBody(httpx.SyncByteStream): + def __iter__(self): + yield f"data: {json.dumps(chunk(OUTPUT))}\n\n".encode() + yield b"data: [DONE]\n\n" + + def close(self): + closed.append(True) + + class AsyncBody(httpx.AsyncByteStream): + async def __aiter__(self): + yield f"data: {json.dumps(chunk(OUTPUT))}\n\n".encode() + yield b"data: [DONE]\n\n" + + async def aclose(self): + closed.append(True) + + def handler(request): + return httpx.Response( + 200, + headers={"content-type": "text/event-stream"}, + stream=AsyncBody() if use_async else SyncBody(), + ) + + def assert_disposed(exporter, native): + # Retain the SDK stream: tracking disposal must be independent of SDK + # garbage collection, network cleanup and async-generator shutdown. + gc.collect() + span = only_span(exporter) + assert span.status.status_code == StatusCode.OK + assert (OUTPUT in serialized(span)) is consume_first + assert not native.response.is_closed + assert not closed + + async def run_async(exporter): + async with client_for("legacy", handler, True) as client: + stream = await client.chat.completions.create(**PARAMS, stream=True) + native = stream.__wrapped__ + if consume_first: + await stream.__anext__() + assert not exporter.get_finished_spans() + del stream + assert_disposed(exporter, native) # No await or loop shutdown needed. + await native.response.aclose() + + with instrumented("legacy", TraceConfig()) as exporter: + if use_async: + asyncio.run(run_async(exporter)) + else: + with client_for("legacy", handler) as client: + stream = client.chat.completions.create(**PARAMS, stream=True) + native = stream.__wrapped__ + if consume_first: + next(stream) + assert not exporter.get_finished_spans() + del stream + assert_disposed(exporter, native) + native.response.close() + gc.collect() + assert closed == [True] + only_span(exporter) diff --git a/python/frameworks/vllm/tests/test_stream_span_disposal.py b/python/frameworks/vllm/tests/test_stream_span_disposal.py new file mode 100644 index 000000000..7ad8a5230 --- /dev/null +++ b/python/frameworks/vllm/tests/test_stream_span_disposal.py @@ -0,0 +1,39 @@ +"""Span-only disposal must be idempotent and safe during garbage collection.""" + +import gc +import sys +from unittest.mock import Mock + +import pytest +from opentelemetry.trace import Span, StatusCode +from traceai_vllm._wrappers import _StreamSpan + + +@pytest.mark.parametrize("error", [None, RuntimeError("synthetic stream failure")]) +def test_disposal_does_not_end_or_overwrite_a_finished_span(error): + span = Mock(spec=Span) + tracking = _StreamSpan(span) + tracking.finish(error) + del tracking + gc.collect() + span.end.assert_called_once_with() + span.set_attributes.assert_called_once() + span.set_status.assert_called_once() + expected = StatusCode.OK if error is None else StatusCode.ERROR + assert span.set_status.call_args.args[0].status_code == expected + if error is None: + span.record_exception.assert_not_called() + else: + span.record_exception.assert_called_once_with(error) + + +def test_disposal_does_not_leak_tracing_failures(monkeypatch): + unraisable = [] + monkeypatch.setattr(sys, "unraisablehook", unraisable.append) + span = Mock(spec=Span) + span.end.side_effect = RuntimeError("synthetic exporter failure") + tracking = _StreamSpan(span) + del tracking + gc.collect() + span.end.assert_called_once_with() + assert not unraisable diff --git a/python/frameworks/vllm/traceai_vllm/_wrappers.py b/python/frameworks/vllm/traceai_vllm/_wrappers.py index 738e14581..6f1d8717f 100644 --- a/python/frameworks/vllm/traceai_vllm/_wrappers.py +++ b/python/frameworks/vllm/traceai_vllm/_wrappers.py @@ -293,6 +293,17 @@ def __init__(self, span: Span) -> None: self.content = "" self.finished = False + def __del__(self) -> None: + # Preserve the old generators' finally cleanup when a caller abandons + # iteration. Only end tracing: SDK transport cleanup may require an + # event loop and remains the caller's responsibility via close/context exit. + try: + self.finish() + except Exception: + # Destruction must not surface tracing failures or assume that + # logging is still available during interpreter shutdown. + pass + def process(self, chunk: Any) -> None: try: self.chunks.append(chunk)