From 82113634941d568223593c66ee5bf83571474266 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Tue, 3 Nov 2026 21:22:11 -0800 Subject: [PATCH 1/6] fix(parallel): protect Responses privacy and interrupted workflows Port and revalidate the relevant open PR 520 work against dev, preserve shared-provider and raw-envelope contracts, scrub MCP URL credentials and native metadata, and add cloud-qualified regression coverage. Co-Authored-By: Claude Code --- .github/workflows/parallel-python-tests.yml | 10 +- python/frameworks/openai/poetry.lock | 16 +- python/frameworks/openai/pyproject.toml | 5 +- .../tests/test_core_dependency_privacy.py | 42 ++ .../tests/test_deepseek_raw_response.py | 8 +- .../openai/tests/test_deepseek_route.py | 3 +- ...test_effective_body_and_raw_consumption.py | 260 ++++++++++ .../tests/test_release_qualification.py | 68 +++ .../openai/tests/test_response_edge_cases.py | 185 +++++++ .../openai/tests/test_response_envelopes.py | 455 +++++++++++++++++ .../test_responses_privacy_and_raw_streams.py | 411 +++++++++++++++ .../openai/tests/test_responses_safety.py | 477 ++++++++++++++++++ .../openai/traceai_openai/__init__.py | 13 + .../_attributes/_responses_api.py | 5 +- .../openai/traceai_openai/_privacy.py | 157 ++++++ .../openai/traceai_openai/_request.py | 360 ++++--------- .../openai/traceai_openai/_request_context.py | 41 ++ .../traceai_openai/_response_accumulator.py | 57 ++- .../openai/traceai_openai/_span_io_handler.py | 15 +- .../openai/traceai_openai/_stream.py | 290 ++++++++++- .../openai/traceai_openai/_with_span.py | 34 +- python/frameworks/parallel/README.md | 2 +- .../parallel/examples/task_workflow.py | 4 +- .../tests/test_metadata_credentials.py | 58 +++ .../parallel/tests/test_task_workflow.py | 52 ++ .../parallel/traceai_parallel/_wrappers.py | 4 +- 26 files changed, 2697 insertions(+), 335 deletions(-) create mode 100644 python/frameworks/openai/tests/test_core_dependency_privacy.py create mode 100644 python/frameworks/openai/tests/test_effective_body_and_raw_consumption.py create mode 100644 python/frameworks/openai/tests/test_release_qualification.py create mode 100644 python/frameworks/openai/tests/test_response_edge_cases.py create mode 100644 python/frameworks/openai/tests/test_response_envelopes.py create mode 100644 python/frameworks/openai/tests/test_responses_privacy_and_raw_streams.py create mode 100644 python/frameworks/openai/tests/test_responses_safety.py create mode 100644 python/frameworks/openai/traceai_openai/_privacy.py create mode 100644 python/frameworks/openai/traceai_openai/_request_context.py create mode 100644 python/frameworks/parallel/tests/test_metadata_credentials.py diff --git a/.github/workflows/parallel-python-tests.yml b/.github/workflows/parallel-python-tests.yml index 553945dd8..62114b47d 100644 --- a/.github/workflows/parallel-python-tests.yml +++ b/.github/workflows/parallel-python-tests.yml @@ -37,7 +37,7 @@ jobs: distribution: wheel - python: '3.13' parallel: '1.3.7' - openai: '3.42.0' + openai: '3.45.0' distribution: sdist env: FI_BASE_URL: http://127.0.0.1:9 @@ -65,7 +65,13 @@ jobs: packages=(python/frameworks/{parallel,openai}/dist/*.tar.gz) fi .parallel-test-venv/bin/pip install "${packages[@]}" \ - "parallel-web==$PARALLEL_VERSION" "openai==$OPENAI_VERSION" pytest pytest-asyncio + "parallel-web==$PARALLEL_VERSION" "openai==$OPENAI_VERSION" \ + ./python/frameworks/{deepseek,fireworks,vllm} pytest pytest-asyncio + - name: Prevent source-tree imports from shadowing installed distributions + shell: bash + run: | + mv python/frameworks/openai/traceai_openai "$RUNNER_TEMP/openai-source" + mv python/frameworks/parallel/traceai_parallel "$RUNNER_TEMP/parallel-source" - name: Test with only loopback networking shell: bash run: | diff --git a/python/frameworks/openai/poetry.lock b/python/frameworks/openai/poetry.lock index ab4084324..804424c6f 100644 --- a/python/frameworks/openai/poetry.lock +++ b/python/frameworks/openai/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 2.1.2 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.2.1 and should not be changed by hand. [[package]] name = "annotated-types" @@ -199,7 +199,7 @@ description = "Backport of PEP 654 (exception groups)" optional = false python-versions = ">=3.7" groups = ["main"] -markers = "python_version < \"3.11\"" +markers = "python_version == \"3.10\"" files = [ {file = "exceptiongroup-1.3.0-py3-none-any.whl", hash = "sha256:4d111e6e0c13d0644cad6ddaa7ed0261a0b36971f6d23e7ec9b4b9097da78a10"}, {file = "exceptiongroup-1.3.0.tar.gz", hash = "sha256:b241f5885f560bc56a59ee63ca4c6a8bfa46ae4ad651af316d4e81817bb9fd88"}, @@ -213,14 +213,14 @@ test = ["pytest (>=6)"] [[package]] name = "fi-instrumentation-otel" -version = "1.1.0" -description = "OpenTelemetry instrumentation" +version = "1.2.2" +description = "OpenTelemetry tracing for AI applications: tracer setup, span helpers, exporters, and semantic conventions for Future AGI." optional = false python-versions = "<3.14,>3.9" groups = ["main"] files = [ - {file = "fi_instrumentation_otel-1.1.0-py3-none-any.whl", hash = "sha256:4fa2bf663e0ce09447aca92ae798cd7b0dee35c5c92c2e033f1d89d158a9e62c"}, - {file = "fi_instrumentation_otel-1.1.0.tar.gz", hash = "sha256:7ec324c3c9242aa87a88cf23a21c11919b8736c6451c5fb50917f592a853b059"}, + {file = "fi_instrumentation_otel-1.2.2-py3-none-any.whl", hash = "sha256:25cf88f93e33aa928d4fae0f631afe8a32565639490862b715869e0633becbaf"}, + {file = "fi_instrumentation_otel-1.2.2.tar.gz", hash = "sha256:b675df47d7a03c506fbd2e4b0420fe4f74d12a6353a88b219bfd97797b276d4c"}, ] [package.dependencies] @@ -1171,5 +1171,5 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.1" -python-versions = ">3.9,<3.14" -content-hash = "0acc5e8ccc1ccab2cf75b5046531d39cacfe7bfc4c63f864c0ebee76e32e06ba" +python-versions = ">=3.10,<3.14" +content-hash = "8c587bffbe3bfb7585a531edb33313b357c386ee6daa19b3b7812957b1013029" diff --git a/python/frameworks/openai/pyproject.toml b/python/frameworks/openai/pyproject.toml index 3a189f628..4e1d15f91 100644 --- a/python/frameworks/openai/pyproject.toml +++ b/python/frameworks/openai/pyproject.toml @@ -15,7 +15,6 @@ classifiers = [ "Programming Language :: Python", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3 :: Only", - "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", @@ -27,8 +26,8 @@ classifiers = [ ] [tool.poetry.dependencies] -python = ">3.9,<3.14" -fi-instrumentation-otel = ">=0.1.11" +python = ">=3.10,<3.14" +fi-instrumentation-otel = ">=1.2.2" openai = ">=1.69.0" [tool.poetry.urls] diff --git a/python/frameworks/openai/tests/test_core_dependency_privacy.py b/python/frameworks/openai/tests/test_core_dependency_privacy.py new file mode 100644 index 000000000..160b0a30b --- /dev/null +++ b/python/frameworks/openai/tests/test_core_dependency_privacy.py @@ -0,0 +1,42 @@ +"""Synthetic fixture to verify declared core compatibility and hidden error details.""" +import asyncio +import httpx +import openai +import pytest +from fi_instrumentation import TraceConfig, register +from fi_instrumentation.fi_types import ProjectType +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +from traceai_openai import OpenAIInstrumentor + +MESSAGE = 'synthetic-private-upstream-error-483129' + +@pytest.mark.parametrize('async_', [False, True]) +def test_hidden_failed_response(async_, monkeypatch): + monkeypatch.setenv('FI_HIDE_ERROR_DETAILS', 'true') + exporter = InMemorySpanExporter() + provider = register(project_name='core-floor-probe', project_type=ProjectType.OBSERVE, + span_exporter=exporter, batch=False, verbose=False) + instrumentor = OpenAIInstrumentor() + instrumentor.instrument(tracer_provider=provider, config=TraceConfig(hide_inputs=True)) + body = {'id': 'resp_probe', 'object': 'response', 'created_at': 1, 'model': 'synthetic', + 'status': 'failed', 'output': [], 'error': {'code': 'server_error', 'message': MESSAGE}, + 'usage': {'input_tokens': 1, 'output_tokens': 0, 'total_tokens': 1}} + def handler(req): return httpx.Response(200, json=body) + def make_client(): + http = (httpx.AsyncClient if async_ else httpx.Client)(transport=httpx.MockTransport(handler), trust_env=False) + return (openai.AsyncOpenAI if async_ else openai.OpenAI)(api_key='synthetic-key', base_url='https://api.parallel.ai/v1', max_retries=0, http_client=http) + try: + if async_: + async def run(): + async with make_client() as client: + assert (await client.responses.create(model='synthetic', input='private input')).status == 'failed' + asyncio.run(run()) + else: + with make_client() as client: + assert client.responses.create(model='synthetic', input='private input').status == 'failed' + spans = exporter.get_finished_spans() + assert len(spans) == 1 + assert MESSAGE not in spans[0].to_json() + finally: + instrumentor.uninstrument() + provider.shutdown() diff --git a/python/frameworks/openai/tests/test_deepseek_raw_response.py b/python/frameworks/openai/tests/test_deepseek_raw_response.py index ccddb4f1e..7aa4b7dbe 100644 --- a/python/frameworks/openai/tests/test_deepseek_raw_response.py +++ b/python/frameworks/openai/tests/test_deepseek_raw_response.py @@ -134,9 +134,9 @@ async def run(): assert b"PRIVATE_STREAM_ERROR" not in spans[0].SerializeToString() -@pytest.mark.parametrize("asynchronous", [False, True]) +@pytest.mark.parametrize("asynchronous,close_method", [(False, "close"), (True, "close"), (True, "aclose")]) @pytest.mark.parametrize("parse_first", [False, True]) -def test_raw_stream_close_before_exhaustion(wire, asynchronous, parse_first): +def test_raw_stream_close_before_exhaustion(wire, asynchronous, parse_first, close_method): spans = wire("shared") params = dict( model="deepseek-reasoner", @@ -155,8 +155,8 @@ async def run(): if parse_first: stream = raw.parse() await stream.__anext__() - await raw.aclose() - await raw.aclose() + await getattr(raw, close_method)() + await getattr(raw, close_method)() if asynchronous: asyncio.run(run()) diff --git a/python/frameworks/openai/tests/test_deepseek_route.py b/python/frameworks/openai/tests/test_deepseek_route.py index 2c786aed0..364170f29 100644 --- a/python/frameworks/openai/tests/test_deepseek_route.py +++ b/python/frameworks/openai/tests/test_deepseek_route.py @@ -555,7 +555,8 @@ def respond(request): if streaming: assert a["output.value"] == ANSWER assert a["gen_ai.output.messages.0.message.content"] == ANSWER - assert spans[0].status.code == (2 if event_type == "response.failed" else 1) + # A terminal incomplete response is not a successful generation either. + assert spans[0].status.code == (1 if event_type == "response.completed" else 2) assert b"PRIVATE_VENDOR_ERROR" not in spans[0].SerializeToString() diff --git a/python/frameworks/openai/tests/test_effective_body_and_raw_consumption.py b/python/frameworks/openai/tests/test_effective_body_and_raw_consumption.py new file mode 100644 index 000000000..74a5c053d --- /dev/null +++ b/python/frameworks/openai/tests/test_effective_body_and_raw_consumption.py @@ -0,0 +1,260 @@ +"""Effective credentials and lazy body lifecycles through real SDK transports.""" + +import asyncio +from copy import deepcopy +import json +from types import SimpleNamespace + +try: + import httpx2 as httpx +except ImportError: + import httpx +import openai +import pytest +from fi_instrumentation import TraceConfig, register +from fi_instrumentation.fi_types import ProjectType +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +from opentelemetry.trace import StatusCode +from traceai_openai import OpenAIInstrumentor +from traceai_openai._privacy import request_credentials + + +@pytest.fixture +def tracing(): + exporter = InMemorySpanExporter() + provider = register( + project_name="body-consumption", + project_type=ProjectType.OBSERVE, + span_exporter=exporter, + batch=False, + verbose=False, + ) + OpenAIInstrumentor().instrument(tracer_provider=provider, config=TraceConfig()) + try: + yield provider, exporter + finally: + OpenAIInstrumentor().uninstrument() + provider.force_flush() + provider.shutdown() + + +def sdk(handler, async_=False): + http = (httpx.AsyncClient if async_ else httpx.Client)( + transport=httpx.MockTransport(handler), + trust_env=False, + ) + return (openai.AsyncOpenAI if async_ else openai.OpenAI)( + api_key="synthetic-static-key", + max_retries=0, + http_client=http, + ) + + +def response_body(status="completed"): + return { + "id": "resp_review", + "object": "response", + "created_at": 1, + "model": "synthetic", + "status": status, + "parallel_tool_calls": True, + "tool_choice": "auto", + "tools": [], + "output": [], + "error": None, + "usage": { + "input_tokens": 5, + "output_tokens": 2, + "total_tokens": 7, + "input_tokens_details": {"cached_tokens": 0}, + "output_tokens_details": {"reasoning_tokens": 0}, + }, + } + + +@pytest.mark.parametrize("async_", [False, True]) +@pytest.mark.parametrize("extra", [False, True]) +@pytest.mark.parametrize("credential", ["authorization", "header"]) +def test_mcp_credentials_in_effective_body(tracing, async_, extra, credential): + _, exporter = tracing + secret = 'synthetic-extra-mcp-credential-17498-"quoted"' + tool = { + "type": "mcp", + "server_label": "synthetic", + "server_url": "https://synthetic.invalid/mcp", + **( + {"authorization": secret} + if credential == "authorization" + else {"headers": {"X-Private-Token": secret}} + ), + } + original_tool = deepcopy(tool) + captured = [] + + def handler(request): + captured.append(json.loads(request.content)) + assert captured[-1]["tools"][0] == tool + return httpx.Response( + 400, + json={ + "error": { + "code": "invalid_request", + "message": "MCP authentication failed for " + secret, + } + }, + ) + + params = {"model": "synthetic", "input": "question"} + params.update( + {"tools": [], "extra_body": {"tools": [tool]}} if extra else {"tools": [tool]} + ) + + async def run(): + async with sdk(handler, True) as client: + await client.responses.create(**params) + + with pytest.raises(openai.BadRequestError) as caught: + if async_: + asyncio.run(run()) + else: + with sdk(handler) as client: + client.responses.create(**params) + assert captured and tool == original_tool + assert secret in caught.value.body["message"] + (span,) = exporter.get_finished_spans() + assert span.status.status_code is StatusCode.ERROR + assert "synthetic-extra-mcp-credential" not in span.to_json() + if extra: + assert "synthetic.invalid/mcp" not in span.to_json() + + +@pytest.mark.parametrize("override", [None, [], openai.Omit()]) +def test_effective_tools_override_original_without_mutation(override): + original = {"tools": [{"type": "mcp", "authorization": "synthetic-original"}]} + extra = {"tools": override} + secrets = request_credentials(None, SimpleNamespace(extra_json=extra), original) + assert "synthetic-original" not in secrets + assert original["tools"][0]["authorization"] == "synthetic-original" + assert extra["tools"] is override + + +@pytest.mark.parametrize("async_", [False, True]) +@pytest.mark.parametrize("mode", ["parse", "json", "text", "read"]) +@pytest.mark.parametrize("status", ["completed", "failed", "incomplete", "cancelled"]) +def test_raw_body_consumption(tracing, async_, mode, status): + _, exporter = tracing + body = response_body(status) + + def handler(request): + return httpx.Response(200, json=body) + + def check(result): + if mode == "parse": + assert result.status == status + elif mode == "json": + assert result == body + else: + assert json.loads(result) == body + + async def run(): + async with sdk(handler, True) as client: + async with client.responses.with_streaming_response.create( + model="synthetic", + input="question", + ) as raw: + check(await getattr(raw, mode)()) + check(await getattr(raw, mode)()) + await raw.close() + + if async_: + asyncio.run(run()) + else: + with sdk(handler) as client: + with client.responses.with_streaming_response.create( + model="synthetic", + input="question", + ) as raw: + check(getattr(raw, mode)()) + check(getattr(raw, mode)()) + raw.close() + (span,) = exporter.get_finished_spans() + assert span.status.status_code is ( + StatusCode.OK if status == "completed" else StatusCode.ERROR + ) + + +@pytest.mark.parametrize("async_", [False, True]) +@pytest.mark.parametrize("mode", ["read", "text", "json"]) +@pytest.mark.parametrize("outcome", ["completed", "unread", "error", "cancel", "sse"]) +def test_raw_consumption_is_lazy_and_errors_finish_once(tracing, async_, mode, outcome): + _, exporter = tracing + reads = [] + closes = [] + error = ( + KeyboardInterrupt("synthetic abort") + if outcome == "cancel" + else RuntimeError("synthetic read failure") + ) + body = json.dumps(response_body()).encode() + + class SyncBody(httpx.SyncByteStream): + def __iter__(self): + reads.append(1) + if outcome in {"error", "cancel"}: + raise error + yield body + + def close(self): + closes.append(1) + + class AsyncBody(httpx.AsyncByteStream): + async def __aiter__(self): + reads.append(1) + if outcome in {"error", "cancel"}: + raise error + yield body + + async def aclose(self): + closes.append(1) + + def handler(request): + return httpx.Response( + 200, + headers={"content-type": "application/json"}, + stream=AsyncBody() if async_ else SyncBody(), + ) + + params = {"model": "synthetic", "input": "question", "stream": outcome == "sse"} + + async def run(): + async with sdk(handler, True) as client: + async with client.responses.with_streaming_response.create(**params) as raw: + assert not reads + assert not exporter.get_finished_spans() + if outcome != "unread": + await getattr(raw, mode)() + await raw.close() + + def run_sync(): + with sdk(handler) as client: + with client.responses.with_streaming_response.create(**params) as raw: + assert not reads + assert not exporter.get_finished_spans() + if outcome != "unread": + getattr(raw, mode)() + raw.close() + + if outcome in {"error", "cancel"}: + with pytest.raises(type(error)) as caught: + asyncio.run(run()) if async_ else run_sync() + assert caught.value is error + else: + asyncio.run(run()) if async_ else run_sync() + assert len(reads) == (0 if outcome == "unread" else 1) + assert closes + (span,) = exporter.get_finished_spans() + assert span.status.status_code is ( + StatusCode.OK if outcome == "completed" else StatusCode.ERROR + ) + if outcome in {"error", "cancel"}: + assert len([event for event in span.events if event.name == "exception"]) == 1 diff --git a/python/frameworks/openai/tests/test_release_qualification.py b/python/frameworks/openai/tests/test_release_qualification.py new file mode 100644 index 000000000..f5d620ba3 --- /dev/null +++ b/python/frameworks/openai/tests/test_release_qualification.py @@ -0,0 +1,68 @@ +"""Independent credential-URL checks with real SDKs and synthetic transports.""" + +import asyncio +import copy +import json + +import httpx +import openai +import pytest + +from .test_responses_safety import ( # noqa: F401 - fixture dependencies + openai_tracing, + response_body, + sdk_client, + tracing, +) + + +@pytest.mark.parametrize("async_", [False, True]) +@pytest.mark.parametrize("outcome", ["response", "error"]) +@pytest.mark.parametrize("extra", [False, True]) +def test_mcp_url_credentials_are_telemetry_only(openai_tracing, async_, outcome, extra): + _, exporter = openai_tracing + secrets = ["synthetic-url-user", "synthetic-url-password", "synthetic-query-token", "synthetic-fragment-token"] + url = f"https://{secrets[0]}:{secrets[1]}@synthetic.invalid/mcp?access_token={secrets[2]}#{secrets[3]}" + tool = {"type": "mcp", "server_label": "synthetic", "server_url": url, + "authorization": "synthetic-mcp-authorization", + "headers": {"X-Custom-Authentication": "synthetic-mcp-header"}} + original = copy.deepcopy(tool) + + def handler(request): + assert json.loads(request.content)["tools"] == [original] + if outcome == "error": + return httpx.Response(400, json={"error": {"message": "MCP rejected " + url}}) + body = response_body() + body["tools"] = [original] + return httpx.Response(200, json=body) + + params = {"model": "parallel", "input": "safe question"} + params.update({"extra_body": {"tools": [tool]}} if extra else {"tools": [tool]}) + + def check(response): + assert response.tools[0].server_url == url + assert response.tools[0].authorization == original["authorization"] + + async def run(): + async with sdk_client(handler, True) as client: + if outcome == "error": + with pytest.raises(openai.BadRequestError) as caught: + await client.responses.create(**params) + assert url in caught.value.body["message"] + else: + check(await client.responses.create(**params)) + + if async_: + asyncio.run(run()) + else: + with sdk_client(handler) as client: + if outcome == "error": + with pytest.raises(openai.BadRequestError) as caught: + client.responses.create(**params) + assert url in caught.value.body["message"] + else: + check(client.responses.create(**params)) + assert tool == original + span, = exporter.get_finished_spans() + for secret in [*secrets, tool["authorization"], *tool["headers"].values()]: + assert secret not in span.to_json() diff --git a/python/frameworks/openai/tests/test_response_edge_cases.py b/python/frameworks/openai/tests/test_response_edge_cases.py new file mode 100644 index 000000000..de31ae8ed --- /dev/null +++ b/python/frameworks/openai/tests/test_response_edge_cases.py @@ -0,0 +1,185 @@ +"""Round 3 independent real SDK probes; synthetic credentials and MockTransport only.""" +import asyncio +import json +import httpx +import openai +from typing import Any, Dict +import pytest +from fi_instrumentation import TraceConfig, register +from fi_instrumentation.fi_types import ProjectType +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +from opentelemetry.trace import StatusCode +from traceai_openai import OpenAIInstrumentor + +SECRET_INPUT = 'synthetic-private-system-instruction-827391' + +def body(status='completed'): + return {'id': 'resp_independent', 'object': 'response', 'created_at': 1, + 'model': 'parallel', 'status': status, 'parallel_tool_calls': True, + 'tool_choice': 'auto', 'tools': [], 'instructions': SECRET_INPUT, + 'output': [{'id': 'msg_1', 'type': 'message', 'role': 'assistant', + 'status': 'completed', 'content': [{'type': 'output_text', + 'text': 'safe public answer', 'annotations': []}]}], + 'error': None, 'usage': {'input_tokens': 5, 'output_tokens': 2, + 'total_tokens': 7, 'input_tokens_details': {'cached_tokens': 0}, + 'output_tokens_details': {'reasoning_tokens': 0}}} + +@pytest.fixture +def tracing(): + exporter = InMemorySpanExporter() + provider = register(project_name='review3', project_type=ProjectType.OBSERVE, + span_exporter=exporter, batch=False, verbose=False) + OpenAIInstrumentor().instrument(tracer_provider=provider, config=TraceConfig(hide_inputs=True)) + try: + yield provider, exporter + finally: + OpenAIInstrumentor().uninstrument() + provider.force_flush() + provider.shutdown() + +def client(handler, async_=False, base_url='https://api.parallel.ai/v1'): + http = (httpx.AsyncClient if async_ else httpx.Client)( + transport=httpx.MockTransport(handler), trust_env=False) + return (openai.AsyncOpenAI if async_ else openai.OpenAI)( + api_key='synthetic-api-key-938472', base_url=base_url, + max_retries=0, http_client=http) + +@pytest.mark.parametrize('async_', [False, True]) +@pytest.mark.parametrize('to_dict', [False, True]) +def test_explicit_dict_parse_keeps_inputs_hidden(tracing, async_, to_dict): + provider, exporter = tracing + def handler(request): + return httpx.Response(200, json=body()) + async def run(): + async with client(handler, True) as c: + async with c.responses.with_streaming_response.create( + model='parallel', instructions=SECRET_INPUT, input='question') as raw: + result = await raw.parse(**({'to': Dict[str, Any] if openai.__version__.startswith('1.') else dict} if to_dict else {})) + assert (result['instructions'] if to_dict else result.instructions) == SECRET_INPUT + if async_: + asyncio.run(run()) + else: + with client(handler) as c: + with c.responses.with_streaming_response.create( + model='parallel', instructions=SECRET_INPUT, input='question') as raw: + result = raw.parse(**({'to': Dict[str, Any] if openai.__version__.startswith('1.') else dict} if to_dict else {})) + assert (result['instructions'] if to_dict else result.instructions) == SECRET_INPUT + provider.force_flush() + (span,) = exporter.get_finished_spans() + assert 'safe public answer' in span.attributes['output.value'] + leaked = {k: v for k, v in span.attributes.items() if SECRET_INPUT in str(v)} + assert not leaked, leaked + +@pytest.mark.parametrize('async_', [False, True]) +@pytest.mark.parametrize('method', ['parse', 'json', 'read', 'text', 'iter_lines']) +def test_successful_raw_body_consumption_is_not_error(tracing, async_, method): + provider, exporter = tracing + payload = json.dumps(body()).encode() + class SyncBody(httpx.SyncByteStream): + def __iter__(self): + yield payload + class AsyncBody(httpx.AsyncByteStream): + async def __aiter__(self): + yield payload + def handler(request): + return httpx.Response(200, headers={'content-type':'application/json'}, + stream=AsyncBody() if async_ else SyncBody()) + async def run(): + async with client(handler, True) as c: + async with c.responses.with_streaming_response.create(model='parallel', input='question') as raw: + if method == 'iter_lines': + result = [line async for line in raw.iter_lines()] + else: + result = await getattr(raw, method)() + assert result + if async_: + asyncio.run(run()) + else: + with client(handler) as c: + with c.responses.with_streaming_response.create(model='parallel', input='question') as raw: + result = list(raw.iter_lines()) if method == 'iter_lines' else getattr(raw, method)() + assert result + provider.force_flush() + (span,) = exporter.get_finished_spans() + assert span.status.status_code == StatusCode.OK, span.to_json() + +@pytest.mark.parametrize('async_', [False, True]) +@pytest.mark.parametrize('instrumented', [False, True]) +def test_raw_malformed_body_remains_inspectable(tracing, async_, instrumented): + provider, exporter = tracing + if not instrumented: + OpenAIInstrumentor().uninstrument() + def handler(request): + return httpx.Response(200, headers={'content-type': 'application/json'}, content=b'{"truncated":') + async def run(): + async with client(handler, True) as c: + raw = await c.responses.with_raw_response.create(model='parallel', input='question') + assert raw.status_code == 200 + assert raw.content == b'{"truncated":' + with pytest.raises(json.JSONDecodeError): + raw.parse() + if async_: + asyncio.run(run()) + else: + with client(handler) as c: + raw = c.responses.with_raw_response.create(model='parallel', input='question') + assert raw.status_code == 200 + assert raw.content == b'{"truncated":' + with pytest.raises(json.JSONDecodeError): + raw.parse() + +@pytest.mark.parametrize('async_', [False, True]) +def test_speech_stream_to_file_not_marked_failed(tracing, async_, tmp_path): + from pathlib import Path + provider, exporter = tracing + payload = b'synthetic-audio-data' + target = tmp_path / ('audio-async.bin' if async_ else 'audio-sync.bin') + class SyncBody(httpx.SyncByteStream): + def __iter__(self): + yield payload + class AsyncBody(httpx.AsyncByteStream): + async def __aiter__(self): + yield payload + def handler(request): + return httpx.Response(200, headers={'content-type':'audio/mpeg'}, + stream=AsyncBody() if async_ else SyncBody()) + async def run(): + async with client(handler, True, base_url='https://api.openai.com/v1') as c: + async with c.audio.speech.with_streaming_response.create( + model='tts-1', voice='alloy', input='Hello') as raw: + await raw.stream_to_file(target) + if async_: + asyncio.run(run()) + else: + with client(handler, base_url='https://api.openai.com/v1') as c: + with c.audio.speech.with_streaming_response.create( + model='tts-1', voice='alloy', input='Hello') as raw: + raw.stream_to_file(target) + assert target.read_bytes() == payload + provider.force_flush() + (span,) = exporter.get_finished_spans() + assert span.status.status_code == StatusCode.OK, span.to_json() + + +@pytest.mark.parametrize('async_', [False, True]) +def test_early_aborted_raw_iterator_is_error(tracing, async_): + provider, exporter = tracing + def handler(request): + return httpx.Response(200, headers={'content-type': 'text/plain'}, content=b'line one\nline two\n') + async def run(): + async with client(handler, True) as c: + async with c.responses.with_streaming_response.create(model='parallel', input='question') as raw: + iterator = raw.iter_lines() + assert await iterator.__anext__() == 'line one' + await iterator.aclose() + if async_: + asyncio.run(run()) + else: + with client(handler) as c: + with c.responses.with_streaming_response.create(model='parallel', input='question') as raw: + iterator = raw.iter_lines() + assert next(iterator) == 'line one' + iterator.close() + provider.force_flush() + (span,) = exporter.get_finished_spans() + assert span.status.status_code == StatusCode.ERROR diff --git a/python/frameworks/openai/tests/test_response_envelopes.py b/python/frameworks/openai/tests/test_response_envelopes.py new file mode 100644 index 000000000..593bd62c6 --- /dev/null +++ b/python/frameworks/openai/tests/test_response_envelopes.py @@ -0,0 +1,455 @@ +"""Independent review probes; all credentials and transports are synthetic.""" + +import asyncio +import json + +import httpx +import openai +import pytest +from fi_instrumentation import register +from fi_instrumentation.fi_types import ProjectType +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +from opentelemetry.trace import StatusCode +from traceai_openai import OpenAIInstrumentor + + +@pytest.fixture +def tracing(): + exporter = InMemorySpanExporter() + provider = register( + project_name="review-edges", + project_type=ProjectType.OBSERVE, + span_exporter=exporter, + batch=False, + verbose=False, + ) + OpenAIInstrumentor().instrument(tracer_provider=provider) + try: + yield provider, exporter + finally: + OpenAIInstrumentor().uninstrument() + provider.force_flush() + provider.shutdown() + + +def client(handler, async_=False, key="synthetic-static-key"): + http = (httpx.AsyncClient if async_ else httpx.Client)( + transport=httpx.MockTransport(handler), trust_env=False + ) + return (openai.AsyncOpenAI if async_ else openai.OpenAI)( + api_key=key, + base_url="https://api.parallel.ai/v1", + max_retries=0, + http_client=http, + ) + + +def body(status="completed"): + return { + "id": "resp_edge", + "object": "response", + "created_at": 1, + "model": "parallel", + "status": status, + "parallel_tool_calls": True, + "tool_choice": "auto", + "tools": [], + "output": [], + "error": None, + "usage": { + "input_tokens": 5, + "output_tokens": 2, + "total_tokens": 7, + "input_tokens_details": {"cached_tokens": 0}, + "output_tokens_details": {"reasoning_tokens": 0}, + }, + } + + +def sse(status): + response = body(status) + if status == "failed": + response["error"] = {"code": "server_error", "message": "synthetic failure"} + kind = "response.created" if status == "in_progress" else "response." + status + return ( + "event: " + + kind + + "\ndata: " + + json.dumps({"type": kind, "sequence_number": 1, "response": response}) + + "\n\n" + ).encode() + + +def test_concurrent_callable_nonstream_credentials(tracing): + if int(openai.__version__.split(".")[0]) < 3: + pytest.skip("callable auth not available at floor") + _provider, exporter = tracing + keys = ["synthetic-alpha-credential-198273", "synthetic-bravo-credential-543215"] + issued = [] + + async def run(): + received = [] + both_sent = asyncio.Event() + + async def token(): + key = keys[len(issued)] + issued.append(key) + return key + + async def handler(request): + key = request.headers["authorization"][7:] + received.append(key) + if len(received) == 2: + both_sent.set() + await asyncio.wait_for(both_sent.wait(), 3) + response = body("failed") + response["error"] = { + "code": "server_error", + "message": "Rejected token " + key, + } + return httpx.Response(200, json=response) + + async with client(handler, True, token) as c: + responses = await asyncio.gather( + *[c.responses.create(model="parallel", input="question") for _ in keys] + ) + assert set(received) == set(keys) + assert {r.error.message for r in responses} == { + "Rejected token " + k for k in keys + } + + asyncio.run(run()) + assert issued == keys + spans = exporter.get_finished_spans() + assert len(spans) == 2 + assert all(s.status.status_code == StatusCode.ERROR for s in spans) + leaked = { + key: [ + name + for span in spans + for name, value in span.attributes.items() + if key in str(value) + ] + for key in keys + } + assert not any(leaked.values()), leaked + + +@pytest.mark.parametrize("async_", [False, True]) +@pytest.mark.parametrize("status", ["failed", "completed", "in_progress"]) +@pytest.mark.parametrize("exhaust", [False, True]) +def test_streaming_response_envelope_lifecycle(tracing, async_, status, exhaust): + provider, exporter = tracing + + def handler(request): + return httpx.Response( + 200, headers={"content-type": "text/event-stream"}, content=sse(status) + ) + + async def run(): + async with client(handler, True) as c: + async with c.responses.with_streaming_response.create( + model="parallel", input="question", stream=True + ) as raw: + stream = await raw.parse() + if exhaust: + events = [chunk async for chunk in stream] + assert events[-1].response.status == status + else: + assert (await stream.__anext__()).response.status == status + + if async_: + asyncio.run(run()) + else: + with client(handler) as c: + with c.responses.with_streaming_response.create( + model="parallel", input="question", stream=True + ) as raw: + stream = raw.parse() + if exhaust: + events = list(stream) + assert events[-1].response.status == status + else: + assert next(stream).response.status == status + provider.force_flush() + spans = exporter.get_finished_spans() + assert len(spans) == 1, f"Expected one finalized span, got {len(spans)}" + (span,) = spans + assert span.status.status_code == ( + StatusCode.OK if status == "completed" else StatusCode.ERROR + ), span.to_json() + if status != "in_progress": + assert span.attributes["gen_ai.usage.total_tokens"] == 7, span.to_json() + + +@pytest.mark.parametrize("async_", [False, True]) +@pytest.mark.parametrize("instrumented", [False, True]) +def test_raw_explicit_stream_parse(tracing, async_, instrumented): + provider, exporter = tracing + if not instrumented: + OpenAIInstrumentor().uninstrument() + from openai.types.responses import ResponseStreamEvent + + def handler(request): + return httpx.Response( + 200, headers={"content-type": "text/event-stream"}, content=sse("failed") + ) + + async def run(): + async with client(handler, True) as c: + raw = await c.responses.with_raw_response.create( + model="parallel", input="question", stream=True + ) + stream = raw.parse(to=openai.AsyncStream[ResponseStreamEvent]) + assert raw.parse(to=openai.AsyncStream[ResponseStreamEvent]) is stream + events = [event async for event in stream] + assert events[-1].response.status == "failed" + await stream.close() + + if async_: + asyncio.run(run()) + else: + with client(handler) as c: + raw = c.responses.with_raw_response.create( + model="parallel", input="question", stream=True + ) + stream = raw.parse(to=openai.Stream[ResponseStreamEvent]) + assert raw.parse(to=openai.Stream[ResponseStreamEvent]) is stream + events = list(stream) + assert events[-1].response.status == "failed" + stream.close() + provider.force_flush() + spans = exporter.get_finished_spans() + if instrumented: + assert len(spans) == 1, f"Expected one finalized span, got {len(spans)}" + assert spans[0].status.status_code is StatusCode.ERROR + assert spans[0].attributes["gen_ai.usage.total_tokens"] == 7 + else: + assert not spans + + +@pytest.mark.parametrize("async_", [False, True]) +@pytest.mark.parametrize("streaming", [False, True]) +@pytest.mark.parametrize("parse", [False, True]) +def test_lazy_envelope_does_not_read_until_requested(tracing, async_, streaming, parse): + provider, exporter = tracing + reads = [] + closes = [] + payload = sse("completed") if streaming else json.dumps(body()).encode() + + class SyncBody(httpx.SyncByteStream): + def __iter__(self): + reads.append("read") + yield payload + + def close(self): + closes.append("close") + + class AsyncBody(httpx.AsyncByteStream): + async def __aiter__(self): + reads.append("read") + yield payload + + async def aclose(self): + closes.append("close") + + def handler(request): + return httpx.Response( + 200, + headers={ + "content-type": "text/event-stream" if streaming else "application/json" + }, + stream=AsyncBody() if async_ else SyncBody(), + ) + + async def run(): + async with client(handler, True) as c: + async with c.responses.with_streaming_response.create( + model="parallel", input="question", stream=streaming + ) as raw: + assert isinstance(raw, openai.AsyncAPIResponse) + assert not reads + assert not exporter.get_finished_spans() + if parse: + result = await raw.parse() + assert await raw.parse() is result + if streaming: + assert not reads + assert (await result.__anext__()).response.status == "completed" + else: + assert result.status == "completed" + + if async_: + asyncio.run(run()) + else: + with client(handler) as c: + with c.responses.with_streaming_response.create( + model="parallel", input="question", stream=streaming + ) as raw: + assert isinstance(raw, openai.APIResponse) + assert not reads + assert not exporter.get_finished_spans() + if parse: + result = raw.parse() + assert raw.parse() is result + if streaming: + assert not reads + assert next(result).response.status == "completed" + else: + assert result.status == "completed" + assert reads == (["read"] if parse else []) + assert closes == ["close"] + provider.force_flush() + (span,) = exporter.get_finished_spans() + assert span.status.status_code == (StatusCode.OK if parse else StatusCode.ERROR) + if parse: + assert span.attributes["gen_ai.usage.total_tokens"] == 7 + + +@pytest.mark.parametrize("async_", [False, True]) +@pytest.mark.parametrize("modern", [False, True]) +@pytest.mark.parametrize("close_http", [False, True]) +def test_explicit_stream_cache_and_envelope_close(tracing, async_, modern, close_http): + provider, exporter = tracing + from openai.types.responses import ResponseStreamEvent + + def handler(request): + return httpx.Response( + 200, headers={"content-type": "text/event-stream"}, content=sse("completed") + ) + + async def consume(raw): + if modern: + default = await raw.parse() + typed = await raw.parse(to=openai.AsyncStream[ResponseStreamEvent]) + assert await raw.parse(to=openai.AsyncStream[ResponseStreamEvent]) is typed + assert await raw.parse() is default + else: + default = raw.parse() + typed = raw.parse(to=openai.AsyncStream[ResponseStreamEvent]) + assert raw.parse(to=openai.AsyncStream[ResponseStreamEvent]) is typed + assert raw.parse() is default + assert typed is not default + assert (await typed.__anext__()).response.status == "completed" + assert not exporter.get_finished_spans() + if close_http: + await raw.http_response.aclose() + elif modern: + await raw.close() + else: + await typed.close() + # Both cache entries and every close path share one terminal result. + await default.close() + await typed.close() + + async def run(): + async with client(handler, True) as c: + if modern: + async with c.responses.with_streaming_response.create( + model="parallel", input="question", stream=True + ) as raw: + await consume(raw) + else: + raw = await c.responses.with_raw_response.create( + model="parallel", input="question", stream=True + ) + await consume(raw) + + def consume_sync(raw): + default = raw.parse() + typed = raw.parse(to=openai.Stream[ResponseStreamEvent]) + assert raw.parse(to=openai.Stream[ResponseStreamEvent]) is typed + assert raw.parse() is default + assert typed is not default + assert next(typed).response.status == "completed" + assert not exporter.get_finished_spans() + if close_http: + raw.http_response.close() + elif modern: + raw.close() + else: + typed.close() + default.close() + typed.close() + + if async_: + asyncio.run(run()) + else: + with client(handler) as c: + if modern: + with c.responses.with_streaming_response.create( + model="parallel", input="question", stream=True + ) as raw: + consume_sync(raw) + else: + raw = c.responses.with_raw_response.create( + model="parallel", input="question", stream=True + ) + consume_sync(raw) + provider.force_flush() + (span,) = exporter.get_finished_spans() + assert span.status.status_code is StatusCode.OK + assert span.attributes["gen_ai.usage.total_tokens"] == 7 + + +@pytest.mark.parametrize("cancel", [False, True]) +def test_async_lazy_parse_failure_finishes_once(tracing, cancel): + provider, exporter = tracing + entered = None + + class BrokenBody(httpx.AsyncByteStream): + async def __aiter__(self): + entered.set() + if cancel: + await asyncio.Event().wait() + raise httpx.ReadError("synthetic unreadable response") + yield b"" # Make the method an async generator. + + def handler(request): + return httpx.Response( + 200, headers={"content-type": "application/json"}, stream=BrokenBody() + ) + + async def run(): + nonlocal entered + entered = asyncio.Event() + async with client(handler, True) as c: + async with c.responses.with_streaming_response.create( + model="parallel", input="question" + ) as raw: + task = asyncio.create_task(raw.parse()) + await asyncio.wait_for(entered.wait(), 3) + if cancel: + task.cancel() + with pytest.raises( + asyncio.CancelledError if cancel else httpx.ReadError + ): + await task + assert len(exporter.get_finished_spans()) == 1 + + asyncio.run(run()) + provider.force_flush() + (span,) = exporter.get_finished_spans() + assert span.status.status_code is StatusCode.ERROR + + +def test_credential_context_is_reset_and_hooks_restore(tracing): + from traceai_openai._request_context import _active_request + + original_sync = OpenAIInstrumentor()._original_process_response + original_async = OpenAIInstrumentor()._original_async_process_response + assert _active_request.get() is None + with client(lambda request: httpx.Response(200, json=body())) as c: + c.responses.create(model="parallel", input="question") + assert _active_request.get() is None + with client( + lambda request: httpx.Response( + 401, json={"error": {"message": "synthetic error"}} + ) + ) as c: + with pytest.raises(openai.AuthenticationError): + c.responses.create(model="parallel", input="question") + assert _active_request.get() is None + OpenAIInstrumentor().uninstrument() + assert openai.OpenAI._process_response is original_sync + assert openai.AsyncOpenAI._process_response is original_async diff --git a/python/frameworks/openai/tests/test_responses_privacy_and_raw_streams.py b/python/frameworks/openai/tests/test_responses_privacy_and_raw_streams.py new file mode 100644 index 000000000..f9a83ae2e --- /dev/null +++ b/python/frameworks/openai/tests/test_responses_privacy_and_raw_streams.py @@ -0,0 +1,411 @@ +"""Responses privacy and raw-envelope contracts using synthetic SDK transports.""" + +import asyncio +import inspect +import json + +import httpx +import openai +import pytest +from fi_instrumentation import TraceConfig, register +from fi_instrumentation.fi_types import ProjectType +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +from opentelemetry.trace import StatusCode +from traceai_openai import OpenAIInstrumentor + +KEY = "synthetic-review-key-not-for-export" +ERROR = "synthetic-private-tool-error" + + +@pytest.fixture +def tracing(): + exporter = InMemorySpanExporter() + provider = register( + project_name="independent-review", + project_type=ProjectType.OBSERVE, + span_exporter=exporter, + batch=False, + verbose=False, + ) + try: + yield provider, exporter + finally: + OpenAIInstrumentor().uninstrument() + provider.force_flush() + provider.shutdown() + + +def client(handler, async_=False, key=KEY): + http = (httpx.AsyncClient if async_ else httpx.Client)( + transport=httpx.MockTransport(handler), trust_env=False + ) + return (openai.AsyncOpenAI if async_ else openai.OpenAI)( + api_key=key, + base_url="https://api.parallel.ai/v1", + max_retries=0, + http_client=http, + ) + + +def body(status="completed"): + return { + "id": "resp_review", + "object": "response", + "created_at": 1, + "model": "parallel", + "status": status, + "parallel_tool_calls": True, + "tool_choice": "auto", + "tools": [], + "output": [], + "error": None, + "usage": { + "input_tokens": 5, + "output_tokens": 2, + "total_tokens": 7, + "input_tokens_details": {"cached_tokens": 0}, + "output_tokens_details": {"reasoning_tokens": 0}, + }, + } + + +def sse(kind, response): + return ( + "event: " + + kind + + "\ndata: " + + json.dumps({"type": kind, "sequence_number": 1, "response": response}) + + "\n\n" + ).encode() + + +@pytest.mark.parametrize("async_", [False, True]) +@pytest.mark.parametrize("traceback_marker", [False, True]) +def test_key_redaction_survives_core_status_cleaning(tracing, async_, traceback_marker): + provider, exporter = tracing + OpenAIInstrumentor().instrument(tracer_provider=provider) + message = "Invalid credential: " + KEY + if traceback_marker: + message += "\nTraceback (most recent call last):\n synthetic remote failure" + + def handler(request): + assert request.headers["authorization"] == "Bearer " + KEY + return httpx.Response( + 401, json={"error": {"code": "invalid_api_key", "message": message}} + ) + + async def run(): + async with client(handler, True) as c: + await c.responses.create(model="parallel", input="synthetic question") + + with pytest.raises(openai.AuthenticationError) as caught: + if async_: + asyncio.run(run()) + else: + with client(handler) as c: + c.responses.create(model="parallel", input="synthetic question") + (span,) = exporter.get_finished_spans() + assert KEY in str(caught.value) + assert KEY not in span.to_json(), span.to_json() + + +@pytest.mark.parametrize("async_", [False, True]) +def test_callable_credentials_are_scrubbed_without_extra_provider_calls( + tracing, async_ +): + if tuple(map(int, openai.__version__.split(".")[:2])) < (3, 0): + pytest.skip("callable OpenAI credentials are a latest-SDK feature") + provider, exporter = tracing + OpenAIInstrumentor().instrument(tracer_provider=provider) + calls = [] + + def token(): + calls.append(1) + return KEY + + async def atoken(): + return token() + + def handler(request): + assert request.headers["authorization"] == "Bearer " + KEY + return httpx.Response( + 401, + json={ + "error": {"code": "invalid_api_key", "message": "Invalid key: " + KEY} + }, + ) + + async def run(): + async with client(handler, True, atoken) as c: + await c.responses.create(model="parallel", input="synthetic question") + + with pytest.raises(openai.AuthenticationError) as caught: + if async_: + asyncio.run(run()) + else: + with client(handler, key=token) as c: + c.responses.create(model="parallel", input="synthetic question") + (span,) = exporter.get_finished_spans() + assert len(calls) == 1 + assert KEY in str(caught.value) + assert caught.value.request.headers["authorization"] == "Bearer " + KEY + assert KEY not in span.to_json(), span.to_json() + + +@pytest.mark.parametrize("async_", [False, True]) +@pytest.mark.parametrize("stream", [False, True]) +@pytest.mark.parametrize("hidden", [False, True]) +@pytest.mark.parametrize( + "kind", + ["http_error", "mcp_protocol_error", "mcp_tool_execution_error", "mcp_list_tools"], +) +@pytest.mark.skipif( + "hide_error_details" not in inspect.signature(TraceConfig).parameters, + reason="Error-detail masking requires a core with hide_error_details support", +) +def test_hidden_error_details_cover_mcp_failure_output( + tracing, async_, stream, hidden, kind +): + provider, exporter = tracing + OpenAIInstrumentor().instrument( + tracer_provider=provider, config=TraceConfig(hide_error_details=hidden) + ) + response = body() + response["output"] = [ + { + "id": "mcp_review", + "type": "mcp_call", + "name": "lookup", + "arguments": "{}", + "server_label": "test", + "error": {"type": "http_error", "code": 500, "message": ERROR}, + "output": "ordinary tool result", + } + ] + if kind == "mcp_list_tools": + response["output"] = [ + { + "id": "mcp_list", + "type": kind, + "server_label": "test", + "tools": [], + "error": ERROR, + } + ] + elif kind == "mcp_tool_execution_error": + response["output"][0]["error"] = {"type": kind, "content": {"text": ERROR}} + else: + response["output"][0]["error"]["type"] = kind + + def handler(request): + if stream: + return httpx.Response( + 200, + headers={"content-type": "text/event-stream"}, + content=sse("response.completed", response), + ) + return httpx.Response(200, json=response) + + async def run(): + async with client(handler, True) as c: + result = await c.responses.create( + model="parallel", input="question", stream=stream + ) + if stream: + events = [event async for event in result] + assert ERROR in events[-1].response.model_dump_json() + else: + assert ERROR in result.model_dump_json() + + if async_: + asyncio.run(run()) + else: + with client(handler) as c: + result = c.responses.create( + model="parallel", input="question", stream=stream + ) + if stream: + assert ERROR in list(result)[-1].response.model_dump_json() + else: + assert ERROR in result.model_dump_json() + (span,) = exporter.get_finished_spans() + assert (ERROR in span.to_json()) is (not hidden), span.to_json() + output = json.loads(span.attributes["output.value"])["output"][0] + if kind != "mcp_list_tools": + assert output["output"] == "ordinary tool result" + assert output["error"]["type"] == kind + if kind != "mcp_tool_execution_error": + assert output["error"]["code"] == 500 + elif hidden: + assert "error" not in output + + +@pytest.mark.parametrize("async_", [False, True]) +@pytest.mark.parametrize("status", ["failed", "incomplete", "completed", "in_progress"]) +@pytest.mark.parametrize("close_http", [False, True]) +def test_raw_stream_lifecycle_retains_sdk_contract(tracing, async_, status, close_http): + provider, exporter = tracing + OpenAIInstrumentor().instrument(tracer_provider=provider) + response = body(status) + if status == "failed": + response["error"] = {"code": "server_error", "message": "synthetic failure"} + kind = "response.created" if status == "in_progress" else "response." + status + + def handler(request): + return httpx.Response( + 200, + headers={"content-type": "text/event-stream"}, + content=sse(kind, response), + ) + + async def run(): + async with client(handler, True) as c: + raw = await c.responses.with_raw_response.create( + model="parallel", input="question", stream=True + ) + assert raw.status_code == 200 + assert raw.headers["content-type"] == "text/event-stream" + assert isinstance(raw, openai._legacy_response.LegacyAPIResponse) + assert not exporter.get_finished_spans() + stream = raw.parse() + assert raw.parse() is stream + assert raw.parse(to=openai.types.responses.Response) is stream + assert (await stream.__anext__()).response.status == status + if close_http: + await raw.http_response.aclose() + else: + await stream.close() + await stream.close() + assert raw.http_response.is_closed + + if async_: + asyncio.run(run()) + else: + with client(handler) as c: + raw = c.responses.with_raw_response.create( + model="parallel", input="question", stream=True + ) + assert raw.status_code == 200 + assert raw.headers["content-type"] == "text/event-stream" + assert isinstance(raw, openai._legacy_response.LegacyAPIResponse) + assert not exporter.get_finished_spans() + stream = raw.parse() + assert raw.parse() is stream + assert raw.parse(to=openai.types.responses.Response) is stream + assert next(stream).response.status == status + if close_http: + raw.http_response.close() + else: + stream.close() + stream.close() + assert raw.http_response.is_closed + (span,) = exporter.get_finished_spans() + assert span.status.status_code is ( + StatusCode.OK if status == "completed" else StatusCode.ERROR + ) + if status != "in_progress": + assert span.attributes["gen_ai.usage.input_tokens"] == 5 + assert span.attributes["gen_ai.usage.output_tokens"] == 2 + assert span.attributes["gen_ai.usage.total_tokens"] == 7 + + +def test_concurrent_callable_credentials_use_each_failed_request(tracing): + if tuple(map(int, openai.__version__.split(".")[:2])) < (3, 0): + pytest.skip("callable OpenAI credentials are a latest-SDK feature") + provider, exporter = tracing + OpenAIInstrumentor().instrument(tracer_provider=provider) + keys = [KEY + "-first", KEY + "-second"] + issued = [] + + async def run(): + both_sent = asyncio.Event() + received = [] + + async def token(): + value = keys[len(issued)] + issued.append(value) + return value + + async def handler(request): + credential = request.headers["authorization"][7:] + received.append(credential) + if len(received) == 2: + both_sent.set() + await asyncio.wait_for(both_sent.wait(), 3) + return httpx.Response( + 401, + json={ + "error": { + "code": "invalid_api_key", + "message": "Invalid key: " + credential, + } + }, + ) + + async with client(handler, True, token) as c: + results = await asyncio.gather( + *[c.responses.create(model="parallel", input="question") for _ in keys], + return_exceptions=True, + ) + assert set(received) == set(keys) + assert all(isinstance(result, openai.AuthenticationError) for result in results) + for result in results: + assert result.request.headers["authorization"][7:] in str(result) + + asyncio.run(run()) + assert issued == keys + spans = exporter.get_finished_spans() + assert len(spans) == 2 + for span in spans: + assert span.status.status_code is StatusCode.ERROR + assert all(key not in span.to_json() for key in keys) + + +@pytest.mark.parametrize("async_", [False, True]) +@pytest.mark.parametrize("raw", [False, True]) +def test_callable_credentials_survive_stream_creation(tracing, async_, raw): + if tuple(map(int, openai.__version__.split(".")[:2])) < (3, 0): + pytest.skip("callable OpenAI credentials are a latest-SDK feature") + provider, exporter = tracing + OpenAIInstrumentor().instrument(tracer_provider=provider) + calls = [] + + def token(): + calls.append(1) + return KEY + + async def atoken(): + return token() + + response = body("failed") + response["error"] = {"code": "server_error", "message": "Rejected " + KEY} + + def handler(request): + assert request.headers["authorization"] == "Bearer " + KEY + return httpx.Response( + 200, + headers={"content-type": "text/event-stream"}, + content=sse("response.failed", response), + ) + + async def run(): + async with client(handler, True, atoken) as c: + api = c.responses.with_raw_response if raw else c.responses + result = await api.create(model="parallel", input="question", stream=True) + stream = result.parse() if raw else result + assert KEY in (await stream.__anext__()).response.error.message + await stream.close() + + if async_: + asyncio.run(run()) + else: + with client(handler, key=token) as c: + api = c.responses.with_raw_response if raw else c.responses + result = api.create(model="parallel", input="question", stream=True) + stream = result.parse() if raw else result + assert KEY in next(stream).response.error.message + stream.close() + assert len(calls) == 1 + (span,) = exporter.get_finished_spans() + assert span.status.status_code is StatusCode.ERROR + assert KEY not in span.to_json() diff --git a/python/frameworks/openai/tests/test_responses_safety.py b/python/frameworks/openai/tests/test_responses_safety.py new file mode 100644 index 000000000..fb2d773ff --- /dev/null +++ b/python/frameworks/openai/tests/test_responses_safety.py @@ -0,0 +1,477 @@ +"""Responses privacy and lifecycle contracts with real SDKs, no vendor traffic.""" +import asyncio +import inspect +import json + +import httpx +import openai +import pytest +from fi_instrumentation import TraceConfig, register +from fi_instrumentation.fi_types import ProjectType +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +from opentelemetry.trace import StatusCode +from traceai_openai import OpenAIInstrumentor + +KEY = "synthetic-parallel-vendor-key-DO-NOT-EXPORT" +QUESTION = "synthetic private question" + +@pytest.fixture +def tracing(): + exporter = InMemorySpanExporter() + provider = register(project_name="recall-offline", project_type=ProjectType.OBSERVE, + span_exporter=exporter, batch=False, verbose=False) + try: + yield provider, exporter + finally: + provider.force_flush() + provider.shutdown() + + +@pytest.fixture +def openai_tracing(tracing): + provider, exporter = tracing + instrumentor = OpenAIInstrumentor() + instrumentor.instrument(tracer_provider=provider) + try: + yield provider, exporter + finally: + instrumentor.uninstrument() + + +def sdk_client(handler, async_=False): + http_client = (httpx.AsyncClient if async_ else httpx.Client)( + transport=httpx.MockTransport(handler), trust_env=False) + return (openai.AsyncOpenAI if async_ else openai.OpenAI)( + api_key=KEY, base_url="https://api.parallel.ai/v1", max_retries=0, http_client=http_client) + + +def response_body(status="completed"): + return { + "id": "resp_synthetic", "object": "response", "created_at": 1, "model": "parallel", + "status": status, "parallel_tool_calls": True, "tool_choice": "auto", "tools": [], + "output": [], "error": {"code": "server_error", "message": "synthetic upstream failed"} if status == "failed" else None, + "usage": {"input_tokens": 5, "output_tokens": 2, "total_tokens": 7, + "input_tokens_details": {"cached_tokens": 0}, "output_tokens_details": {"reasoning_tokens": 0}}, + } + + +def sse_event(kind, response): + return ("event: " + kind + "\ndata: " + json.dumps({"type": kind, "sequence_number": 1, "response": response}) + "\n\n").encode() + + +@pytest.mark.parametrize("async_", [False, True]) +@pytest.mark.parametrize("stream", [False, True]) +def test_failed_response_must_not_export_success(openai_tracing, async_, stream): + _, exporter = openai_tracing + def handler(request): + if stream: + return httpx.Response(200, headers={"content-type": "text/event-stream"}, + content=sse_event("response.failed", response_body("failed"))) + return httpx.Response(200, json=response_body("failed")) + async def run(): + async with sdk_client(handler, True) as client: + result = await client.responses.create(model="parallel", input=QUESTION, stream=stream) + if stream: + events = [event async for event in result] + assert events[0].type == "response.failed" + else: + assert result.status == "failed" + if async_: + asyncio.run(run()) + else: + with sdk_client(handler) as client: + result = client.responses.create(model="parallel", input=QUESTION, stream=stream) + if stream: + events = list(result) # Exhaustion is part of the lifecycle contract. + assert events[0].type == "response.failed" + else: + assert result.status == "failed" + spans = exporter.get_finished_spans() + assert len(spans) == 1 + assert spans[0].attributes["gen_ai.provider.name"] == "parallel" + assert spans[0].status.status_code is StatusCode.ERROR, spans[0].to_json() + + +@pytest.mark.parametrize("async_", [False, True]) +def test_early_stream_close_must_finish_span(openai_tracing, async_): + provider, exporter = openai_tracing + def handler(request): + return httpx.Response(200, headers={"content-type": "text/event-stream"}, + content=sse_event("response.created", response_body("in_progress")) + + sse_event("response.completed", response_body())) + async def run(): + async with sdk_client(handler, True) as client: + stream = await client.responses.create(model="parallel", input=QUESTION, stream=True) + assert (await stream.__anext__()).type == "response.created" + await stream.close() + await stream.close() + assert stream.response.is_closed + if async_: + asyncio.run(run()) + else: + with sdk_client(handler) as client: + stream = client.responses.create(model="parallel", input=QUESTION, stream=True) + assert next(stream).type == "response.created" + stream.close() + stream.close() + assert stream.response.is_closed + provider.force_flush() + spans = exporter.get_finished_spans() + assert len(spans) == 1, f"closed SDK stream exported {len(spans)} spans" + assert spans[0].status.status_code is StatusCode.ERROR + + +@pytest.mark.parametrize("stage", ["request", "stream"]) +def test_async_cancellation_must_finish_span(openai_tracing, stage): + provider, exporter = openai_tracing + async def run(): + waiting = asyncio.Event() + class WaitingStream(httpx.AsyncByteStream): + async def __aiter__(self): + yield sse_event("response.created", response_body("in_progress")) + waiting.set() + await asyncio.Event().wait() + async def handler(request): + if stage == "request": + waiting.set() + await asyncio.Event().wait() + return httpx.Response(200, headers={"content-type": "text/event-stream"}, stream=WaitingStream()) + async with sdk_client(handler, True) as client: + stream = None + async def call(): + nonlocal stream + stream = await client.responses.create(model="parallel", input=QUESTION, stream=stage == "stream") + if stage == "stream": + async for _ in stream: + pass + task = asyncio.create_task(call()) + await asyncio.wait_for(waiting.wait(), timeout=3) + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + if stream is not None: + await stream.close() + asyncio.run(run()) + provider.force_flush() + spans = exporter.get_finished_spans() + assert len(spans) == 1, f"cancelled SDK {stage} exported {len(spans)} spans" + assert spans[0].status.status_code is StatusCode.ERROR + + +@pytest.mark.parametrize("async_", [False, True]) +def test_api_key_in_error_must_be_redacted(openai_tracing, async_): + _, exporter = openai_tracing + def handler(request): + assert request.headers["authorization"] == "Bearer " + KEY + return httpx.Response(401, json={"error": {"code": "invalid_api_key", "message": "Invalid key: " + KEY}}) + async def run(): + async with sdk_client(handler, True) as client: + await client.responses.create(model="parallel", input=QUESTION) + with pytest.raises(openai.AuthenticationError) as caught: + if async_: + asyncio.run(run()) + else: + with sdk_client(handler) as client: + client.responses.create(model="parallel", input=QUESTION) + spans = exporter.get_finished_spans() + assert len(spans) == 1 + assert spans[0].status.status_code is StatusCode.ERROR + assert KEY in str(caught.value) # The caller still receives the original SDK exception. + assert KEY not in spans[0].to_json(), "vendor credential exported in error status/event" + + +@pytest.mark.parametrize("async_", [False, True]) +def test_hidden_instructions_not_reexported_in_output(tracing, async_): + provider, exporter = tracing + instruction = "synthetic confidential system instruction DO-NOT-EXPORT" + instrumentor = OpenAIInstrumentor() + instrumentor.instrument(tracer_provider=provider, config=TraceConfig(hide_inputs=True)) + def handler(request): + assert json.loads(request.content)["instructions"] == instruction + body = response_body() + body["instructions"] = instruction + body["output"] = [{"type": "message", "id": "msg_1", "role": "assistant", "status": "completed", + "content": [{"type": "output_text", "text": "safe answer", "annotations": []}]}] + return httpx.Response(200, json=body) + async def run(): + async with sdk_client(handler, True) as client: + response = await client.responses.create(model="parallel", input=QUESTION, instructions=instruction) + assert response.instructions == instruction + try: + if async_: + asyncio.run(run()) + else: + with sdk_client(handler) as client: + response = client.responses.create(model="parallel", input=QUESTION, instructions=instruction) + assert response.instructions == instruction + span, = exporter.get_finished_spans() + assert span.attributes["input.value"] == "__REDACTED__" + assert "safe answer" in span.attributes["output.value"] + assert instruction not in span.to_json(), "FI_HIDE_INPUTS instruction leaked via output.value" + finally: + instrumentor.uninstrument() + + +@pytest.mark.parametrize("async_", [False, True]) +def test_mcp_credentials_not_captured_as_tool_definitions(openai_tracing, async_): + _, exporter = openai_tracing + oauth = "synthetic-oauth-credential-DO-NOT-EXPORT" + apikey = "synthetic-mcp-api-key-DO-NOT-EXPORT" + tool = {"type": "mcp", "server_label": "synthetic", "server_url": "https://synthetic.invalid/mcp", + "require_approval": "never", "authorization": oauth, "headers": {"X-Api-Key": apikey}} + def handler(request): + parsed = json.loads(request.content) + assert parsed["tools"][0]["authorization"] == oauth + assert parsed["tools"][0]["headers"]["X-Api-Key"] == apikey + # Parallel explicitly scrubs tool credentials in its response. + body = response_body() + body["tools"] = [{**tool, "authorization": None, "headers": None}] + return httpx.Response(200, json=body) + async def run(): + async with sdk_client(handler, True) as client: + await client.responses.create(model="parallel", input=QUESTION, tools=[tool]) + if async_: + asyncio.run(run()) + else: + with sdk_client(handler) as client: + client.responses.create(model="parallel", input=QUESTION, tools=[tool]) + span, = exporter.get_finished_spans() + assert oauth not in span.to_json(), "MCP OAuth token exported in gen_ai.tool.definitions" + assert apikey not in span.to_json(), "MCP API key exported in gen_ai.tool.definitions" + + +@pytest.mark.parametrize("async_", [False, True]) +def test_sdk_responses_stream_context_exit_finishes_span(openai_tracing, async_): + provider, exporter = openai_tracing + def handler(request): + return httpx.Response(200, headers={"content-type": "text/event-stream"}, + content=sse_event("response.created", response_body("in_progress")) + + sse_event("response.completed", response_body())) + async def run(): + async with sdk_client(handler, True) as client, client.responses.stream( + model="parallel", input=QUESTION + ) as stream: + async for event in stream: + assert event.type == "response.created" + break + if async_: + asyncio.run(run()) + else: + with sdk_client(handler) as client, client.responses.stream( + model="parallel", input=QUESTION + ) as stream: + for event in stream: + assert event.type == "response.created" + break + provider.force_flush() + spans = exporter.get_finished_spans() + assert len(spans) == 1, f"SDK context exit exported {len(spans)} spans" + assert spans[0].status.status_code is StatusCode.ERROR + + + +@pytest.mark.parametrize("async_", [False, True]) +@pytest.mark.parametrize("mode", ["complete", "failed", "incomplete", "error", "eof", "empty"]) +def test_terminal_status_and_close_are_consistent(openai_tracing, async_, mode): + _, exporter = openai_tracing + status = {"complete": "completed", "failed": "failed", "incomplete": "incomplete"}.get(mode) + body = response_body(status or "in_progress") + if mode == "failed": + body["error"]["message"] = "upstream rejected " + KEY + payload = sse_event("response.created", response_body("in_progress")) + if status: + payload += sse_event("response." + status, body) + elif mode == "error": + payload += ('data: ' + json.dumps({"type": "error", "code": "server_error", + "message": "upstream rejected " + KEY, "param": None, + "sequence_number": 2}) + '\n\n').encode() + + def handler(request): + return httpx.Response(200, headers={"content-type": "text/event-stream"}, content=payload) + + async def run(): + async with sdk_client(handler, True) as client: + stream = await client.responses.create(model="parallel", input=QUESTION, stream=True) + if mode != "empty": + async for event in stream: + if status and event.type == "response." + status: + break # Terminal event received, but no EOF read yet. + await stream.close() + await stream.close() + assert stream.response.is_closed + + if async_: + asyncio.run(run()) + else: + with sdk_client(handler) as client: + stream = client.responses.create(model="parallel", input=QUESTION, stream=True) + if mode != "empty": + for event in stream: + if status and event.type == "response." + status: + break + stream.close() + stream.close() + assert stream.response.is_closed + span, = exporter.get_finished_spans() + assert span.status.status_code is (StatusCode.OK if mode == "complete" else StatusCode.ERROR) + assert KEY not in span.to_json() + if status: + assert span.attributes["gen_ai.usage.input_tokens"] == 5 + + +@pytest.mark.parametrize("async_", [False, True]) +@pytest.mark.parametrize("stream", [False, True]) +@pytest.mark.parametrize("flag", ["hide_inputs", "hide_input_text", "hide_input_messages"]) +def test_echoed_request_fields_never_become_output(tracing, async_, stream, flag): + provider, exporter = tracing + instruction = "synthetic-private-system-text" + instrumentor = OpenAIInstrumentor() + instrumentor.instrument(tracer_provider=provider, config=TraceConfig(**{flag: True})) + body = response_body() + # Empty output exercises the accumulator's JSON fallback instead of output_text. + body.update(instructions=instruction, metadata={"prompt": instruction}, + input=instruction, text={"format": {"type": "json_schema", "name": instruction}}) + + def handler(request): + if stream: + return httpx.Response(200, headers={"content-type": "text/event-stream"}, + content=sse_event("response.completed", body)) + return httpx.Response(200, json=body) + + async def run(): + async with sdk_client(handler, True) as client: + result = await client.responses.create(model="parallel", input=QUESTION, + instructions=instruction, stream=stream) + if stream: + events = [event async for event in result] + assert events[0].response.instructions == instruction + else: + assert result.instructions == instruction + + try: + if async_: + asyncio.run(run()) + else: + with sdk_client(handler) as client: + result = client.responses.create(model="parallel", input=QUESTION, + instructions=instruction, stream=stream) + if stream: + events = list(result) + assert events[0].response.instructions == instruction + else: + assert result.instructions == instruction + span, = exporter.get_finished_spans() + assert instruction not in span.to_json() + output = json.loads(span.attributes["output.value"]) + assert output["status"] == "completed" + assert output["usage"]["input_tokens"] == 5 + finally: + instrumentor.uninstrument() + + +@pytest.mark.parametrize("async_", [False, True]) +def test_raw_failed_response_retains_sdk_contract(openai_tracing, async_): + _, exporter = openai_tracing + + def handler(request): + return httpx.Response(200, json=response_body("failed")) + + async def run(): + async with sdk_client(handler, True) as client: + raw = await client.responses.with_raw_response.create(model="parallel", input=QUESTION) + assert raw.status_code == 200 + assert raw.parse().status == "failed" + + if async_: + asyncio.run(run()) + else: + with sdk_client(handler) as client: + raw = client.responses.with_raw_response.create(model="parallel", input=QUESTION) + assert raw.status_code == 200 + assert raw.parse().status == "failed" + span, = exporter.get_finished_spans() + assert span.status.status_code is StatusCode.ERROR + + +@pytest.mark.parametrize("async_", [False, True]) +@pytest.mark.parametrize("stream", [False, True]) +@pytest.mark.skipif( + "hide_error_details" not in inspect.signature(TraceConfig).parameters, + reason="Error-detail masking requires a core with hide_error_details support", +) +def test_hidden_error_details_apply_to_failed_response_json(tracing, async_, stream): + provider, exporter = tracing + instrumentor = OpenAIInstrumentor() + instrumentor.instrument(tracer_provider=provider, config=TraceConfig(hide_error_details=True)) + message = "synthetic-private-error-details" + body = response_body("failed") + body["error"]["message"] = message + + def handler(request): + if stream: + return httpx.Response(200, headers={"content-type": "text/event-stream"}, + content=sse_event("response.failed", body)) + return httpx.Response(200, json=body) + + async def run(): + async with sdk_client(handler, True) as client: + result = await client.responses.create(model="parallel", input=QUESTION, stream=stream) + if stream: + events = [event async for event in result] + assert events[0].response.error.message == message + else: + assert result.error.message == message + + try: + if async_: + asyncio.run(run()) + else: + with sdk_client(handler) as client: + result = client.responses.create(model="parallel", input=QUESTION, stream=stream) + if stream: + events = list(result) + assert events[0].response.error.message == message + else: + assert result.error.message == message + span, = exporter.get_finished_spans() + assert span.status.status_code is StatusCode.ERROR + assert message not in span.to_json() + finally: + instrumentor.uninstrument() + + +@pytest.mark.parametrize("async_", [False, True]) +def test_mcp_wire_and_function_schema_are_not_mutated(openai_tracing, async_): + _, exporter = openai_tracing + token = 'synthetic-token-with-"quote"-and-\\slash' + mcp = {"type": "mcp", "server_label": "synthetic", "server_url": "https://synthetic.invalid/mcp", + "authorization": token, "headers": {"X-Custom-Credential": token}} + function = {"type": "function", "name": "lookup", "parameters": { + "type": "object", "properties": {"authorization": {"type": "string"}}}} + tools = [mcp, function] + original = json.loads(json.dumps(tools)) + + def handler(request): + assert json.loads(request.content)["tools"] == original + body = response_body() + body["tools"] = tools # Don't rely on a provider scrubbing its response. + return httpx.Response(200, json=body) + + async def run(): + async with sdk_client(handler, True) as client: + result = await client.responses.create(model="parallel", input=QUESTION, tools=tools) + assert result.tools[0].authorization == token + + if async_: + asyncio.run(run()) + else: + with sdk_client(handler) as client: + result = client.responses.create(model="parallel", input=QUESTION, tools=tools) + assert result.tools[0].authorization == token + assert tools == original + span, = exporter.get_finished_spans() + assert "synthetic-token-with" not in span.to_json() + definitions = json.loads(span.attributes["gen_ai.tool.definitions"]) + assert "authorization" not in definitions[0] + assert "headers" not in definitions[0] + assert definitions[1] == function diff --git a/python/frameworks/openai/traceai_openai/__init__.py b/python/frameworks/openai/traceai_openai/__init__.py index 48686b521..73110e68d 100644 --- a/python/frameworks/openai/traceai_openai/__init__.py +++ b/python/frameworks/openai/traceai_openai/__init__.py @@ -16,6 +16,7 @@ from opentelemetry.instrumentation.instrumentor import BaseInstrumentor # type: ignore from traceai_openai._client_provider import set_client_provider as set_client_provider from traceai_openai._request import _AsyncRequest, _Request +from traceai_openai._request_context import capture_response_credentials from traceai_openai.package import _instruments from traceai_openai.version import __version__ from wrapt import wrap_function_wrapper @@ -38,6 +39,8 @@ class OpenAIInstrumentor(BaseInstrumentor): # type: ignore __slots__ = ( "_original_request", "_original_async_request", + "_original_process_response", + "_original_async_process_response", ) def instrumentation_dependencies(self) -> Collection[str]: @@ -67,6 +70,14 @@ def _instrument(self, **kwargs: Any) -> None: name="AsyncOpenAI.request", wrapper=_AsyncRequest(tracer=tracer, openai=openai, config=config), ) + self._original_process_response = openai.OpenAI._process_response + self._original_async_process_response = openai.AsyncOpenAI._process_response + for client in ("OpenAI", "AsyncOpenAI"): + wrap_function_wrapper( + module=_MODULE, + name=f"{client}._process_response", + wrapper=capture_response_credentials, + ) if Protect is not None: self._original_protect = Protect.protect wrap_function_wrapper( @@ -81,3 +92,5 @@ def _uninstrument(self, **kwargs: Any) -> None: openai = import_module(_MODULE) openai.OpenAI.request = self._original_request openai.AsyncOpenAI.request = self._original_async_request + openai.OpenAI._process_response = self._original_process_response + openai.AsyncOpenAI._process_response = self._original_async_process_response diff --git a/python/frameworks/openai/traceai_openai/_attributes/_responses_api.py b/python/frameworks/openai/traceai_openai/_attributes/_responses_api.py index 0080d5fcb..1f94c15f5 100644 --- a/python/frameworks/openai/traceai_openai/_attributes/_responses_api.py +++ b/python/frameworks/openai/traceai_openai/_attributes/_responses_api.py @@ -6,7 +6,8 @@ from opentelemetry.util.types import AttributeValue from typing_extensions import assert_never -from fi_instrumentation import safe_json_dumps +from fi_instrumentation import safe_json_dumps +from traceai_openai._privacy import safe_tool from fi_instrumentation.fi_types import ( ImageAttributes, MessageAttributes, @@ -237,7 +238,7 @@ def _get_attributes_from_response_tool( obj: responses.tool.Tool, prefix: str = "", ) -> Iterator[Tuple[str, AttributeValue]]: - yield (f"{prefix}{ToolAttributes.TOOL_JSON_SCHEMA}", obj.model_dump_json()) + yield (f"{prefix}{ToolAttributes.TOOL_JSON_SCHEMA}", safe_json_dumps(safe_tool(obj.model_dump(mode="json")))) @classmethod @stop_on_exception diff --git a/python/frameworks/openai/traceai_openai/_privacy.py b/python/frameworks/openai/traceai_openai/_privacy.py new file mode 100644 index 000000000..b4d92fecf --- /dev/null +++ b/python/frameworks/openai/traceai_openai/_privacy.py @@ -0,0 +1,157 @@ +"""Telemetry-only copies; never mutate SDK requests, responses or exceptions.""" + +from __future__ import annotations + +import json +from copy import deepcopy +from collections.abc import Iterable, Mapping +from typing import Any +from urllib.parse import parse_qsl, unquote, urlsplit, urlunsplit + +from fi_instrumentation import TraceConfig + +REDACTED = "__REDACTED__" + + +def redact(value: Any, secrets: Iterable[str]) -> Any: + if isinstance(value, str): + for secret in secrets: + value = value.replace(secret, REDACTED) + return value + if isinstance(value, (list, tuple)): + return type(value)(redact(item, secrets) for item in value) + return value + + +def request_credentials(instance: Any, options: Any, parameters: Mapping) -> tuple[str, ...]: + """Read static credentials only; never invoke a caller's token provider.""" + values = [getattr(instance, "api_key", None)] + for headers in (getattr(instance, "_custom_headers", None), getattr(options, "headers", None)): + if isinstance(headers, Mapping): + for name, value in headers.items(): + if str(name).lower() in {"authorization", "api-key", "x-api-key"}: + values.append(value) + if isinstance(value, str) and value.lower().startswith("bearer "): + values.append(value[7:]) + # extra_body is a shallow SDK override. Inspect its effective tools only + # for private redaction; never add extra_body to exported request metadata. + extra_json = getattr(options, "extra_json", None) + tools = ( + extra_json["tools"] + if isinstance(extra_json, Mapping) and "tools" in extra_json + else parameters.get("tools") + ) + if isinstance(tools, (list, tuple)): + for tool in tools: + if isinstance(tool, Mapping) and tool.get("type") == "mcp": + values.append(tool.get("authorization")) + headers = tool.get("headers") + if isinstance(headers, Mapping): + values.extend(headers.values()) + for name, value in tool.items(): + if isinstance(name, str) and name.endswith(("_url", "_uri")) and isinstance(value, str): + values.extend(_url_credentials(value)) + secrets = set() + for value in values: + if isinstance(value, str) and value: + secrets.update((value, json.dumps(value)[1:-1], repr(value)[1:-1])) + return tuple(sorted(secrets, key=len, reverse=True)) + + +def response_credentials(response: Any) -> tuple[str, ...]: + """Use the request actually sent, not a shared client's last refreshed token. + + Raw responses and streams retain their HTTP response. Reading its request + headers never calls the user's token provider or modifies authentication. + """ + http_response = getattr(response, "http_response", None) + if http_response is None: + http_response = getattr(response, "response", None) + request = getattr(http_response, "request", None) + return request_credentials(None, request, {}) + + +def _url_credentials(value: str) -> Iterable[str]: + """Remember private URL components for upstream errors that echo credentials.""" + try: + url = urlsplit(value) + except ValueError: + yield value + return + if url.username or url.password or url.query or url.fragment: + yield value + for part in (url.username, url.password, url.fragment): + if part: + yield part + yield unquote(part) + for name, secret in parse_qsl(url.query): + # Do not treat ordinary query options (e.g. version=1) as credentials. + name = name.lower().replace("-", "_") + if any(marker in name for marker in ("token", "key", "secret", "password", "signature", "auth")) or name == "sig": + yield secret + + +def _safe_url(value: str) -> str: + try: + url = urlsplit(value) + # Keep endpoint attribution, not userinfo, signed queries or fragments. + return urlunsplit((url.scheme, url.netloc.rsplit("@", 1)[-1], url.path, "", "")) + except ValueError: + return REDACTED + + +def safe_tool(tool: Any) -> Any: + if isinstance(tool, Mapping) and tool.get("type") == "mcp": + return { + key: _safe_url(value) + if isinstance(key, str) and key.endswith(("_url", "_uri")) and isinstance(value, str) + else value + for key, value in tool.items() + if key not in {"authorization", "headers"} + } + return tool + + +def safe_request(parameters: Mapping) -> dict: + """MCP headers are arbitrary authentication material, not tool schemas.""" + result = dict(parameters) + tools = result.get("tools") + if isinstance(tools, (list, tuple)): + result["tools"] = [safe_tool(tool) for tool in tools] + return result + + +def response_output(response: Any, config: TraceConfig | None = None) -> dict: + """Only response-owned data belongs in output, not echoed instructions/tools. + + An allowlist also keeps new SDK request-echo fields from bypassing input + privacy flags. The SDK response returned to the application is unchanged. + """ + # Top-level vendor errors are not assistant output. Preserve the existing + # exclusion even when error-detail masking is off; status records failure. + fields = {"id", "object", "created_at", "model", "status", "output", "usage", + "incomplete_details"} + if config is not None and getattr(config, "hide_error_details", False): + fields.discard("incomplete_details") + if isinstance(response, Mapping): + output = {key: deepcopy(value) for key, value in response.items() + if key in fields and value is not None} + else: + output = response.model_dump(include=fields, exclude_none=True, mode="json") + if config is not None and getattr(config, "hide_error_details", False): + # These provider-owned error fields are not ordinary tool results or + # assistant text. Work on the dump only, leaving SDK objects untouched. + for item in output.get("output", []): + if not isinstance(item, dict) or item.get("type") not in { + "mcp_call", "mcp_list_tools", "image_generation_call", + }: + continue + error = item.pop("error", None) + if isinstance(error, dict) and error.get("type") in { + "http_error", "mcp_protocol_error", "mcp_tool_execution_error", + }: + classification = {"type": error["type"]} + if type(error.get("code")) is int: + classification["code"] = error["code"] + item["error"] = classification + return output diff --git a/python/frameworks/openai/traceai_openai/_request.py b/python/frameworks/openai/traceai_openai/_request.py index be83c32ed..775571280 100644 --- a/python/frameworks/openai/traceai_openai/_request.py +++ b/python/frameworks/openai/traceai_openai/_request.py @@ -1,7 +1,7 @@ -import inspect import logging from abc import ABC from contextlib import contextmanager +from inspect import iscoroutinefunction from itertools import chain from types import ModuleType from typing import Any, Awaitable, Callable, Iterable, Iterator, Mapping, Optional, Tuple @@ -11,6 +11,7 @@ FiSpanKindValues, SpanAttributes, ) +from openai._legacy_response import LegacyAPIResponse from openai.types.create_embedding_response import CreateEmbeddingResponse from opentelemetry import context as context_api from opentelemetry import trace as trace_api @@ -20,15 +21,24 @@ from traceai_openai._client_provider import _get_client_provider from traceai_openai._deepseek import _DeepSeekChatAccumulator from traceai_openai._providers import provider_from_host +from traceai_openai._privacy import redact, request_credentials, response_credentials, response_output, safe_request +from traceai_openai._request_context import capture_request_credentials from traceai_openai._request_attributes_extractor import _RequestAttributesExtractor from traceai_openai._response_accumulator import ( _ChatCompletionAccumulator, _CompletionAccumulator, _ResponsesAccumulator, + _response_status, ) from traceai_openai._response_attributes_extractor import _ResponseAttributesExtractor from traceai_openai._span_io_handler import add_io_to_span_attributes -from traceai_openai._stream import _AsyncStream, _ResponseAccumulator, _Stream +from traceai_openai._stream import ( + _AsyncRawResponse, + _AsyncStream, + _RawResponse, + _ResponseAccumulator, + _Stream, +) from traceai_openai._utils import ( _as_input_attributes, _finish_tracing, @@ -37,7 +47,6 @@ ) from traceai_openai._with_span import _WithSpan from typing_extensions import TypeAlias -from wrapt import ObjectProxy __all__ = ( "_Request", @@ -60,12 +69,13 @@ def _start_as_current_span( attributes: Iterable[Tuple[str, AttributeValue]], context_attributes: Iterable[Tuple[str, AttributeValue]], extra_attributes: Iterable[Tuple[str, AttributeValue]], + secrets: Tuple[str, ...] = (), ) -> Iterator[_WithSpan]: # Keep primary attributes so finalization can restore them after optional # details: OTel's bounded attribute map evicts the oldest entries. initial_attributes = {} try: - initial_attributes = dict(attributes) + initial_attributes = {key: redact(value, secrets) for key, value in attributes} span = self._tracer.start_span(name=span_name, attributes=initial_attributes) except Exception: logger.exception("Failed to start span") @@ -78,6 +88,7 @@ def _start_as_current_span( ) as span: yield _WithSpan( span=span, + secrets=secrets, initial_attributes=initial_attributes, context_attributes=dict(context_attributes), extra_attributes=dict(extra_attributes), @@ -117,6 +128,7 @@ def responses_accumulator(request_parameters: _RequestParameters) -> Any: return _ResponsesAccumulator( request_parameters=request_parameters, chat_completion_type=openai.types.responses.response.Response, + config=self._config, response_attributes_extractor=self._response_attributes_extractor, ) @@ -197,17 +209,6 @@ def _get_extra_attributes_from_request( def _is_streaming(self, response: Any) -> bool: return isinstance(response, self._stream_types) - def _is_raw_stream(self, response: Any) -> bool: - parse = getattr(response, "parse", None) - return bool( - callable(parse) - and ( - getattr(response, "_stream", False) - or getattr(response, "_is_sse_stream", False) - or inspect.iscoroutinefunction(parse) - ) - ) - def _is_async_stream(self, response: Any) -> bool: """Check if the response is an asynchronous stream.""" return hasattr(response, "__aiter__") and callable(response.__aiter__) @@ -224,93 +225,81 @@ def _finalize_response( Monkey-patch the response object to trace the stream, or finish tracing if the response is not a stream. """ - # Raw streams and asynchronously parsed raw responses expose headers before - # their contents are ready. Preserve the raw interface and transfer span - # ownership when the application parses the response. - if self._is_raw_stream(response): - return _RawStreamResponse( - response=response, - finalize=lambda parsed: self._finalize_response( - response=parsed, - with_span=with_span, - cast_to=cast_to, - request_parameters=request_parameters, - instance=instance, - ), - with_span=with_span, - ) - if hasattr(response, "parse") and callable(response.parse): - try: - response.parse() - except Exception: - logger.exception(f"Failed to parse response of type {type(response)}") + response_accumulator_factory = self._response_accumulator_factories.get(cast_to) + response_accumulator = ( + response_accumulator_factory(request_parameters) + if response_accumulator_factory + else None + ) provider = self._provider_for_instance(instance) extractor = self._response_attributes_extractor if provider == "deepseek": - # Keep provider-specific extraction local to this request: concurrent - # requests through another provider must not inherit DeepSeek aliases. + # Keep provider-specific extraction local to each request. extractor = _ResponseAttributesExtractor(self._openai, self._config, provider=provider) + if cast_to is self._openai.types.chat.ChatCompletion: + response_accumulator = _DeepSeekChatAccumulator( + request_parameters=request_parameters, + chat_completion_type=cast_to, + response_attributes_extractor=extractor, + ) + elif cast_to is self._openai.types.responses.response.Response: + response_accumulator = _ResponsesAccumulator( + request_parameters=request_parameters, + chat_completion_type=cast_to, + response_attributes_extractor=extractor, + config=self._config, + ) - if self._is_streaming(response): - # Determine if the response is asynchronous - if self._is_async_stream(response): - stream_wrapper = _AsyncStream - else: - stream_wrapper = _Stream - - # Get the response accumulator if available - response_accumulator_factory = self._response_accumulator_factories.get( - cast_to - ) - response_accumulator = ( - response_accumulator_factory(request_parameters) - if response_accumulator_factory - else None - ) - - if provider == "deepseek": - if cast_to is self._openai.types.chat.ChatCompletion: - response_accumulator = _DeepSeekChatAccumulator( - request_parameters=request_parameters, - chat_completion_type=cast_to, - response_attributes_extractor=extractor, - ) - elif cast_to is self._openai.types.responses.response.Response: - response_accumulator = _ResponsesAccumulator( - request_parameters=request_parameters, - chat_completion_type=cast_to, - response_attributes_extractor=extractor, - ) - - # Wrap the response with the appropriate stream wrapper - wrapped_stream = stream_wrapper( - stream=response, - with_span=with_span, - response_accumulator=response_accumulator, - ) - - return wrapped_stream - - else: - failed_response = ( - isinstance(response, self._openai.types.responses.response.Response) - and response.status == "failed" + def on_parse(parsed: Any) -> Any: + if self._is_streaming(parsed): + stream_wrapper = _AsyncStream if self._is_async_stream(parsed) else _Stream + return stream_wrapper( + stream=parsed, + with_span=with_span, + response_accumulator=response_accumulator, + ) + if isinstance(parsed, CreateEmbeddingResponse): + with_span.set_attribute( + SpanAttributes.EMBEDDING_EMBEDDINGS, self._raw_embeddings_json(parsed) + ) + telemetry_output = ( + response_output(parsed, self._config) + if cast_to is self._openai.types.responses.response.Response + and isinstance(parsed, dict) else parsed ) + add_io_to_span_attributes(with_span, None, telemetry_output, config=self._config) _finish_tracing( - status=trace_api.Status( - status_code=( - trace_api.StatusCode.ERROR if failed_response else trace_api.StatusCode.OK - ) - ), + status=_response_status(parsed), with_span=with_span, has_attributes=_ResponseAttributes( request_parameters=request_parameters, - response=response, + response=parsed, response_attributes_extractor=extractor, ), ) - return response + return parsed + + if hasattr(response, "parse") and callable(response.parse): + # Modern envelopes are deliberately lazy, even for non-SSE responses. + # Do not call async parse from sync I/O/attribute extraction paths. + raw_wrapper = ( + _AsyncRawResponse if iscoroutinefunction(response.parse) else _RawResponse + ) + raw = raw_wrapper(response, with_span, response_accumulator, on_parse) + if isinstance(response, LegacyAPIResponse): + # Preserve eager legacy non-stream telemetry. Stream parse caches + # remain SDK-owned, and every later explicit parse type is traced. + try: + parsed = response.parse() + except Exception: + # Optional eager telemetry must not change SDK behavior: callers + # still receive the envelope and may inspect malformed bodies. + logger.debug("Could not eagerly parse raw response for telemetry", exc_info=True) + else: + raw._trace_parsed(parsed) + return raw + return on_parse(response) def _raw_embeddings_json(self, response: CreateEmbeddingResponse) -> str: """The raw embeddings response; vectors are replaced by their dimension @@ -335,145 +324,6 @@ def response_to_dict(self, response: Any) -> Any: return str(response) -class _RawStreamResponse(ObjectProxy): # type: ignore - """Preserve raw headers while tracing the SDK's lazily parsed response.""" - - __slots__ = ("_self_finalize", "_self_with_span", "_self_parsed", "_self_parsed_raw") - - def __init__( - self, response: Any, finalize: Callable[[Any], Any], with_span: _WithSpan - ) -> None: - super().__init__(response) - self._self_finalize = finalize - self._self_with_span = with_span - self._self_parsed = None - self._self_parsed_raw = None - - def _self_error(self, error: BaseException) -> None: - if not self._self_with_span.is_finished: - self._self_with_span.record_exception(error) - self._self_with_span.finish_tracing( - status=trace_api.Status( - status_code=trace_api.StatusCode.ERROR, - description=f"{type(error).__name__}: {error}", - ) - ) - - def _self_wrap(self, parsed: Any) -> Any: - if self._self_parsed is not None and parsed is self._self_parsed_raw: - return self._self_parsed - wrapped = self._self_finalize(parsed) - self._self_parsed_raw = parsed - self._self_parsed = wrapped - return wrapped - - def parse(self, *args: Any, **kwargs: Any) -> Any: - try: - parsed = self.__wrapped__.parse(*args, **kwargs) - except BaseException as error: - self._self_error(error) - raise - if inspect.isawaitable(parsed): - - async def await_parsed() -> Any: - try: - return self._self_wrap(await parsed) - except BaseException as error: - self._self_error(error) - raise - - return await_parsed() - try: - return self._self_wrap(parsed) - except BaseException as error: - self._self_error(error) - raise - - def close(self) -> Any: - # For async raw responses, callers should await the returned close. - if isinstance(self._self_parsed, _AsyncStream) or inspect.iscoroutinefunction( - getattr(self.__wrapped__, "close", None) - ): - return self.aclose() - try: - if isinstance(self._self_parsed, (_Stream, _AsyncStream)): - self._self_parsed.close() - close = getattr(self.__wrapped__, "close", None) - if close is None: - close = self.__wrapped__.http_response.close - return close() - except BaseException as error: - self._self_error(error) - raise - finally: - if not self._self_with_span.is_finished: - self._self_with_span.finish_tracing( - status=trace_api.Status(status_code=trace_api.StatusCode.OK) - ) - - async def aclose(self) -> None: - try: - if isinstance(self._self_parsed, (_Stream, _AsyncStream)): - closed = self._self_parsed.close() - if inspect.isawaitable(closed): - await closed - close = getattr(self.__wrapped__, "aclose", None) - if close is None: - close = getattr(self.__wrapped__.http_response, "aclose", None) - if close is None: - close = ( - getattr(self.__wrapped__, "close", None) - or self.__wrapped__.http_response.close - ) - closed = close() - if inspect.isawaitable(closed): - await closed - except BaseException as error: - self._self_error(error) - raise - finally: - if not self._self_with_span.is_finished: - self._self_with_span.finish_tracing( - status=trace_api.Status(status_code=trace_api.StatusCode.OK) - ) - - def __enter__(self) -> "_RawStreamResponse": - enter = getattr(self.__wrapped__, "__enter__", None) - if enter is not None: - enter() - return self - - def __exit__(self, exc_type: Any, exc: Any, tb: Any) -> Any: - try: - exit_raw = getattr(self.__wrapped__, "__exit__", None) - return exit_raw(exc_type, exc, tb) if exit_raw is not None else None - except BaseException as close_error: - exc = close_error - raise - finally: - if exc is not None: - self._self_error(exc) - self.close() - - async def __aenter__(self) -> "_RawStreamResponse": - enter = getattr(self.__wrapped__, "__aenter__", None) - if enter is not None: - await enter() - return self - - async def __aexit__(self, exc_type: Any, exc: Any, tb: Any) -> Any: - try: - exit_raw = getattr(self.__wrapped__, "__aexit__", None) - return await exit_raw(exc_type, exc, tb) if exit_raw is not None else None - except BaseException as close_error: - exc = close_error - raise - finally: - if exc is not None: - self._self_error(exc) - await self.aclose() - - class _Request(_WithTracer, _WithOpenAI): def __call__( self, @@ -486,6 +336,8 @@ def __call__( return wrapped(*args, **kwargs) try: cast_to, request_parameters = _parse_request_args(args) + secrets = request_credentials(instance, args[1], request_parameters) + request_parameters = safe_request(request_parameters) # Extract input data more robustly input_data = ( request_parameters.get("messages") @@ -500,6 +352,7 @@ def __call__( with self._start_as_current_span( span_name=span_name, + secrets=secrets, attributes=chain( self._get_attributes_from_instance(instance), self._get_attributes_from_request( @@ -519,22 +372,11 @@ def __call__( ) add_io_to_span_attributes(with_span, input_data, None) try: - response = wrapped(*args, **kwargs) - - if isinstance(response, CreateEmbeddingResponse): - with_span.set_attribute( - SpanAttributes.EMBEDDING_EMBEDDINGS, - self._raw_embeddings_json(response), - ) - - # Add output data to span attributes after getting response - if not self._is_streaming(response) and not self._is_raw_stream(response): - add_io_to_span_attributes( - with_span, - None, - response, - is_streaming=self._is_streaming(response), - ) + with capture_request_credentials(instance, with_span): + response = wrapped(*args, **kwargs) + with_span.add_secrets(request_credentials(instance, None, {})) + with_span.add_secrets(response_credentials(response)) + return self._finalize_response( response=response, with_span=with_span, @@ -542,7 +384,8 @@ def __call__( request_parameters=request_parameters, instance=instance, ) - except Exception as e: + except BaseException as e: + with_span.add_secrets(request_credentials(instance, getattr(e, "request", None), {})) status = trace_api.Status( status_code=trace_api.StatusCode.ERROR, description=f"{type(e).__name__}: {e}", @@ -564,6 +407,8 @@ async def __call__( return await wrapped(*args, **kwargs) try: cast_to, request_parameters = _parse_request_args(args) + secrets = request_credentials(instance, args[1], request_parameters) + request_parameters = safe_request(request_parameters) # Extract input data more robustly input_data = ( request_parameters.get("messages") @@ -578,6 +423,7 @@ async def __call__( return await wrapped(*args, **kwargs) with self._start_as_current_span( span_name=span_name, + secrets=secrets, attributes=chain( self._get_attributes_from_instance(instance), self._get_attributes_from_request( @@ -597,20 +443,14 @@ async def __call__( ) add_io_to_span_attributes(with_span, input_data, None) try: - response = await wrapped(*args, **kwargs) - - if isinstance(response, CreateEmbeddingResponse): - with_span.set_attribute( - SpanAttributes.EMBEDDING_EMBEDDINGS, - self._raw_embeddings_json(response), - ) - - if not self._is_raw_stream(response): - add_io_to_span_attributes( - with_span, None, response, is_streaming=self._is_streaming(response) - ) - # Cancellation can arrive before a stream/response takes ownership. + with capture_request_credentials(instance, with_span): + response = await wrapped(*args, **kwargs) + with_span.add_secrets(request_credentials(instance, None, {})) + with_span.add_secrets(response_credentials(response)) except BaseException as exception: + with_span.add_secrets(request_credentials( + instance, getattr(exception, "request", None), {} + )) with_span.record_exception(exception) status = trace_api.Status( status_code=trace_api.StatusCode.ERROR, @@ -672,12 +512,6 @@ def __init__( request_parameters: Mapping[str, Any], response_attributes_extractor: _ResponseAttributesExtractor, ) -> None: - if hasattr(response, "parse") and callable(response.parse): - # E.g. see https://github.com/openai/openai-python/blob/f1c7d714914e3321ca2e72839fe2d132a8646e7f/src/openai/_base_client.py#L518 # noqa: E501 - try: - response = response.parse() - except Exception: - logger.exception(f"Failed to parse response of type {type(response)}") self._request_parameters = request_parameters self._response = response self._response_attributes_extractor = response_attributes_extractor diff --git a/python/frameworks/openai/traceai_openai/_request_context.py b/python/frameworks/openai/traceai_openai/_request_context.py new file mode 100644 index 000000000..c81131cf7 --- /dev/null +++ b/python/frameworks/openai/traceai_openai/_request_context.py @@ -0,0 +1,41 @@ +"""Capture request-local credentials before the SDK discards its HTTP envelope.""" + +import logging +from contextlib import contextmanager +from contextvars import ContextVar +from typing import Any, Iterator, Optional, Tuple + +from traceai_openai._privacy import request_credentials +from traceai_openai._with_span import _WithSpan + +logger = logging.getLogger(__name__) +_active_request: ContextVar[Optional[Tuple[Any, _WithSpan]]] = ContextVar( + "traceai_openai_request", default=None +) + + +@contextmanager +def capture_request_credentials(client: Any, span: _WithSpan) -> Iterator[None]: + token = _active_request.set((client, span)) + try: + yield + finally: + _active_request.reset(token) + + +def capture_response_credentials( + wrapped: Any, instance: Any, args: Any, kwargs: Any +) -> Any: + # Both SDK bounds pass response by keyword to _process_response, before + # parsing removes the request. ContextVars isolate concurrent calls without + # modifying shared clients/transports or invoking a token provider again. + active = _active_request.get() + if active is not None and active[0] is instance: + try: + response = kwargs.get("response") + active[1].add_secrets( + request_credentials(None, getattr(response, "request", None), {}) + ) + except Exception: + logger.warning("Failed to capture request credentials") + return wrapped(*args, **kwargs) diff --git a/python/frameworks/openai/traceai_openai/_response_accumulator.py b/python/frameworks/openai/traceai_openai/_response_accumulator.py index f5930f7f3..3202cac09 100644 --- a/python/frameworks/openai/traceai_openai/_response_accumulator.py +++ b/python/frameworks/openai/traceai_openai/_response_accumulator.py @@ -17,16 +17,19 @@ Type, ) -from fi_instrumentation import safe_json_dumps +from fi_instrumentation import TraceConfig, safe_json_dumps from fi_instrumentation.fi_types import FiMimeTypeValues, SpanAttributes from opentelemetry import trace as trace_api from opentelemetry.util.types import AttributeValue +from opentelemetry.trace import Status, StatusCode +from traceai_openai._privacy import response_output from traceai_openai._utils import _as_output_attributes, _to_dict, _ValueAndType, _io_value_and_type if TYPE_CHECKING: from openai.types import Completion from openai.types.chat import ChatCompletion from openai.types.responses.response import Response + from openai.types.responses import ResponseStreamEvent __all__ = ( "_CompletionAccumulator", @@ -43,8 +46,21 @@ def get_attributes_from_response( ) -> Iterator[Tuple[str, AttributeValue]]: ... +def _response_status(response: Any) -> Status: + status = ( + response.get("status") + if isinstance(response, Mapping) + else getattr(response, "status", None) + ) + if isinstance(status, str) and status in {"failed", "cancelled", "incomplete"}: + # The vendor's error message can contain credentials or request content. + return Status(StatusCode.ERROR, "Response did not complete") + return Status(StatusCode.OK) + + class _ResponsesAccumulator: __slots__ = ( + "_config", "_is_null", "_values", "_cached_result", @@ -58,35 +74,36 @@ def __init__( request_parameters: Mapping[str, Any], chat_completion_type: Type["Response"], response_attributes_extractor: Optional[_CanGetAttributesFromResponse] = None, + config: Optional[TraceConfig] = None, ) -> None: + self._config = config self._chat_completion_type = chat_completion_type self._request_parameters = request_parameters self._response_attributes_extractor = response_attributes_extractor self._is_null = True self._cached_result: Optional[Dict[str, Any]] = None - self._values: Optional["Response"] = None + self._values: Optional["ResponseStreamEvent"] = None def process_chunk(self, chunk: Any) -> None: - if getattr(chunk, "type", None) in ( - "response.completed", - "response.incomplete", - "response.failed", - ): + if getattr(chunk, "type", None) in { + "response.completed", "response.failed", "response.incomplete", "error" + }: self._is_null = False self._cached_result = None - self._values = chunk.response + self._values = chunk def _result(self) -> Any: if self._is_null: return None - return self._values + if self._values: + return getattr(self._values, "response", None) - def get_status(self) -> Optional[trace_api.Status]: - if self._values is not None and self._values.status == "failed": - # Vendor error messages can contain private input. Status alone is - # sufficient here; do not copy error details into output or events. - return trace_api.Status(status_code=trace_api.StatusCode.ERROR) - return None + def get_status(self) -> Status: + if self._values is None: + return Status(StatusCode.ERROR, "Response stream ended before a terminal event") + if self._values.type != "response.completed": + return Status(StatusCode.ERROR, "Response stream did not complete") + return _response_status(self._result()) def get_attributes(self) -> Iterator[Tuple[str, AttributeValue]]: if not (result := self._result()): @@ -110,12 +127,10 @@ def get_attributes(self) -> Iterator[Tuple[str, AttributeValue]]: if output_content: yield SpanAttributes.OUTPUT_VALUE, output_content else: - # Preserve non-text output, but never treat vendor error details as - # assistant output (output.value has a separate privacy policy). - output = { - key: value for key, value in _to_dict(result).items() if key != "error" - } - yield from _as_output_attributes(_io_value_and_type(output)) + # Keep response-owned data, never echoed request fields. + yield from _as_output_attributes( + _io_value_and_type(response_output(result, self._config)), + ) def get_extra_attributes(self) -> Iterator[Tuple[str, AttributeValue]]: if not (result := self._result()): diff --git a/python/frameworks/openai/traceai_openai/_span_io_handler.py b/python/frameworks/openai/traceai_openai/_span_io_handler.py index ab4c25920..772db2da3 100644 --- a/python/frameworks/openai/traceai_openai/_span_io_handler.py +++ b/python/frameworks/openai/traceai_openai/_span_io_handler.py @@ -2,8 +2,10 @@ import logging from typing import Any, Optional +from fi_instrumentation import TraceConfig from fi_instrumentation.fi_types import FiMimeTypeValues, SpanAttributes from traceai_openai._with_span import _WithSpan +from traceai_openai._privacy import response_output logger = logging.getLogger(__name__) logger.addHandler(logging.NullHandler()) @@ -88,7 +90,7 @@ def _set_mime_type(span: _WithSpan, key: str, value: Any) -> None: span.set_attribute(key, mime) -def _json_output_for(output_data: Any) -> Optional[str]: +def _json_output_for(output_data: Any, config: Optional[TraceConfig] = None) -> Optional[str]: """JSON for responses whose output is not a chat message: the Responses API object as JSON, and a compact summary for embeddings (never the vectors).""" try: @@ -98,11 +100,7 @@ def _json_output_for(output_data: Any) -> Optional[str]: return None if isinstance(output_data, Response): # Lead with the answer text (the SDK's `output_text`) so previews show it. - # Errors are not assistant output. Keep vendor details out of the raw - # output fallback, whose privacy policy is independent of error masking. - body = json.loads( - output_data.model_dump_json(exclude_none=True, exclude={"error"}) - ) + body = response_output(output_data, config) output_text = getattr(output_data, "output_text", None) if output_text: body = {"output_text": output_text, **body} @@ -120,7 +118,8 @@ def _json_output_for(output_data: Any) -> Optional[str]: def add_io_to_span_attributes( - span: _WithSpan, input_data: Any, output_data: Any, is_streaming: bool = False + span: _WithSpan, input_data: Any, output_data: Any, is_streaming: bool = False, + config: Optional[TraceConfig] = None ) -> None: """Add input/output data to span attributes. @@ -139,7 +138,7 @@ def add_io_to_span_attributes( if output_data is not None and not is_streaming: if hasattr(output_data, "parse") and not hasattr(output_data, "model_dump"): output_data = output_data.parse() - if (json_output := _json_output_for(output_data)) is not None: + if (json_output := _json_output_for(output_data, config)) is not None: span.set_attribute(SpanAttributes.OUTPUT_VALUE, json_output) span.set_attribute( SpanAttributes.OUTPUT_MIME_TYPE, FiMimeTypeValues.JSON.value diff --git a/python/frameworks/openai/traceai_openai/_stream.py b/python/frameworks/openai/traceai_openai/_stream.py index 40d8c7932..914e72e2e 100644 --- a/python/frameworks/openai/traceai_openai/_stream.py +++ b/python/frameworks/openai/traceai_openai/_stream.py @@ -1,9 +1,10 @@ import logging -from typing import Any, AsyncIterator, Generator, Iterator, Optional, Protocol, Tuple +from typing import Any, AsyncIterator, Callable, Generator, Iterator, Optional, Protocol, Tuple from openai.types.chat import ChatCompletionChunk from opentelemetry import trace as trace_api from opentelemetry.util.types import AttributeValue +from traceai_openai._response_accumulator import _response_status from traceai_openai._utils import _finish_tracing from traceai_openai._with_span import _WithSpan from wrapt import ObjectProxy @@ -42,6 +43,249 @@ def _finish_on_exit(stream: Any, error: Optional[BaseException]) -> None: stream._finish_tracing(status=status) +class _ResponseCloseProxy(ObjectProxy): + """Responses.stream() closes the HTTP response, not the raw SDK stream. + + Expose a per-call proxy instead of patching HTTPX classes or the client's + transport. Internal SDK reads still use the original response unchanged. + """ + + def __init__(self, response: Any, stream: Any) -> None: + super().__init__(response) + self._self_stream = stream + + def close(self) -> None: + error = None + try: + self.__wrapped__.close() + except BaseException as exc: + error = exc + raise + finally: + _finish_on_exit(self._self_stream, error) + + async def aclose(self) -> None: + error = None + try: + await self.__wrapped__.aclose() + except BaseException as exc: + error = exc + raise + finally: + _finish_on_exit(self._self_stream, error) + + +class _RawResponse(ObjectProxy): + """Trace every parse-cache entry without eagerly reading a lazy envelope.""" + + def __init__( + self, + response: Any, + with_span: _WithSpan, + response_accumulator: Optional[_ResponseAccumulator], + on_parse: Callable[[Any], Any], + ) -> None: + super().__init__(response) + self._self_with_span = with_span + self._self_response_accumulator = response_accumulator + self._self_on_parse = on_parse + self._self_parsed = {} + self._self_body_status: Optional[trace_api.Status] = None + + def _record_body_consumption(self) -> None: + # read/text/json consume a non-SSE envelope without calling parse(). + # Do not force SDK parsing (or its post_parser) on a raw-body caller. + # SSE still needs a terminal event from the traced iterator. + if ( + self._self_is_finished + or self._self_body_status is not None + or getattr(self.__wrapped__, "_is_sse_stream", False) + ): + return + try: + body = self.__wrapped__.http_response.json() + except Exception: + body = None # Successful raw reads need not contain JSON. + self._self_body_status = _response_status(body) + + def _consume_body(self, method: str, *args: Any, **kwargs: Any) -> Any: + try: + result = getattr(self.__wrapped__, method)(*args, **kwargs) + except BaseException as exc: + _finish_on_exit(self, exc) + raise + self._record_body_consumption() + return result + + def read(self, *args: Any, **kwargs: Any) -> Any: + return self._consume_body("read", *args, **kwargs) + + def text(self, *args: Any, **kwargs: Any) -> Any: + return self._consume_body("text", *args, **kwargs) + + def json(self, *args: Any, **kwargs: Any) -> Any: + return self._consume_body("json", *args, **kwargs) + + def _consume_iterator(self, method: str, *args: Any, **kwargs: Any) -> Iterator[Any]: + try: + for chunk in getattr(self.__wrapped__, method)(*args, **kwargs): + yield chunk + except BaseException as exc: + _finish_on_exit(self, exc) + raise + else: + # The caller may stop iteration early; only exhaustion consumes it. + self._record_body_consumption() + + def iter_bytes(self, *args: Any, **kwargs: Any) -> Iterator[bytes]: + return self._consume_iterator("iter_bytes", *args, **kwargs) + + def iter_text(self, *args: Any, **kwargs: Any) -> Iterator[str]: + return self._consume_iterator("iter_text", *args, **kwargs) + + def iter_lines(self, *args: Any, **kwargs: Any) -> Iterator[str]: + return self._consume_iterator("iter_lines", *args, **kwargs) + + def stream_to_file(self, *args: Any, **kwargs: Any) -> None: + self._consume_body("stream_to_file", *args, **kwargs) + + @property + def _self_is_finished(self) -> bool: + return self._self_with_span.is_finished + + def _trace_parsed(self, parsed: Any) -> Any: + # The SDK owns cache keys (including generic stream types). Cache by + # identity and retain the original through the proxy, so repeated parse + # calls return the same wrapper without collapsing distinct SDK entries. + key = id(parsed) + if key not in self._self_parsed: + self._self_parsed[key] = self._self_on_parse(parsed) + return self._self_parsed[key] + + def parse(self, *args: Any, **kwargs: Any) -> Any: + try: + parsed = self.__wrapped__.parse(*args, **kwargs) + except BaseException as exc: + _finish_on_exit(self, exc) + raise + return self._trace_parsed(parsed) + + @property + def http_response(self) -> Any: + return _ResponseCloseProxy(self.__wrapped__.http_response, self) + + def close(self) -> Any: + if any(isinstance(parsed, _AsyncStream) for parsed in self._self_parsed.values()): + return self.aclose() + error = None + try: + # LegacyAPIResponse has no close method. Preserve the convenience + # close path already exposed by our raw-stream wrapper on dev. + close = getattr(self.__wrapped__, "close", None) + if close is None: + close = self.__wrapped__.http_response.close + close() + except BaseException as exc: + error = exc + raise + finally: + _finish_on_exit(self, error) + + async def aclose(self) -> None: + error = None + try: + await self.__wrapped__.http_response.aclose() + except BaseException as exc: + error = exc + raise + finally: + _finish_on_exit(self, error) + + def _finish_tracing(self, status: Optional[trace_api.Status] = None) -> None: + if self._self_is_finished: + return + accumulator = self._self_response_accumulator + if status is None or status.status_code is not trace_api.StatusCode.ERROR: + get_status = getattr(accumulator, "get_status", None) + if self._self_body_status is not None: + status = self._self_body_status + elif get_status is not None: + status = get_status() or status + elif not self._self_parsed: + status = trace_api.Status( + trace_api.StatusCode.ERROR, "Response closed before parsing" + ) + if accumulator is not None: + _finish_tracing(self._self_with_span, accumulator, status) + else: + self._self_with_span.finish_tracing(status=status) + + +class _AsyncRawResponse(_RawResponse): + """AsyncAPIResponse has awaitable parse/close; LegacyAPIResponse does not.""" + + async def _consume_body(self, method: str, *args: Any, **kwargs: Any) -> Any: + try: + result = await getattr(self.__wrapped__, method)(*args, **kwargs) + except BaseException as exc: + _finish_on_exit(self, exc) + raise + self._record_body_consumption() + return result + + async def read(self, *args: Any, **kwargs: Any) -> Any: + return await self._consume_body("read", *args, **kwargs) + + async def text(self, *args: Any, **kwargs: Any) -> Any: + return await self._consume_body("text", *args, **kwargs) + + async def json(self, *args: Any, **kwargs: Any) -> Any: + return await self._consume_body("json", *args, **kwargs) + + async def _consume_iterator(self, method: str, *args: Any, **kwargs: Any) -> AsyncIterator[Any]: + try: + async for chunk in getattr(self.__wrapped__, method)(*args, **kwargs): + yield chunk + except BaseException as exc: + _finish_on_exit(self, exc) + raise + else: + self._record_body_consumption() + + async def iter_bytes(self, *args: Any, **kwargs: Any) -> AsyncIterator[bytes]: + async for chunk in self._consume_iterator("iter_bytes", *args, **kwargs): + yield chunk + + async def iter_text(self, *args: Any, **kwargs: Any) -> AsyncIterator[str]: + async for chunk in self._consume_iterator("iter_text", *args, **kwargs): + yield chunk + + async def iter_lines(self, *args: Any, **kwargs: Any) -> AsyncIterator[str]: + async for chunk in self._consume_iterator("iter_lines", *args, **kwargs): + yield chunk + + async def stream_to_file(self, *args: Any, **kwargs: Any) -> None: + await self._consume_body("stream_to_file", *args, **kwargs) + + async def parse(self, *args: Any, **kwargs: Any) -> Any: + try: + parsed = await self.__wrapped__.parse(*args, **kwargs) + except BaseException as exc: + _finish_on_exit(self, exc) + raise + return self._trace_parsed(parsed) + + async def close(self) -> None: + error = None + try: + await self.__wrapped__.close() + except BaseException as exc: + error = exc + raise + finally: + _finish_on_exit(self, error) + + class _Stream(ObjectProxy): # type: ignore __slots__ = ( "_self_with_span", @@ -62,6 +306,13 @@ def __init__( self._self_is_finished = with_span.is_finished self._self_response_accumulator = response_accumulator + @property + def response(self) -> Any: + response = self.__wrapped__.response + if getattr(self._self_response_accumulator, "get_status", None) is not None: + return _ResponseCloseProxy(response, self) + return response + def __enter__(self) -> "_Stream": # ObjectProxy otherwise returns the SDK stream, bypassing our iterator. self.__wrapped__.__enter__() @@ -76,10 +327,10 @@ def __exit__(self, exc_type: Any, exc: Any, tb: Any) -> Any: finally: _finish_on_exit(self, exc) - def close(self) -> Any: + def close(self) -> None: error = None try: - return self.__wrapped__.close() + self.__wrapped__.close() except BaseException as exc: error = exc raise @@ -127,11 +378,12 @@ def _finish_tracing( self, status: Optional[trace_api.Status] = None, ) -> None: - if ( - self._self_response_accumulator is not None - and (status is None or status.status_code != trace_api.StatusCode.ERROR) - ): - status = self._self_response_accumulator.get_status() or status + if self._self_is_finished: + return + if status is None or status.status_code is not trace_api.StatusCode.ERROR: + get_status = getattr(self._self_response_accumulator, "get_status", None) + if get_status is not None: + status = get_status() or status _finish_tracing( status=status, with_span=self._self_with_span, @@ -168,6 +420,13 @@ def __init__( self._self_is_finished = with_span.is_finished self._self_response_accumulator = response_accumulator + @property + def response(self) -> Any: + response = self.__wrapped__.response + if getattr(self._self_response_accumulator, "get_status", None) is not None: + return _ResponseCloseProxy(response, self) + return response + async def __aenter__(self) -> "_AsyncStream": await self.__wrapped__.__aenter__() return self @@ -181,10 +440,10 @@ async def __aexit__(self, exc_type: Any, exc: Any, tb: Any) -> Any: finally: _finish_on_exit(self, exc) - async def close(self) -> Any: + async def close(self) -> None: error = None try: - return await self.__wrapped__.close() + await self.__wrapped__.close() except BaseException as exc: error = exc raise @@ -232,11 +491,12 @@ def _finish_tracing( self, status: Optional[trace_api.Status] = None, ) -> None: - if ( - self._self_response_accumulator is not None - and (status is None or status.status_code != trace_api.StatusCode.ERROR) - ): - status = self._self_response_accumulator.get_status() or status + if self._self_is_finished: + return + if status is None or status.status_code is not trace_api.StatusCode.ERROR: + get_status = getattr(self._self_response_accumulator, "get_status", None) + if get_status is not None: + status = get_status() or status _finish_tracing( status=status, with_span=self._self_with_span, diff --git a/python/frameworks/openai/traceai_openai/_with_span.py b/python/frameworks/openai/traceai_openai/_with_span.py index 923a31af8..c3b336328 100644 --- a/python/frameworks/openai/traceai_openai/_with_span.py +++ b/python/frameworks/openai/traceai_openai/_with_span.py @@ -1,9 +1,11 @@ import logging -from typing import Any, Optional +import traceback +from typing import Any, Optional, Tuple from fi_instrumentation.fi_types import SpanAttributes from opentelemetry import trace as trace_api from opentelemetry.util.types import Attributes +from traceai_openai._privacy import redact logger = logging.getLogger(__name__) logger.addHandler(logging.NullHandler()) @@ -22,6 +24,7 @@ class _WithSpan: __slots__ = ( "_span", + "_secrets", "_primary_attributes", "_context_attributes", "_extra_attributes", @@ -34,8 +37,10 @@ def __init__( context_attributes: Attributes = None, extra_attributes: Attributes = None, initial_attributes: Attributes = None, + secrets: Tuple[str, ...] = (), ) -> None: self._span = span + self._secrets = secrets self._primary_attributes = dict(initial_attributes or {}) self._context_attributes = context_attributes self._extra_attributes = extra_attributes @@ -49,11 +54,25 @@ def __init__( def is_finished(self) -> bool: return self._is_finished + def add_secrets(self, secrets: Tuple[str, ...]) -> None: + # Credentials resolved by an SDK token provider are available only after + # sending the request. Retain all known values for this span's lifetime. + self._secrets = tuple(sorted(set((*self._secrets, *secrets)), key=len, reverse=True)) + def record_exception(self, exception: BaseException) -> None: if self._is_finished: return try: - self._span.record_exception(exception) + # Emit the standard event through FITracer's masking API, without + # handing raw credential-bearing exception text to an exporter. + cls = type(exception) + self._span.add_event("exception", attributes={ + "exception.type": f"{cls.__module__}.{cls.__qualname__}", + "exception.message": redact(str(exception), self._secrets), + "exception.stacktrace": redact("".join(traceback.format_exception( + cls, exception, exception.__traceback__)), self._secrets), + "exception.escaped": "False", + }) except Exception: logger.exception("Failed to record exception on span") @@ -96,7 +115,15 @@ def finish_tracing( self.set_attribute(key, value) if status is not None: try: - self._span.set_status(status=status) + description = redact(status.description, self._secrets) + if description: + # Core removes tracebacks by consulting the in-flight SDK + # exception. Its original message may contain credentials. + # Remove the traceback here so core cannot reconstruct that + # unsanitized message; the redacted event keeps the stack. + description = description.partition("Traceback (most recent call last)")[0] + description = description.rstrip() or None + self._span.set_status(status=trace_api.Status(status.status_code, description)) except Exception: logger.exception("Failed to set status code on span") try: @@ -110,6 +137,7 @@ def set_attribute(self, key: str, value: Any) -> None: if self._is_finished: return try: + value = redact(value, self._secrets) self._span.set_attribute(key, value) if key in self._primary_attributes and value is not None: self._primary_attributes[key] = value diff --git a/python/frameworks/parallel/README.md b/python/frameworks/parallel/README.md index ab85d79d7..f6e9448fa 100644 --- a/python/frameworks/parallel/README.md +++ b/python/frameworks/parallel/README.md @@ -234,7 +234,7 @@ def research(client: Parallel, tracer: FITracer, question: str) -> TaskRunResult span.set_attribute("parallel.run_id", run.run_id) span.set_attribute("parallel.processor", "base") result = client.task_run.result(run.run_id, api_timeout=600, timeout=610) - except Exception: + except BaseException: span.set_status(Status(StatusCode.ERROR, "Task request failed")) raise span.set_attribute("parallel.task.status", result.run.status) diff --git a/python/frameworks/parallel/examples/task_workflow.py b/python/frameworks/parallel/examples/task_workflow.py index a3c32c20f..cbaf2b09f 100644 --- a/python/frameworks/parallel/examples/task_workflow.py +++ b/python/frameworks/parallel/examples/task_workflow.py @@ -25,7 +25,7 @@ def research(client: Parallel, tracer: FITracer, question: str) -> TaskRunResult span.set_attribute("parallel.run_id", run.run_id) span.set_attribute("parallel.processor", "base") result = client.task_run.result(run.run_id, api_timeout=600, timeout=610) - except Exception: + except BaseException: span.set_status(Status(StatusCode.ERROR, "Task request failed")) raise span.set_attribute("parallel.task.status", result.run.status) @@ -48,7 +48,7 @@ async def research_async(client: AsyncParallel, tracer: FITracer, question: str) span.set_attribute("parallel.cancelled", True) span.set_status(Status(StatusCode.ERROR, "Task wait cancelled")) raise - except Exception: + except BaseException: span.set_status(Status(StatusCode.ERROR, "Task request failed")) raise span.set_attribute("parallel.task.status", result.run.status) diff --git a/python/frameworks/parallel/tests/test_metadata_credentials.py b/python/frameworks/parallel/tests/test_metadata_credentials.py new file mode 100644 index 000000000..2b4582363 --- /dev/null +++ b/python/frameworks/parallel/tests/test_metadata_credentials.py @@ -0,0 +1,58 @@ +"""Provider metadata still obeys the credential and PII policy.""" + +import asyncio + +import httpx +import pytest +from fi_instrumentation import TraceConfig +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +from parallel import AsyncParallel, Parallel +from traceai_parallel import ParallelInstrumentor + + +@pytest.mark.parametrize("async_", [False, True]) +@pytest.mark.parametrize("pii", [False, True]) +def test_metadata_redacts_before_capping_without_mutating_response(async_, pii): + key = "synthetic-metadata-key" + email = "synthetic.person@example.com" + text = "x" * 240 + key + " " + email + exporter = InMemorySpanExporter() + provider = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + instrumentor = ParallelInstrumentor() + instrumentor.instrument(tracer_provider=provider, config=TraceConfig(pii_redaction=pii)) + + def handler(request): + assert request.headers["x-api-key"] == key + return httpx.Response(200, json={"search_id": "synthetic-search", "results": [], + "usage": [{"name": text, "count": 1}, {"name": email, "count": 2}], + "warnings": [{"type": text, "message": "safe"}, {"type": email, "message": "safe"}]}) + + def check(result): + assert result.usage[0].name == text + assert result.warnings[0].type == text + + async def run(): + async with AsyncParallel(api_key=key, max_retries=0, http_client=httpx.AsyncClient( + transport=httpx.MockTransport(handler), trust_env=False)) as client: + check(await client.search(search_queries=["safe"])) + + try: + if async_: + asyncio.run(run()) + else: + with Parallel(api_key=key, max_retries=0, http_client=httpx.Client( + transport=httpx.MockTransport(handler), trust_env=False)) as client: + check(client.search(search_queries=["safe"])) + span, = exporter.get_finished_spans() + # A cap at 256 bytes would leave only the beginning of the credential. + assert "synthetic-meta" not in span.to_json() + assert (email not in span.to_json()) is pii + assert span.attributes["parallel.usage.counts"] == (1, 2) + assert len(span.attributes["parallel.usage.names"][0].encode()) <= 256 + assert len(span.events[0].attributes["parallel.warning.type"].encode()) <= 256 + finally: + instrumentor.uninstrument() + provider.shutdown() diff --git a/python/frameworks/parallel/tests/test_task_workflow.py b/python/frameworks/parallel/tests/test_task_workflow.py index 52d40bdb4..0d52b9c40 100644 --- a/python/frameworks/parallel/tests/test_task_workflow.py +++ b/python/frameworks/parallel/tests/test_task_workflow.py @@ -205,3 +205,55 @@ def log_message(self, *_): server.shutdown() server.server_close() thread.join(timeout=3) + + +@pytest.mark.parametrize("stage", ["create", "wait"]) +@pytest.mark.parametrize("error_cls", [KeyboardInterrupt, SystemExit]) +@pytest.mark.parametrize("mode", ["sync", "async", "readme"]) +def test_task_interrupt_is_error_with_registered_processor(stage, error_cls, mode): + from fi_instrumentation import register + from fi_instrumentation.fi_types import ProjectType + + exporter = InMemorySpanExporter() + provider = register(project_name="task-interrupt", project_type=ProjectType.OBSERVE, + span_exporter=exporter, batch=False, verbose=False) + abort = error_cls("synthetic local abort " + KEY) + + def handler(request): + if stage == "create" or request.method == "GET": + raise abort + return httpx.Response(200, json=run_body("queued")) + + tracer = FITracer(provider.get_tracer("task-interrupt")) + + async def run_async(): + async with AsyncParallel(api_key=KEY, max_retries=0, http_client=httpx.AsyncClient( + transport=httpx.MockTransport(handler), trust_env=False, + )) as client: + with pytest.raises(error_cls) as caught: + await example.research_async(client, tracer, QUESTION) + assert caught.value is abort + + try: + if mode == "async": + asyncio.run(run_async()) + else: + research = example.research + if mode == "readme": + section = (EXAMPLE.parents[1] / "README.md").read_text().split("## Trace a Task workflow")[1] + snippet = section.split("```python\n")[1].split("\nprovider = register")[0] + namespace = {} + exec(snippet, namespace) + research = namespace["research"] + with Parallel(api_key=KEY, max_retries=0, http_client=httpx.Client( + transport=httpx.MockTransport(handler), trust_env=False, + )) as client: + with pytest.raises(error_cls) as caught: + research(client, tracer, QUESTION) + assert caught.value is abort + span, = exporter.get_finished_spans() + assert span.status.status_code is StatusCode.ERROR + assert KEY not in span.to_json() + assert span.events == () + finally: + provider.shutdown() diff --git a/python/frameworks/parallel/traceai_parallel/_wrappers.py b/python/frameworks/parallel/traceai_parallel/_wrappers.py index 61d095a03..84a6ee7d1 100644 --- a/python/frameworks/parallel/traceai_parallel/_wrappers.py +++ b/python/frameworks/parallel/traceai_parallel/_wrappers.py @@ -295,7 +295,7 @@ def _response_attributes( name = getattr(item, "name", None) count = getattr(item, "count", None) if isinstance(name, str) and isinstance(count, int) and not isinstance(count, bool): - names.append(_cap(name, MAX_NAME_BYTES)) + names.append(_clean(name, keys, MAX_NAME_BYTES, options.pii_redaction)) counts.append(count) if names: attributes[USAGE_NAMES] = names @@ -309,7 +309,7 @@ def _response_attributes( event: Dict[str, Any] = {} kind = getattr(warning, "type", None) if isinstance(kind, str): - event[WARNING_TYPE] = _cap(kind, MAX_NAME_BYTES) + event[WARNING_TYPE] = _clean(kind, keys, MAX_NAME_BYTES, options.pii_redaction) message = getattr(warning, "message", None) if isinstance(message, str) and not options.hide_outputs: event[WARNING_MESSAGE] = _clean( From d5af037cf62dbd1f269f1c9335674b7b0a266636 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Tue, 3 Nov 2026 21:30:52 -0800 Subject: [PATCH 2/6] test(parallel): require private input isolation in downstream recipes Keep the exact historical published-distribution control while enforcing the repaired Responses output contract for current source. E2B q1104-parallel-fix-r1-recipe passes 187 cases in each of three lanes. Co-Authored-By: Claude Code --- .../tests/test_perplexity_recipe.py | 36 +++++++++++++++---- 1 file changed, 29 insertions(+), 7 deletions(-) diff --git a/python/examples/perplexity/tests/test_perplexity_recipe.py b/python/examples/perplexity/tests/test_perplexity_recipe.py index 6ab2c93a2..b93530b1a 100644 --- a/python/examples/perplexity/tests/test_perplexity_recipe.py +++ b/python/examples/perplexity/tests/test_perplexity_recipe.py @@ -37,6 +37,19 @@ == Path(CORE_DISTRIBUTION.locate_file("fi_instrumentation/__init__.py")).resolve() ) +# The recipe CI deliberately retains the old published SDK as a compatibility +# control. Only that exact installed distribution exports echoed instructions; +# current source must not reintroduce them to satisfy legacy expectations. +try: + OPENAI_DISTRIBUTION = distribution("traceAI-openai") +except PackageNotFoundError: + LEGACY_OUTPUT = False +else: + LEGACY_OUTPUT = OPENAI_DISTRIBUTION.version == "0.1.10" and ( + Path(traceai_openai.__file__).resolve() + == Path(OPENAI_DISTRIBUTION.locate_file("traceai_openai/__init__.py")).resolve() + ) + TEST_DIR = Path(__file__).resolve().parent RECIPE_DIR = TEST_DIR.parent VENDOR_KEY = "placeholder-perplexity-key" @@ -254,7 +267,10 @@ def test_responses_at_documented_host(tracing): assert raw["usage"]["input_tokens_details"][key] == value assert raw["usage"]["output_tokens_details"] == fake.USAGE["output_tokens_details"] assert raw["usage"]["tool_calls_details"] == fake.USAGE["tool_calls_details"] - assert fake.INSTRUCTIONS_MARKER in raw["instructions"] + if LEGACY_OUTPUT: + assert fake.INSTRUCTIONS_MARKER in raw["instructions"] + else: + assert "instructions" not in raw assert fake.SEARCH_URL in attrs["output.value"] assert fake.SEARCH_SNIPPET in attrs["output.value"] @@ -349,9 +365,11 @@ def handler(request): assert _call(handler, prompt=marker) == fake.ANSWER span, _ = _assert_export(receiver, provider, project, hide_inputs=hidden) assert (marker in json.dumps(span)) is (not hidden) - # Output echoes remain visible even when the input itself is hidden. - for output_marker in (fake.QUERY_MARKER, fake.INSTRUCTIONS_MARKER): - assert output_marker in _attributes(span)["output.value"] + # Search queries are response-owned output; instructions are input. + output = _attributes(span)["output.value"] + assert fake.QUERY_MARKER in output + assert (fake.INSTRUCTIONS_MARKER in output) is LEGACY_OUTPUT + assert fake.ANSWER in output @pytest.mark.parametrize("stream", [False, True], ids=["nonstream", "stream"]) @@ -363,8 +381,11 @@ def test_hide_outputs_with_control(tracing, monkeypatch, stream): span, attrs = _assert_export(receiver, provider, project) _assert_usage(attrs) assert (fake.ANSWER in json.dumps(span)) is (not hidden) - for marker in (fake.SEARCH_URL, fake.SEARCH_SNIPPET, fake.QUERY_MARKER, fake.INSTRUCTIONS_MARKER): + for marker in (fake.SEARCH_URL, fake.SEARCH_SNIPPET, fake.QUERY_MARKER): assert (marker in json.dumps(span)) is (not hidden and not stream) + assert (fake.INSTRUCTIONS_MARKER in json.dumps(span)) is ( + LEGACY_OUTPUT and not hidden and not stream + ) @pytest.mark.parametrize("hidden", [False, True], ids=["visible", "hidden"]) @@ -381,8 +402,9 @@ def test_empty_text_stream_raw_fallback(tracing, monkeypatch, hidden): list(client.responses.create(model=fake.REQUEST_MODEL, input="empty stream", stream=True)) span, attrs = _assert_export(receiver, provider, project) _assert_usage(attrs) - for marker in (fake.SEARCH_URL, fake.QUERY_MARKER, fake.INSTRUCTIONS_MARKER): + for marker in (fake.SEARCH_URL, fake.QUERY_MARKER): assert (marker in json.dumps(span)) is (not hidden) + assert (fake.INSTRUCTIONS_MARKER in json.dumps(span)) is (LEGACY_OUTPUT and not hidden) def test_cache_details_require_output_details(tracing): @@ -457,7 +479,7 @@ def test_subprocess_app(tmp_path, stream, cli_model): assert (fake.SEARCH_URL in json.dumps(attrs)) is (not stream) assert (fake.SEARCH_SNIPPET in json.dumps(attrs)) is (not stream) assert (fake.QUERY_MARKER in json.dumps(attrs)) is (not stream) - assert (fake.INSTRUCTIONS_MARKER in json.dumps(attrs)) is (not stream) + assert (fake.INSTRUCTIONS_MARKER in json.dumps(attrs)) is (LEGACY_OUTPUT and not stream) [export] = receiver.requests() assert export["path"] == "/tracer/v1/traces" assert export["headers"]["x-api-key"] == FI_KEY From c93aadcb4028172a3f51999a460c0bc3f384d3b9 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Tue, 3 Nov 2026 22:04:17 -0800 Subject: [PATCH 3/6] fix(openai): preserve raw response privacy and legacy text Omit opaque Responses parse output from telemetry instead of exporting echoed request fields. Preserve LegacyAPIResponse.text as a string property without changing modern text() methods or application parse results. Add sync/async regressions for scalar parsing, lazy reads, cache identity, and legacy property/parse/close behavior. Co-Authored-By: Claude Code --- .../tests/test_raw_response_contracts.py | 181 ++++++++++++++++++ .../openai/traceai_openai/_request.py | 25 ++- .../openai/traceai_openai/_stream.py | 14 ++ 3 files changed, 212 insertions(+), 8 deletions(-) create mode 100644 python/frameworks/openai/tests/test_raw_response_contracts.py diff --git a/python/frameworks/openai/tests/test_raw_response_contracts.py b/python/frameworks/openai/tests/test_raw_response_contracts.py new file mode 100644 index 000000000..a152153a6 --- /dev/null +++ b/python/frameworks/openai/tests/test_raw_response_contracts.py @@ -0,0 +1,181 @@ +"""Raw-response return types must not change SDK or telemetry privacy contracts.""" + +import asyncio +import json +from contextlib import contextmanager + +import httpx +import openai +import pytest +from fi_instrumentation import TraceConfig +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +from opentelemetry.trace import StatusCode +from traceai_openai import OpenAIInstrumentor + +PRIVATE = "synthetic-private-instructions-not-for-export" + + +def response_body(): + return { + "id": "resp_raw_contract", + "object": "response", + "created_at": 1, + "model": "synthetic", + "status": "completed", + "parallel_tool_calls": True, + "tool_choice": "auto", + "tools": [], + "instructions": PRIVATE, + "output": [], + "error": None, + } + + +@contextmanager +def tracing(instrumented=True): + exporter = InMemorySpanExporter() + provider = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + instrumentor = OpenAIInstrumentor() + if instrumented: + instrumentor.instrument( + tracer_provider=provider, config=TraceConfig(hide_inputs=True) + ) + try: + yield exporter + finally: + if instrumented: + instrumentor.uninstrument() + provider.shutdown() + + +def client(handler, async_=False): + http = (httpx.AsyncClient if async_ else httpx.Client)( + transport=httpx.MockTransport(handler), trust_env=False + ) + return (openai.AsyncOpenAI if async_ else openai.OpenAI)( + api_key="synthetic-raw-contract-key", + base_url="https://synthetic.invalid/v1", + max_retries=0, + http_client=http, + ) + + +@pytest.mark.parametrize("async_", [False, True]) +@pytest.mark.parametrize("parse_type", [str, bytes]) +@pytest.mark.parametrize("body_kind", ["response", "text", "array"]) +def test_scalar_parse_preserves_values_laziness_and_input_privacy( + async_, parse_type, body_kind +): + content = { + "response": json.dumps(response_body()), + "text": PRIVATE, + "array": json.dumps([{"instructions": PRIVATE}]), + }[body_kind].encode() + expected = content if parse_type is bytes else content.decode() + reads = [] + closes = [] + + class SyncBody(httpx.SyncByteStream): + def __iter__(self): + reads.append(1) + yield content + + def close(self): + closes.append(1) + + class AsyncBody(httpx.AsyncByteStream): + async def __aiter__(self): + reads.append(1) + yield content + + async def aclose(self): + closes.append(1) + + def handler(request): + assert json.loads(request.content)["instructions"] == PRIVATE + return httpx.Response( + 200, + headers={"content-type": "application/json"}, + stream=AsyncBody() if async_ else SyncBody(), + ) + + params = {"model": "synthetic", "input": "question", "instructions": PRIVATE} + with tracing() as exporter: + async def run(): + async with client(handler, True) as sdk: + async with sdk.responses.with_streaming_response.create(**params) as raw: + assert not reads + assert not exporter.get_finished_spans() + parsed = await raw.parse(to=parse_type) + assert type(parsed) is parse_type + assert parsed == expected + assert await raw.parse(to=parse_type) is parsed + await raw.close() + + if async_: + asyncio.run(run()) + else: + with client(handler) as sdk: + with sdk.responses.with_streaming_response.create(**params) as raw: + assert not reads + assert not exporter.get_finished_spans() + parsed = raw.parse(to=parse_type) + assert type(parsed) is parse_type + assert parsed == expected + assert raw.parse(to=parse_type) is parsed + raw.close() + + assert len(reads) == 1 + assert closes + (span,) = exporter.get_finished_spans() + assert span.status.status_code is StatusCode.OK + assert PRIVATE not in span.to_json() + # Opaque scalar parses cannot safely be classified as assistant output. + assert "output.value" not in span.attributes + + +@pytest.mark.parametrize("async_", [False, True]) +@pytest.mark.parametrize("instrumented", [False, True]) +def test_legacy_text_remains_a_string_property(async_, instrumented): + body = response_body() + + def handler(request): + return httpx.Response(200, json=body) + + def check(raw): + assert isinstance(raw.text, str) + assert json.loads(raw.text) == body + assert raw.text == raw.http_response.text + parsed = raw.parse() + assert parsed.instructions == PRIVATE + assert raw.parse() is parsed + + with tracing(instrumented) as exporter: + async def run(): + async with client(handler, True) as sdk: + raw = await sdk.responses.with_raw_response.create( + model="synthetic", input="question", instructions=PRIVATE + ) + check(raw) + await raw.http_response.aclose() + await raw.http_response.aclose() + + if async_: + asyncio.run(run()) + else: + with client(handler) as sdk: + raw = sdk.responses.with_raw_response.create( + model="synthetic", input="question", instructions=PRIVATE + ) + check(raw) + raw.http_response.close() + raw.http_response.close() + + spans = exporter.get_finished_spans() + assert len(spans) == int(instrumented) + if instrumented: + assert spans[0].status.status_code is StatusCode.OK + assert PRIVATE not in spans[0].to_json() diff --git a/python/frameworks/openai/traceai_openai/_request.py b/python/frameworks/openai/traceai_openai/_request.py index 775571280..211c01a4e 100644 --- a/python/frameworks/openai/traceai_openai/_request.py +++ b/python/frameworks/openai/traceai_openai/_request.py @@ -35,6 +35,7 @@ from traceai_openai._stream import ( _AsyncRawResponse, _AsyncStream, + _LegacyRawResponse, _RawResponse, _ResponseAccumulator, _Stream, @@ -263,11 +264,16 @@ def on_parse(parsed: Any) -> Any: with_span.set_attribute( SpanAttributes.EMBEDDING_EMBEDDINGS, self._raw_embeddings_json(parsed) ) - telemetry_output = ( - response_output(parsed, self._config) - if cast_to is self._openai.types.responses.response.Response - and isinstance(parsed, dict) else parsed - ) + telemetry_output = parsed + if cast_to is self._openai.types.responses.response.Response: + if isinstance(parsed, Mapping): + telemetry_output = response_output(parsed, self._config) + elif not isinstance(parsed, self._openai.types.responses.response.Response): + # Explicit parse types (including str/bytes) may contain the + # entire envelope, with echoed instructions and tools. Omit + # opaque output rather than bypass the response allowlist; + # never parse again or change the application's return value. + telemetry_output = None add_io_to_span_attributes(with_span, None, telemetry_output, config=self._config) _finish_tracing( status=_response_status(parsed), @@ -283,9 +289,12 @@ def on_parse(parsed: Any) -> Any: if hasattr(response, "parse") and callable(response.parse): # Modern envelopes are deliberately lazy, even for non-SSE responses. # Do not call async parse from sync I/O/attribute extraction paths. - raw_wrapper = ( - _AsyncRawResponse if iscoroutinefunction(response.parse) else _RawResponse - ) + if isinstance(response, LegacyAPIResponse): + raw_wrapper = _LegacyRawResponse + else: + raw_wrapper = ( + _AsyncRawResponse if iscoroutinefunction(response.parse) else _RawResponse + ) raw = raw_wrapper(response, with_span, response_accumulator, on_parse) if isinstance(response, LegacyAPIResponse): # Preserve eager legacy non-stream telemetry. Stream parse caches diff --git a/python/frameworks/openai/traceai_openai/_stream.py b/python/frameworks/openai/traceai_openai/_stream.py index 914e72e2e..521d7d14a 100644 --- a/python/frameworks/openai/traceai_openai/_stream.py +++ b/python/frameworks/openai/traceai_openai/_stream.py @@ -221,6 +221,20 @@ def _finish_tracing(self, status: Optional[trace_api.Status] = None) -> None: self._self_with_span.finish_tracing(status=status) +class _LegacyRawResponse(_RawResponse): + """Legacy envelopes expose text as a property, including on async clients.""" + + @property + def text(self) -> str: + try: + result = self.__wrapped__.text + except BaseException as exc: + _finish_on_exit(self, exc) + raise + self._record_body_consumption() + return result + + class _AsyncRawResponse(_RawResponse): """AsyncAPIResponse has awaitable parse/close; LegacyAPIResponse does not.""" From cf07a03dde2861b47b59d8b43ff1bad6bd6b7a7a Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Tue, 3 Nov 2026 22:53:35 -0800 Subject: [PATCH 4/6] fix(openai): retain terminal status in raw response readers Inspect consumed envelopes independently of scalar parse types, and retain a bounded raw-iterator body for generation status. Leave oversized or opaque generation status unknown rather than reporting success. Preserve SDK caching, lazy reads, exact chunks, and input privacy. Co-Authored-By: Claude Code --- .../tests/test_raw_response_contracts.py | 4 +- .../openai/tests/test_raw_terminal_status.py | 251 ++++++++++++++++++ .../openai/traceai_openai/_request.py | 12 +- .../traceai_openai/_response_accumulator.py | 8 + .../openai/traceai_openai/_stream.py | 76 +++++- 5 files changed, 343 insertions(+), 8 deletions(-) create mode 100644 python/frameworks/openai/tests/test_raw_terminal_status.py diff --git a/python/frameworks/openai/tests/test_raw_response_contracts.py b/python/frameworks/openai/tests/test_raw_response_contracts.py index a152153a6..21e29aeb7 100644 --- a/python/frameworks/openai/tests/test_raw_response_contracts.py +++ b/python/frameworks/openai/tests/test_raw_response_contracts.py @@ -131,7 +131,9 @@ async def run(): assert len(reads) == 1 assert closes (span,) = exporter.get_finished_spans() - assert span.status.status_code is StatusCode.OK + assert span.status.status_code is ( + StatusCode.OK if body_kind == "response" else StatusCode.UNSET + ) assert PRIVATE not in span.to_json() # Opaque scalar parses cannot safely be classified as assistant output. assert "output.value" not in span.attributes diff --git a/python/frameworks/openai/tests/test_raw_terminal_status.py b/python/frameworks/openai/tests/test_raw_terminal_status.py new file mode 100644 index 000000000..64a6b6a4e --- /dev/null +++ b/python/frameworks/openai/tests/test_raw_terminal_status.py @@ -0,0 +1,251 @@ +"""Terminal generation status survives scalar parsing and unbuffered body readers.""" + +import asyncio +import json + +import httpx +import pytest +from opentelemetry.trace import StatusCode + +from .test_raw_response_contracts import PRIVATE, client, response_body, tracing + + +@pytest.mark.parametrize("async_", [False, True]) +@pytest.mark.parametrize("method", ["parse_str", "parse_bytes", "iter_bytes", "iter_text", "iter_lines"]) +@pytest.mark.parametrize("status", ["completed", "failed", "incomplete", "cancelled"]) +def test_unbuffered_terminal_status(async_, method, status): + body = response_body() + body["status"] = status + payload = json.dumps(body, indent=2).encode() + reads = [] + closes = [] + + class SyncBody(httpx.SyncByteStream): + def __iter__(self): + for offset in range(0, len(payload), 7): + reads.append(offset) + yield payload[offset:offset + 7] + + def close(self): + closes.append(1) + + class AsyncBody(httpx.AsyncByteStream): + async def __aiter__(self): + for chunk in SyncBody(): + yield chunk + + async def aclose(self): + closes.append(1) + + def handler(request): + return httpx.Response( + 200, + headers={"content-type": "application/json"}, + stream=AsyncBody() if async_ else SyncBody(), + ) + + params = {"model": "synthetic", "input": "question", "instructions": PRIVATE} + parse_type = str if method == "parse_str" else bytes + + def check_result(result): + if method == "parse_str": + assert type(result) is str and result == payload.decode() + elif method == "parse_bytes": + assert type(result) is bytes and result == payload + elif method == "iter_bytes": + assert result == [payload[offset:offset + 7] for offset in range(0, len(payload), 7)] + elif method == "iter_text": + assert result == [payload[offset:offset + 7].decode() for offset in range(0, len(payload), 7)] + else: + assert result == payload.decode().splitlines() + + with tracing() as exporter: + async def run(): + async with client(handler, True) as sdk: + async with sdk.responses.with_streaming_response.create(**params) as raw: + assert not reads and not exporter.get_finished_spans() + if method.startswith("parse"): + result = await raw.parse(to=parse_type) + assert await raw.parse(to=parse_type) is result + else: + result = [chunk async for chunk in getattr(raw, method)()] + with pytest.raises(httpx.ResponseNotRead): + _ = raw.http_response.content + check_result(result) + await raw.close() + + if async_: + asyncio.run(run()) + else: + with client(handler) as sdk: + with sdk.responses.with_streaming_response.create(**params) as raw: + assert not reads and not exporter.get_finished_spans() + if method.startswith("parse"): + result = raw.parse(to=parse_type) + assert raw.parse(to=parse_type) is result + else: + result = list(getattr(raw, method)()) + with pytest.raises(httpx.ResponseNotRead): + _ = raw.http_response.content + check_result(result) + raw.close() + + assert reads and closes + (span,) = exporter.get_finished_spans() + assert PRIVATE not in span.to_json() + assert "output.value" not in span.attributes + assert span.status.status_code is ( + StatusCode.OK if status == "completed" else StatusCode.ERROR + ) + + +@pytest.mark.parametrize("async_", [False, True]) +@pytest.mark.parametrize("method", ["iter_bytes", "iter_text", "iter_lines"]) +@pytest.mark.parametrize("outcome", ["opaque", "truncated", "oversized", "unread", "partial", "error", "cancel", "sse"]) +def test_raw_iterator_unknown_and_interrupted_consumption(async_, method, outcome): + reads = [] + closes = [] + error = KeyboardInterrupt("synthetic cancel") if outcome == "cancel" else RuntimeError("synthetic failure") + payload = { + "opaque": PRIVATE.encode(), + "truncated": b'{"status": "completed",', + "oversized": json.dumps({"instructions": PRIVATE * 3000, "status": "failed"}).encode(), + }.get(outcome, b'{\n"status": "completed"\n}\n') + + class SyncBody(httpx.SyncByteStream): + def __iter__(self): + reads.append(1) + # The newline lets even the line reader yield before an early exit. + yield payload[:2] + if outcome in {"error", "cancel"}: + raise error + yield payload[2:] + + def close(self): + closes.append(1) + + class AsyncBody(httpx.AsyncByteStream): + async def __aiter__(self): + for chunk in SyncBody(): + yield chunk + + async def aclose(self): + closes.append(1) + + def handler(request): + return httpx.Response( + 200, headers={"content-type": "application/json"}, + stream=AsyncBody() if async_ else SyncBody(), + ) + + params = {"model": "synthetic", "input": "question", "stream": outcome == "sse"} + with tracing() as exporter: + async def run(): + async with client(handler, True) as sdk: + async with sdk.responses.with_streaming_response.create(**params) as raw: + iterator = getattr(raw, method)() + assert not reads and not exporter.get_finished_spans() + if outcome == "partial": + await iterator.__anext__() + elif outcome != "unread": + assert [chunk async for chunk in iterator] + await raw.close() + await iterator.aclose() + + def sync(): + with client(handler) as sdk: + with sdk.responses.with_streaming_response.create(**params) as raw: + iterator = getattr(raw, method)() + assert not reads and not exporter.get_finished_spans() + if outcome == "partial": + next(iterator) + elif outcome != "unread": + assert list(iterator) + raw.close() + iterator.close() + + if outcome in {"error", "cancel"}: + with pytest.raises(type(error)) as caught: + asyncio.run(run()) if async_ else sync() + assert caught.value is error + else: + asyncio.run(run()) if async_ else sync() + assert len(reads) == int(outcome != "unread") + assert closes + (span,) = exporter.get_finished_spans() + assert span.status.status_code is ( + StatusCode.UNSET if outcome in {"opaque", "truncated", "oversized"} else StatusCode.ERROR + ) + assert PRIVATE not in span.to_json() + assert "output.value" not in span.attributes + if outcome in {"error", "cancel"}: + assert len([event for event in span.events if event.name == "exception"]) == 1 + + +def test_raw_body_inspection_is_bounded_and_discards_consumed_content(): + from traceai_openai._stream import _RawBodyBuffer + + buffer = _RawBodyBuffer() + buffer.add(b'{"status": "failed"}') + assert buffer.take() == {"status": "failed"} + assert buffer._body is None + for chunk in (b"x" * 1024, "é" * 1024): + buffer = _RawBodyBuffer() + for _ in range(200): + buffer.add(chunk) + assert buffer._body is None or len(buffer._body) <= buffer._LIMIT + assert buffer.take() is None + + +@pytest.mark.parametrize("async_", [False, True]) +@pytest.mark.parametrize("parse_type", [str, bytes]) +def test_scalar_parse_does_not_repeat_sdk_parse_or_post_parser(async_, parse_type): + body = response_body() + body["status"] = "failed" + calls = [] + post_parses = [] + + def handler(request): + return httpx.Response(200, json=body) + + def post_parser(parsed): + post_parses.append(parsed) + return parsed + + with tracing() as exporter: + async def run(): + async with client(handler, True) as sdk: + async with sdk.responses.with_streaming_response.create(model="synthetic", input="question") as raw: + raw.__wrapped__._options.post_parser = post_parser + original = raw.__wrapped__.parse + + async def parse(*args, **kwargs): + calls.append(1) + return await original(*args, **kwargs) + + raw.__wrapped__.parse = parse + parsed = await raw.parse(to=parse_type) + assert await raw.parse(to=parse_type) is parsed + assert json.loads(parsed) == body + + if async_: + asyncio.run(run()) + else: + with client(handler) as sdk: + with sdk.responses.with_streaming_response.create(model="synthetic", input="question") as raw: + raw.__wrapped__._options.post_parser = post_parser + original = raw.__wrapped__.parse + + def parse(*args, **kwargs): + calls.append(1) + return original(*args, **kwargs) + + raw.__wrapped__.parse = parse + parsed = raw.parse(to=parse_type) + assert raw.parse(to=parse_type) is parsed + assert json.loads(parsed) == body + assert len(calls) == 2 # Exactly the application's two explicit calls. + assert len(post_parses) == 1 # The second call is served from the SDK cache. + (span,) = exporter.get_finished_spans() + assert span.status.status_code is StatusCode.ERROR + assert PRIVATE not in span.to_json() diff --git a/python/frameworks/openai/traceai_openai/_request.py b/python/frameworks/openai/traceai_openai/_request.py index 211c01a4e..4b5f9c495 100644 --- a/python/frameworks/openai/traceai_openai/_request.py +++ b/python/frameworks/openai/traceai_openai/_request.py @@ -28,6 +28,7 @@ _ChatCompletionAccumulator, _CompletionAccumulator, _ResponsesAccumulator, + _raw_response_status, _response_status, ) from traceai_openai._response_attributes_extractor import _ResponseAttributesExtractor @@ -265,6 +266,7 @@ def on_parse(parsed: Any) -> Any: SpanAttributes.EMBEDDING_EMBEDDINGS, self._raw_embeddings_json(parsed) ) telemetry_output = parsed + status = _response_status(parsed) if cast_to is self._openai.types.responses.response.Response: if isinstance(parsed, Mapping): telemetry_output = response_output(parsed, self._config) @@ -274,9 +276,17 @@ def on_parse(parsed: Any) -> Any: # opaque output rather than bypass the response allowlist; # never parse again or change the application's return value. telemetry_output = None + if hasattr(response, "http_response"): + # Inspect only the already-consumed envelope, independently of + # parse(to=...) and post_parser. Never call SDK parse twice. + try: + body = response.http_response.json() + except Exception: + body = None + status = _raw_response_status(body) add_io_to_span_attributes(with_span, None, telemetry_output, config=self._config) _finish_tracing( - status=_response_status(parsed), + status=status, with_span=with_span, has_attributes=_ResponseAttributes( request_parameters=request_parameters, diff --git a/python/frameworks/openai/traceai_openai/_response_accumulator.py b/python/frameworks/openai/traceai_openai/_response_accumulator.py index 3202cac09..cae10697d 100644 --- a/python/frameworks/openai/traceai_openai/_response_accumulator.py +++ b/python/frameworks/openai/traceai_openai/_response_accumulator.py @@ -58,6 +58,14 @@ def _response_status(response: Any) -> Status: return Status(StatusCode.OK) +def _raw_response_status(body: Any) -> Status: + """Raw HTTP success alone does not establish a completed generation.""" + status = body.get("status") if isinstance(body, Mapping) else None + if isinstance(status, str) and status in {"completed", "failed", "cancelled", "incomplete"}: + return _response_status(body) + return Status(StatusCode.UNSET) + + class _ResponsesAccumulator: __slots__ = ( "_config", diff --git a/python/frameworks/openai/traceai_openai/_stream.py b/python/frameworks/openai/traceai_openai/_stream.py index 521d7d14a..841d5bc84 100644 --- a/python/frameworks/openai/traceai_openai/_stream.py +++ b/python/frameworks/openai/traceai_openai/_stream.py @@ -1,10 +1,15 @@ +import json import logging from typing import Any, AsyncIterator, Callable, Generator, Iterator, Optional, Protocol, Tuple from openai.types.chat import ChatCompletionChunk from opentelemetry import trace as trace_api from opentelemetry.util.types import AttributeValue -from traceai_openai._response_accumulator import _response_status +from traceai_openai._response_accumulator import ( + _ResponsesAccumulator, + _raw_response_status, + _response_status, +) from traceai_openai._utils import _finish_tracing from traceai_openai._with_span import _WithSpan from wrapt import ObjectProxy @@ -75,6 +80,42 @@ async def aclose(self) -> None: _finish_on_exit(self._self_stream, error) +class _RawBodyBuffer: + """Bounded inspection of chunks the caller consumed, never an extra HTTP read. + + HTTPX iterators do not populate response.content. Retain at most 64 KiB for + status inspection, then discard it: an oversized/opaque body has unknown + generation status, not an affirmative success. No body is exported. + """ + + _LIMIT = 64 * 1024 + + def __init__(self) -> None: + self._body: Optional[bytearray] = bytearray() + + def add(self, chunk: Any) -> None: + if self._body is None: + return + # Check character count before encoding so a huge text chunk cannot + # allocate an unbounded temporary copy. Then enforce the byte limit too. + remaining = self._LIMIT - len(self._body) + if len(chunk) > remaining: + self._body = None + return + data = chunk.encode("utf-8") if isinstance(chunk, str) else chunk + if len(data) > remaining: + self._body = None + else: + self._body.extend(data) + + def take(self) -> Any: + body, self._body = self._body, None + try: + return json.loads(body) if body is not None else None + except (ValueError, UnicodeError): + return None + + class _RawResponse(ObjectProxy): """Trace every parse-cache entry without eagerly reading a lazy envelope.""" @@ -92,7 +133,16 @@ def __init__( self._self_parsed = {} self._self_body_status: Optional[trace_api.Status] = None - def _record_body_consumption(self) -> None: + def _body_buffer(self) -> Optional[_RawBodyBuffer]: + if ( + isinstance(self._self_response_accumulator, _ResponsesAccumulator) + and not self._self_is_finished + and not getattr(self.__wrapped__, "_is_sse_stream", False) + ): + return _RawBodyBuffer() + return None + + def _record_body_consumption(self, buffer: Optional[_RawBodyBuffer] = None) -> None: # read/text/json consume a non-SSE envelope without calling parse(). # Do not force SDK parsing (or its post_parser) on a raw-body caller. # SSE still needs a terminal event from the traced iterator. @@ -103,10 +153,14 @@ def _record_body_consumption(self) -> None: ): return try: - body = self.__wrapped__.http_response.json() + body = buffer.take() if buffer is not None else self.__wrapped__.http_response.json() except Exception: body = None # Successful raw reads need not contain JSON. - self._self_body_status = _response_status(body) + self._self_body_status = ( + _raw_response_status(body) + if isinstance(self._self_response_accumulator, _ResponsesAccumulator) + else _response_status(body) + ) def _consume_body(self, method: str, *args: Any, **kwargs: Any) -> Any: try: @@ -127,15 +181,20 @@ def json(self, *args: Any, **kwargs: Any) -> Any: return self._consume_body("json", *args, **kwargs) def _consume_iterator(self, method: str, *args: Any, **kwargs: Any) -> Iterator[Any]: + buffer = self._body_buffer() try: for chunk in getattr(self.__wrapped__, method)(*args, **kwargs): + if buffer is not None: + buffer.add(chunk) + if method == "iter_lines": + buffer.add(b"\n") yield chunk except BaseException as exc: _finish_on_exit(self, exc) raise else: # The caller may stop iteration early; only exhaustion consumes it. - self._record_body_consumption() + self._record_body_consumption(buffer) def iter_bytes(self, *args: Any, **kwargs: Any) -> Iterator[bytes]: return self._consume_iterator("iter_bytes", *args, **kwargs) @@ -257,14 +316,19 @@ async def json(self, *args: Any, **kwargs: Any) -> Any: return await self._consume_body("json", *args, **kwargs) async def _consume_iterator(self, method: str, *args: Any, **kwargs: Any) -> AsyncIterator[Any]: + buffer = self._body_buffer() try: async for chunk in getattr(self.__wrapped__, method)(*args, **kwargs): + if buffer is not None: + buffer.add(chunk) + if method == "iter_lines": + buffer.add(b"\n") yield chunk except BaseException as exc: _finish_on_exit(self, exc) raise else: - self._record_body_consumption() + self._record_body_consumption(buffer) async def iter_bytes(self, *args: Any, **kwargs: Any) -> AsyncIterator[bytes]: async for chunk in self._consume_iterator("iter_bytes", *args, **kwargs): From adb8289f996419c205b59fcdb3b205862909873b Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Wed, 4 Nov 2026 03:23:22 -0800 Subject: [PATCH 5/6] fix(openai): enforce raw embedding privacy and finish legacy spans Apply endpoint-based vector-free summaries to mapping parse results and omit opaque embedding output without changing SDK return values or lazy reads. Finish already-buffered legacy spans with unknown status when optional telemetry parsing fails. Add sync/async regression coverage for parse overloads, privacy settings, cache identity, malformed content/text-only consumers, and subsequent SDK parse behavior. Co-Authored-By: Claude Code --- .../test_raw_embedding_and_legacy_errors.py | 188 ++++++++++++++++++ .../tests/test_raw_response_contracts.py | 5 +- .../openai/traceai_openai/_request.py | 20 +- .../openai/traceai_openai/_span_io_handler.py | 27 ++- 4 files changed, 226 insertions(+), 14 deletions(-) create mode 100644 python/frameworks/openai/tests/test_raw_embedding_and_legacy_errors.py diff --git a/python/frameworks/openai/tests/test_raw_embedding_and_legacy_errors.py b/python/frameworks/openai/tests/test_raw_embedding_and_legacy_errors.py new file mode 100644 index 000000000..06e37a23f --- /dev/null +++ b/python/frameworks/openai/tests/test_raw_embedding_and_legacy_errors.py @@ -0,0 +1,188 @@ +"""Endpoint policy and buffered legacy lifetime must survive explicit parse types.""" + +import asyncio +import json +from typing import Any, Dict + +import httpx +import openai +import pytest +from fi_instrumentation import TraceConfig +from opentelemetry.trace import StatusCode +from traceai_openai import set_client_provider + +from .test_raw_response_contracts import client, tracing + + +@pytest.mark.parametrize("async_", [False, True]) +@pytest.mark.parametrize("legacy", [False, True]) +@pytest.mark.parametrize( + "parse_type", + ["default", None, openai.types.CreateEmbeddingResponse, str, bytes, + Dict[str, Any], httpx.Response], +) +@pytest.mark.parametrize("policy", ["hide_vectors", "hide_outputs", "default"]) +def test_embedding_parse_types_preserve_values_and_privacy(async_, legacy, parse_type, policy): + vector = [0.123456789, 0.987654321] + body = { + "object": "list", + "model": "text-embedding-synthetic", + "data": [{"object": "embedding", "index": 0, "embedding": vector}], + "usage": {"prompt_tokens": 3, "total_tokens": 3}, + } + content = json.dumps(body).encode() + reads = [] + closes = [] + + class SyncBody(httpx.SyncByteStream): + def __iter__(self): + reads.append(1) + yield content + + def close(self): + closes.append(1) + + class AsyncBody(httpx.AsyncByteStream): + async def __aiter__(self): + reads.append(1) + yield content + + async def aclose(self): + closes.append(1) + + def handler(request): + assert request.url.path == "/v1/embeddings" + assert json.loads(request.content)["encoding_format"] == "float" + return httpx.Response( + 200, headers={"content-type": "application/json"}, + stream=AsyncBody() if async_ else SyncBody(), + ) + + def check_value(parsed): + if parse_type in ("default", None, openai.types.CreateEmbeddingResponse): + assert isinstance(parsed, openai.types.CreateEmbeddingResponse) + assert parsed.data[0].embedding == vector + elif parse_type is str: + assert type(parsed) is str + assert parsed == content.decode() + elif parse_type is bytes: + assert type(parsed) is bytes + assert parsed == content + elif parse_type is httpx.Response: + assert isinstance(parsed, httpx.Response) + assert parsed.content == content + else: + assert type(parsed) is dict + assert parsed == body + + def parse_legacy(raw): + if parse_type is bytes: + # Unlike modern APIResponse, LegacyAPIResponse rejects bytes. + with pytest.raises(RuntimeError, match="Unsupported type"): + raw.parse(to=bytes) + else: + parsed = raw.parse(**parse_kwargs) + check_value(parsed) + assert raw.parse(**parse_kwargs) is parsed + + config = TraceConfig( + hide_embedding_vectors=policy == "hide_vectors", hide_outputs=policy == "hide_outputs" + ) + params = dict(model=body["model"], input="safe", encoding_format="float") + parse_kwargs = {} if parse_type == "default" else {"to": parse_type} + with tracing(config=config) as exporter: + async def run(): + async with client(handler, True) as sdk: + set_client_provider(sdk, provider="openai") + if legacy: + raw = await sdk.embeddings.with_raw_response.create(**params) + parse_legacy(raw) + else: + async with sdk.embeddings.with_streaming_response.create(**params) as raw: + assert not reads + assert not exporter.get_finished_spans() + parsed = await raw.parse(**parse_kwargs) + check_value(parsed) + assert await raw.parse(**parse_kwargs) is parsed + await raw.close() + + if async_: + asyncio.run(run()) + else: + with client(handler) as sdk: + set_client_provider(sdk, provider="openai") + if legacy: + raw = sdk.embeddings.with_raw_response.create(**params) + parse_legacy(raw) + else: + with sdk.embeddings.with_streaming_response.create(**params) as raw: + assert not reads + assert not exporter.get_finished_spans() + parsed = raw.parse(**parse_kwargs) + check_value(parsed) + assert raw.parse(**parse_kwargs) is parsed + raw.close() + + assert reads == [1] + assert closes + (span,) = exporter.get_finished_spans() + assert span.status.status_code is StatusCode.OK + assert span.attributes["gen_ai.span.kind"] == "EMBEDDING" + if policy != "default": + assert all(str(value) not in span.to_json() for value in vector) + if policy != "hide_outputs": + if legacy or parse_type in ("default", None, openai.types.CreateEmbeddingResponse, Dict[str, Any]): + assert json.loads(span.attributes["output.value"]) == { + "model": body["model"], "embeddings": 1, "dimensions": 2, + } + else: + assert "output.value" not in span.attributes + + +@pytest.mark.parametrize("async_", [False, True]) +@pytest.mark.parametrize("body", [b'{"truncated":', b""]) +@pytest.mark.parametrize("consumer", ["content", "text"]) +@pytest.mark.parametrize("later_parse", [False, True]) +@pytest.mark.parametrize("instrumented", [False, True]) +def test_malformed_legacy_finishes_without_requiring_parse_or_close( + async_, body, consumer, later_parse, instrumented +): + def handler(request): + return httpx.Response(200, content=body, headers={"content-type": "application/json"}) + + with tracing(instrumented) as exporter: + def check(raw): + expected = body if consumer == "content" else body.decode() + assert getattr(raw, consumer) == expected + # Buffered legacy envelopes have no native close() requirement. + assert len(exporter.get_finished_spans()) == int(instrumented) + if later_parse: + # Telemetry's optional failed parse must not replace SDK behavior. + for _ in range(2): + with pytest.raises(json.JSONDecodeError): + raw.parse() + assert raw.parse(to=str) == body.decode() + with pytest.raises(RuntimeError, match="Unsupported type"): + raw.parse(to=bytes) + assert getattr(raw, consumer) == expected + + async def run(): + async with client(handler, True) as sdk: + set_client_provider(sdk, provider="openai") + raw = await sdk.responses.with_raw_response.create(model="synthetic", input="safe") + check(raw) + + if async_: + asyncio.run(run()) + else: + with client(handler) as sdk: + set_client_provider(sdk, provider="openai") + raw = sdk.responses.with_raw_response.create(model="synthetic", input="safe") + check(raw) + + spans = exporter.get_finished_spans() + assert len(spans) == int(instrumented) + if instrumented: + assert spans[0].status.status_code is StatusCode.UNSET + assert "output.value" not in spans[0].attributes + assert not spans[0].events diff --git a/python/frameworks/openai/tests/test_raw_response_contracts.py b/python/frameworks/openai/tests/test_raw_response_contracts.py index 21e29aeb7..6e77565eb 100644 --- a/python/frameworks/openai/tests/test_raw_response_contracts.py +++ b/python/frameworks/openai/tests/test_raw_response_contracts.py @@ -34,14 +34,15 @@ def response_body(): @contextmanager -def tracing(instrumented=True): +def tracing(instrumented=True, config=None): exporter = InMemorySpanExporter() provider = TracerProvider() provider.add_span_processor(SimpleSpanProcessor(exporter)) instrumentor = OpenAIInstrumentor() if instrumented: instrumentor.instrument( - tracer_provider=provider, config=TraceConfig(hide_inputs=True) + tracer_provider=provider, + config=config if config is not None else TraceConfig(hide_inputs=True), ) try: yield exporter diff --git a/python/frameworks/openai/traceai_openai/_request.py b/python/frameworks/openai/traceai_openai/_request.py index 4b5f9c495..8c3912cdb 100644 --- a/python/frameworks/openai/traceai_openai/_request.py +++ b/python/frameworks/openai/traceai_openai/_request.py @@ -32,7 +32,7 @@ _response_status, ) from traceai_openai._response_attributes_extractor import _ResponseAttributesExtractor -from traceai_openai._span_io_handler import add_io_to_span_attributes +from traceai_openai._span_io_handler import _embedding_summary, add_io_to_span_attributes from traceai_openai._stream import ( _AsyncRawResponse, _AsyncStream, @@ -265,8 +265,17 @@ def on_parse(parsed: Any) -> Any: with_span.set_attribute( SpanAttributes.EMBEDDING_EMBEDDINGS, self._raw_embeddings_json(parsed) ) - telemetry_output = parsed + telemetry_output = attributes_response = parsed status = _response_status(parsed) + if cast_to is CreateEmbeddingResponse and not isinstance(parsed, CreateEmbeddingResponse): + # Endpoint policy must not depend on the application's parse type. + # Keep only a vector-free summary for mappings; opaque parse values + # cannot safely be exported. Do not reparse or mutate SDK results. + telemetry_output = ( + _embedding_summary(parsed.get("model"), parsed.get("data")) + if isinstance(parsed, Mapping) else None + ) + attributes_response = telemetry_output if cast_to is self._openai.types.responses.response.Response: if isinstance(parsed, Mapping): telemetry_output = response_output(parsed, self._config) @@ -290,7 +299,7 @@ def on_parse(parsed: Any) -> Any: with_span=with_span, has_attributes=_ResponseAttributes( request_parameters=request_parameters, - response=parsed, + response=attributes_response, response_attributes_extractor=extractor, ), ) @@ -315,6 +324,11 @@ def on_parse(parsed: Any) -> Any: # Optional eager telemetry must not change SDK behavior: callers # still receive the envelope and may inspect malformed bodies. logger.debug("Could not eagerly parse raw response for telemetry", exc_info=True) + if not getattr(response, "_is_sse_stream", False): + # This body is already buffered, and legacy callers need + # not parse or close the envelope. End once with an unknown + # outcome, without exporting an instrumentation exception. + with_span.finish_tracing(status=trace_api.Status(trace_api.StatusCode.UNSET)) else: raw._trace_parsed(parsed) return raw diff --git a/python/frameworks/openai/traceai_openai/_span_io_handler.py b/python/frameworks/openai/traceai_openai/_span_io_handler.py index 772db2da3..a59508cbc 100644 --- a/python/frameworks/openai/traceai_openai/_span_io_handler.py +++ b/python/frameworks/openai/traceai_openai/_span_io_handler.py @@ -1,6 +1,6 @@ import json import logging -from typing import Any, Optional +from typing import Any, Dict, Mapping, Optional from fi_instrumentation import TraceConfig from fi_instrumentation.fi_types import FiMimeTypeValues, SpanAttributes @@ -90,6 +90,22 @@ def _set_mime_type(span: _WithSpan, key: str, value: Any) -> None: span.set_attribute(key, mime) +def _embedding_summary(model: Any, data: Any) -> Dict[str, Any]: + """Summarize typed or mapping embeddings without copying or exporting vectors.""" + data = data if isinstance(data, list) else [] + first = data[0] if data else None + vector = ( + first.get("embedding") if isinstance(first, Mapping) + else getattr(first, "embedding", None) + ) + summary: Dict[str, Any] = {"embeddings": len(data)} + if isinstance(model, str): + summary["model"] = model + if isinstance(vector, list): + summary["dimensions"] = len(vector) + return summary + + def _json_output_for(output_data: Any, config: Optional[TraceConfig] = None) -> Optional[str]: """JSON for responses whose output is not a chat message: the Responses API object as JSON, and a compact summary for embeddings (never the vectors).""" @@ -106,14 +122,7 @@ def _json_output_for(output_data: Any, config: Optional[TraceConfig] = None) -> body = {"output_text": output_text, **body} return json.dumps(body, ensure_ascii=False) if isinstance(output_data, CreateEmbeddingResponse): - data = list(output_data.data or []) - first = getattr(data[0], "embedding", None) if data else None - summary = { - "model": output_data.model, - "embeddings": len(data), - "dimensions": len(first) if isinstance(first, list) else None, - } - return json.dumps({k: v for k, v in summary.items() if v is not None}) + return json.dumps(_embedding_summary(output_data.model, output_data.data)) return None From fe63ace65427b79613da5542cf3376e1e517c78f Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Wed, 4 Nov 2026 18:27:15 -0800 Subject: [PATCH 6/6] fix(openai): preserve mixed MCP and assistant stream output Retain sanitized response-owned MCP items alongside the assistant answer. Cloud regressions and independent verification cover floor/current SDKs, privacy and stream terminal states. Co-Authored-By: Claude Code --- .../tests/test_responses_mixed_mcp_output.py | 251 ++++++++++++++++++ .../traceai_openai/_response_accumulator.py | 18 +- 2 files changed, 264 insertions(+), 5 deletions(-) create mode 100644 python/frameworks/openai/tests/test_responses_mixed_mcp_output.py diff --git a/python/frameworks/openai/tests/test_responses_mixed_mcp_output.py b/python/frameworks/openai/tests/test_responses_mixed_mcp_output.py new file mode 100644 index 000000000..6e7dabdda --- /dev/null +++ b/python/frameworks/openai/tests/test_responses_mixed_mcp_output.py @@ -0,0 +1,251 @@ +"""Mixed MCP/assistant Responses retain output through real SDKs and OTLP.""" + +import asyncio +import base64 +import json +import sys +from pathlib import Path + +import httpx +import pytest +from fi_instrumentation import TraceConfig +from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +from opentelemetry.trace import StatusCode +from traceai_openai import OpenAIInstrumentor + +from .test_responses_privacy_and_raw_streams import KEY, body, client, sse + +sys.path.insert(0, str(Path(__file__).resolve().parents[3] / "tests")) +from harness import Receiver, _flatten_attributes + +ANSWER = "Mixed MCP assistant answer" +INSTRUCTION = "mixed-private-instruction" +MCP_AUTH = "synthetic-mixed-mcp-authorization" +MCP_HEADER = "synthetic-mixed-mcp-header" +TOOL_ERROR = "mixed-private-tool-error" +TOP_ERROR = "mixed-private-top-level-error" +TOOL = { + "type": "mcp", "server_label": "synthetic", + "server_url": "https://synthetic.invalid/mcp", "require_approval": "never", + "authorization": MCP_AUTH, "headers": {"X-Synthetic-Secret": MCP_HEADER}, +} + + +@pytest.fixture(scope="module") +def receiver(): + with Receiver() as receiver: + yield receiver + + +def mixed_body(status="completed", answer_first=False): + response = body(status) + response["instructions"] = INSTRUCTION + response["tools"] = [TOOL] + response["error"] = {"code": "server_error", "message": TOP_ERROR} + response["incomplete_details"] = {"reason": "max_output_tokens"} + calls = [ + { + "id": "mcp_mixed_" + str(i), "type": "mcp_call", "name": "lookup", + "server_label": "synthetic", "arguments": json.dumps({"city": "city-" + str(i)}), + "output": "tool-result-" + str(i) + " " + KEY + " " + MCP_AUTH + " " + MCP_HEADER, + "error": {"type": "http_error", "code": 500, "message": TOOL_ERROR} if i else None, + } + for i in range(2) + ] + message = { + "type": "message", "id": "msg_mixed", "role": "assistant", "status": "completed", + "content": [{"type": "output_text", "text": ANSWER, "annotations": []}], + } + response["output"] = [message, *calls] if answer_first else [*calls, message] + return response + + +def collect(receiver, response, *, async_, stream=True, policy="default", early=False): + receiver.clear() + provider = TracerProvider() + memory = InMemorySpanExporter() + provider.add_span_processor(SimpleSpanProcessor(memory)) + provider.add_span_processor(SimpleSpanProcessor(OTLPSpanExporter(endpoint=receiver.endpoint))) + config = TraceConfig(**({policy: True} if policy != "default" else {})) + instrumentor = OpenAIInstrumentor() + instrumentor.instrument(tracer_provider=provider, config=config) + seen = [] + + def handler(request): + assert request.url.host == "api.parallel.ai" + params = json.loads(request.content) + assert params["tools"] == [TOOL] + assert params["instructions"] == INSTRUCTION + seen.append(params) + if stream: + created = {**response, "status": "in_progress", "output": []} + terminal = "response." + response["status"] + if response["status"] == "cancelled": + terminal = "response.completed" # Body status must still prevent success. + return httpx.Response(200, headers={"content-type": "text/event-stream"}, + content=sse("response.created", created) + sse(terminal, response)) + return httpx.Response(200, json=response) + + params = dict(model="parallel", input="question", instructions=INSTRUCTION, tools=[TOOL], stream=stream) + + def sync_call(): + with client(handler) as sdk: + result = sdk.responses.create(**params) + if stream: + if early: + assert next(iter(result)).type == "response.created" + result.close() + result.close() + return None + events = list(result) + result.close() + return events[-1].response + return result + + async def async_call(): + async with client(handler, True) as sdk: + result = await sdk.responses.create(**params) + if stream: + if early: + assert (await result.__aiter__().__anext__()).type == "response.created" + await result.close() + await result.close() + return None + events = [event async for event in result] + await result.close() + return events[-1].response + return result + + try: + with provider.get_tracer("mixed-mcp-test").start_as_current_span("workflow") as parent: + result = asyncio.run(async_call()) if async_ else sync_call() + provider.force_flush() + spans = memory.get_finished_spans() + assert len(spans) == 2 + span = next(s for s in spans if s.name != "workflow") + assert span.parent.span_id == parent.get_span_context().span_id + assert span.attributes["gen_ai.provider.name"] == "parallel" + assert span.attributes["gen_ai.request.model"] == "parallel" + assert len(seen) == 1 + wire = receiver.spans() + assert len(wire) == 2 + exported = next(s for s in wire if s["name"] != "workflow") + assert base64.b64decode(exported["spanId"]).hex() == format(span.context.span_id, "016x") + assert base64.b64decode(exported["parentSpanId"]).hex() == format(parent.get_span_context().span_id, "016x") + attrs = _flatten_attributes(exported["attributes"]) + if early: + assert "output.value" not in attrs + else: + assert attrs["output.value"] == span.attributes["output.value"] + return result, span, exported, config + finally: + instrumentor.uninstrument() + provider.shutdown() + + +@pytest.mark.parametrize("async_", [False, True]) +@pytest.mark.parametrize("stream", [False, True]) +@pytest.mark.parametrize("answer_first", [False, True]) +@pytest.mark.parametrize("policy", ["default", "hide_inputs", "hide_outputs", "hide_error_details"]) +def test_mixed_output_retains_distinct_calls_and_privacy(receiver, async_, stream, answer_first, policy): + response = mixed_body(answer_first=answer_first) + result, span, exported, config = collect(receiver, response, async_=async_, stream=stream, policy=policy) + # Neither the wire request nor the SDK response is sanitized for the application. + assert result.instructions == INSTRUCTION + calls = [item for item in result.output if item.type == "mcp_call"] + assert [item.id for item in calls] == ["mcp_mixed_0", "mcp_mixed_1"] + for item in calls: + assert KEY in item.output and MCP_AUTH in item.output and MCP_HEADER in item.output + assert TOOL_ERROR in result.model_dump_json() + assert span.status.status_code is StatusCode.OK + assert exported["status"]["code"] == "STATUS_CODE_OK" + wire = json.dumps(exported) + for secret in (KEY, MCP_AUTH, MCP_HEADER, TOP_ERROR): + assert secret not in wire + if policy == "hide_inputs": + assert INSTRUCTION not in wire + if policy == "hide_outputs": + assert span.attributes["output.value"] == "__REDACTED__" + for text in (ANSWER, "tool-result-", "city-", "mcp_mixed_", TOOL_ERROR): + assert text not in wire + return + output = json.loads(span.attributes["output.value"]) + assert next(iter(output)) == "output_text" + assert output["output_text"] == ANSWER # Preserve the answer-first preview. + assert span.attributes["output.mime_type"] == "application/json" + for request_field in ("instructions", "input", "tools", "error"): + assert request_field not in output + captured = [item for item in output["output"] if item["type"] == "mcp_call"] + assert [item["id"] for item in captured] == ["mcp_mixed_0", "mcp_mixed_1"] + for i, item in enumerate(captured): + assert item["name"] == "lookup" + assert json.loads(item["arguments"]) == {"city": "city-" + str(i)} + assert item["output"].startswith("tool-result-" + str(i)) + assert item["server_label"] == "synthetic" + assert (TOOL_ERROR not in wire) is config.hide_error_details + if config.hide_error_details: + assert captured[1]["error"] == {"type": "http_error", "code": 500} + else: + assert captured[1]["error"]["message"] == TOOL_ERROR + + +@pytest.mark.parametrize("async_", [False, True]) +@pytest.mark.parametrize("kind", ["mcp_list_tools", "mcp_approval_request", "mcp_approval_response"]) +@pytest.mark.parametrize("hide_errors", [False, True]) +def test_mixed_mcp_metadata_items_survive(receiver, async_, kind, hide_errors): + response = mixed_body() + item = {"id": "mcp_metadata", "type": kind} + if kind == "mcp_list_tools": + item.update(server_label="synthetic", error=TOOL_ERROR, + tools=[{"name": "lookup", "input_schema": {"type": "object"}}]) + elif kind == "mcp_approval_request": + item.update(server_label="synthetic", name="lookup", arguments='{"city":"city-0"}') + else: + item.update(approval_request_id="mcp_approval", approve=True, reason="approved in fixture") + response["output"] = [item, response["output"][-1]] + result, span, exported, _ = collect( + receiver, response, async_=async_, policy="hide_error_details" if hide_errors else "default" + ) + assert result.output[0].id == "mcp_metadata" + captured = json.loads(span.attributes["output.value"])["output"][0] + expected = dict(item) + if kind == "mcp_list_tools" and hide_errors: + expected.pop("error") + assert TOOL_ERROR not in json.dumps(exported) + assert TOOL_ERROR in result.model_dump_json() + assert captured == expected + assert span.status.status_code is StatusCode.OK + + +@pytest.mark.parametrize("async_", [False, True]) +@pytest.mark.parametrize("status", ["failed", "incomplete", "cancelled"]) +def test_mixed_output_does_not_turn_non_success_into_success(receiver, async_, status): + result, span, exported, _ = collect(receiver, mixed_body(status), async_=async_, policy="hide_error_details") + assert result.status == status + assert span.status.status_code is StatusCode.ERROR + assert exported["status"]["code"] == "STATUS_CODE_ERROR" + output = json.loads(span.attributes["output.value"]) + assert output["status"] == status + assert "tool-result-0" in span.attributes["output.value"] + assert TOOL_ERROR not in json.dumps(exported) + + +@pytest.mark.parametrize("async_", [False, True]) +def test_text_only_stream_keeps_plain_answer(receiver, async_): + response = mixed_body() + response["output"] = response["output"][-1:] + _, span, _, _ = collect(receiver, response, async_=async_) + assert span.attributes["output.value"] == ANSWER + assert span.status.status_code is StatusCode.OK + + +@pytest.mark.parametrize("async_", [False, True]) +def test_interrupted_mixed_stream_finishes_once_without_inventing_output(receiver, async_): + result, span, exported, _ = collect(receiver, mixed_body(), async_=async_, early=True) + assert result is None + assert span.status.status_code is StatusCode.ERROR + assert exported["status"]["code"] == "STATUS_CODE_ERROR" + assert "tool-result-" not in json.dumps(exported) diff --git a/python/frameworks/openai/traceai_openai/_response_accumulator.py b/python/frameworks/openai/traceai_openai/_response_accumulator.py index cae10697d..cd82f5f38 100644 --- a/python/frameworks/openai/traceai_openai/_response_accumulator.py +++ b/python/frameworks/openai/traceai_openai/_response_accumulator.py @@ -132,13 +132,21 @@ def get_attributes(self) -> Iterator[Tuple[str, AttributeValue]]: if output_items: output_content = " ".join(output_items) - if output_content: + if output_content and not any( + getattr(item, "type", None) in { + "mcp_call", "mcp_list_tools", "mcp_approval_request", "mcp_approval_response" + } + for item in result.output + ): yield SpanAttributes.OUTPUT_VALUE, output_content else: - # Keep response-owned data, never echoed request fields. - yield from _as_output_attributes( - _io_value_and_type(response_output(result, self._config)), - ) + # Text alone would discard MCP results in a mixed response. + # Keep response-owned data, never echoed request fields, and lead + # with the answer just as non-streaming Responses previews do. + output = response_output(result, self._config) + if output_content: + output = {"output_text": output_content, **output} + yield from _as_output_attributes(_io_value_and_type(output)) def get_extra_attributes(self) -> Iterator[Tuple[str, AttributeValue]]: if not (result := self._result()):