From c8eeccfae2810c80f321b8c065baacf05cc11126 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Mon, 2 Nov 2026 05:39:13 -0800 Subject: [PATCH 1/2] fix(truefoundry): reject incomplete streamed answers and close streams Co-Authored-By: Claude Code --- .../gateways/truefoundry/client_example.py | 24 ++++-- .../tests/test_stream_completion.py | 85 +++++++++++++++++++ 2 files changed, 102 insertions(+), 7 deletions(-) create mode 100644 examples/gateways/truefoundry/tests/test_stream_completion.py diff --git a/examples/gateways/truefoundry/client_example.py b/examples/gateways/truefoundry/client_example.py index bf88e34a5..64314b4aa 100755 --- a/examples/gateways/truefoundry/client_example.py +++ b/examples/gateways/truefoundry/client_example.py @@ -80,17 +80,27 @@ def agent(user_prompt: str) -> str: def run_stream(client: OpenAI, model: str, prompt: str) -> str: - """One streamed completion. The LLM span ends when the stream is fully read.""" + """Return a complete answer, closing the stream on success or failure.""" parts = [] - stream = client.chat.completions.create( + finish_reason = None + with client.chat.completions.create( model=model, messages=[{"role": "user", "content": prompt}], stream=True, stream_options={"include_usage": True}, - ) - for chunk in stream: - if chunk.choices and chunk.choices[0].delta.content: - parts.append(chunk.choices[0].delta.content) + ) as stream: + for chunk in stream: + for choice in chunk.choices: + if choice.index != 0: + continue + if choice.delta.content: + parts.append(choice.delta.content) + if choice.finish_reason is not None: + finish_reason = choice.finish_reason + # EOF and [DONE] alone do not prove the model finished its answer. + # Token limits and content filtering also produce partial answers. + if finish_reason != "stop": + raise RuntimeError("The gateway did not return a complete answer") return "".join(parts) @@ -119,7 +129,7 @@ def main(argv: list) -> int: code = 0 try: print(run_stream(client, model, prompt) if stream else run_agent(client, tracer, model, prompt)) - except Exception as exc: # the failed gateway call is already recorded as an ERROR span + except Exception as exc: print(f"gateway call failed: {type(exc).__name__}", file=sys.stderr) code = 1 finally: diff --git a/examples/gateways/truefoundry/tests/test_stream_completion.py b/examples/gateways/truefoundry/tests/test_stream_completion.py new file mode 100644 index 000000000..55ca6803a --- /dev/null +++ b/examples/gateways/truefoundry/tests/test_stream_completion.py @@ -0,0 +1,85 @@ +"""The normal application must not print a partial answer as success.""" + +import json +from unittest.mock import Mock + +import pytest + +import client_example +from test_client_tracing import MODEL, httpx + + +@pytest.mark.parametrize( + "ending", ["eof", "done", "usage", "empty", "stop", "length", "content_filter"] +) +def test_stream_main_requires_complete_answer(monkeypatch, capsys, ending): + chunk = { + "id": "synthetic-stream", + "object": "chat.completion.chunk", + "created": 1, + "model": MODEL, + "choices": [ + {"index": 0, "delta": {"content": "answer text"}, "finish_reason": None} + ], + } + payload = "" if ending == "empty" else "data: " + json.dumps(chunk) + "\n\n" + if ending in {"stop", "length", "content_filter"}: + chunk["choices"] = [{"index": 0, "delta": {}, "finish_reason": ending}] + payload += "data: " + json.dumps(chunk) + "\n\n" + if ending in {"usage", "stop"}: + chunk["choices"] = [] + chunk["usage"] = {"prompt_tokens": 2, "completion_tokens": 2, "total_tokens": 4} + payload += "data: " + json.dumps(chunk) + "\n\n" + if ending in {"done", "stop"}: + payload += "data: [DONE]\n\n" + responses = [] + + def handler(request): + response = httpx.Response( + 200, + headers={"content-type": "text/event-stream"}, + stream=httpx.ByteStream(payload.encode()), + ) + responses.append(response) + return response + + provider = Mock() + with client_example.OpenAI( + api_key="synthetic", + max_retries=0, + http_client=httpx.Client(transport=httpx.MockTransport(handler)), + ) as client: + monkeypatch.setattr(client_example, "build", lambda: (provider, client, None)) + code = client_example.main(["client_example.py", "--stream", "question"]) + output = capsys.readouterr() + assert code == (0 if ending == "stop" else 1) + assert output.out.strip() == ("answer text" if ending == "stop" else "") + assert bool(output.err) == (ending != "stop") + assert responses[0].is_closed + provider.force_flush.assert_called_once_with() + provider.shutdown.assert_called_once_with() + + +@pytest.mark.parametrize( + "error", [RuntimeError("synthetic failure"), KeyboardInterrupt()] +) +def test_stream_cleanup_preserves_iteration_error(error): + class InterruptedStream: + closed = False + + def __enter__(self): + return self + + def __exit__(self, *args): + self.closed = True + + def __iter__(self): + raise error + + stream = InterruptedStream() + client = Mock() + client.chat.completions.create.return_value = stream + with pytest.raises(type(error)) as caught: + client_example.run_stream(client, MODEL, "question") + assert caught.value is error + assert stream.closed From 01fc0721d10dd6e6f9fdec1020371dee821ccc1f Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Wed, 4 Nov 2026 08:26:28 -0800 Subject: [PATCH 2/2] docs(truefoundry): qualify incomplete-stream span status Limit ERROR status guidance to instrumented SDK and transport failures. Explain that application completeness validation cannot change the finished OK span with the existing published pins. Exercise the real entry point and loopback OTLP for all stream endings while retaining output, closure, flush and shutdown assertions. Co-Authored-By: Claude Code --- .../tests/test_stream_completion.py | 60 ++++++++++++++++--- .../integrations/gateways/truefoundry.mdx | 4 +- 2 files changed, 55 insertions(+), 9 deletions(-) diff --git a/examples/gateways/truefoundry/tests/test_stream_completion.py b/examples/gateways/truefoundry/tests/test_stream_completion.py index 55ca6803a..002f479a1 100644 --- a/examples/gateways/truefoundry/tests/test_stream_completion.py +++ b/examples/gateways/truefoundry/tests/test_stream_completion.py @@ -4,15 +4,30 @@ from unittest.mock import Mock import pytest +from traceai_openai import OpenAIInstrumentor import client_example from test_client_tracing import MODEL, httpx +from test_page_snippet import PRIVACY_FLAGS, attributes, decode_request, receiver @pytest.mark.parametrize( "ending", ["eof", "done", "usage", "empty", "stop", "length", "content_filter"] ) -def test_stream_main_requires_complete_answer(monkeypatch, capsys, ending): +def test_stream_main_requires_complete_answer(monkeypatch, capsys, receiver, ending): + endpoint, received = receiver + for name, value in { + "FI_BASE_URL": endpoint, + "FI_API_KEY": "synthetic-fi-key", + "FI_SECRET_KEY": "synthetic-fi-secret", + "TFY_API_KEY": "synthetic-tfy-token", + "TFY_MODEL": MODEL, + "TFY_BASE_URL": "https://gateway.truefoundry.ai", + "OTEL_EXPORTER_OTLP_TRACES_COMPRESSION": "none", + }.items(): + monkeypatch.setenv(name, value) + for flag in PRIVACY_FLAGS: + monkeypatch.setenv(flag, "true") chunk = { "id": "synthetic-stream", "object": "chat.completion.chunk", @@ -43,22 +58,51 @@ def handler(request): responses.append(response) return response - provider = Mock() - with client_example.OpenAI( - api_key="synthetic", - max_retries=0, - http_client=httpx.Client(transport=httpx.MockTransport(handler)), - ) as client: - monkeypatch.setattr(client_example, "build", lambda: (provider, client, None)) + class MockedOpenAI(client_example.OpenAI): + def __init__(self, **kwargs): + super().__init__( + http_client=httpx.Client(transport=httpx.MockTransport(handler)), + max_retries=0, + **kwargs, + ) + + original_build = client_example.build + providers = [] + + def build(): + provider, client, tracer = original_build() + provider = Mock(wraps=provider) + providers.append(provider) + return provider, client, tracer + + monkeypatch.setattr(client_example, "OpenAI", MockedOpenAI) + monkeypatch.setattr(client_example, "build", build) + try: code = client_example.main(["client_example.py", "--stream", "question"]) + finally: + OpenAIInstrumentor().uninstrument() output = capsys.readouterr() assert code == (0 if ending == "stop" else 1) assert output.out.strip() == ("answer text" if ending == "stop" else "") assert bool(output.err) == (ending != "stop") assert responses[0].is_closed + (provider,) = providers provider.force_flush.assert_called_once_with() provider.shutdown.assert_called_once_with() + (batch,) = received + path, _, _, request = decode_request(batch) + assert path == "/tracer/v1/traces" + (resource,) = request.resource_spans + (scope,) = resource.scope_spans + (span,) = scope.spans + assert span.name == "ChatCompletion" + # The pinned instrumentor finishes on stream exhaustion, before the app + # validates finish_reason. An OK span does not prove answer completeness. + assert span.status.code == 1 # OTLP StatusCode.OK, including rejected answers + if ending == "stop": + assert attributes(span.attributes)["gen_ai.usage.total_tokens"] == 4 + @pytest.mark.parametrize( "error", [RuntimeError("synthetic failure"), KeyboardInterrupt()] diff --git a/src/pages/docs/integrations/gateways/truefoundry.mdx b/src/pages/docs/integrations/gateways/truefoundry.mdx index ef73bd666..0e7fbb967 100644 --- a/src/pages/docs/integrations/gateways/truefoundry.mdx +++ b/src/pages/docs/integrations/gateways/truefoundry.mdx @@ -88,7 +88,9 @@ Open **Observe**, select `my-chatbot`, and open the trace. For an agent with a t | Tokens | Input, output and total counts when the response supplies `usage`. For streaming, request `stream_options={"include_usage": True}` and consume the stream to completion. | | Provider | `openai`, the instrumentor's fallback for this OpenAI-compatible host. This is not a TrueFoundry label or evidence of which upstream provider handled the request. | | Prompt, response, tool calls | Captured by default; controlled by the settings below. | -| Error | Status `ERROR`. Error details are captured unless hidden; the error type remains available when details are hidden. | +| SDK or transport error | Status `ERROR` for failures raised by the instrumented OpenAI call. Error details are captured unless hidden; the error type remains available when details are hidden. | + +With the pinned versions, `client_example.py --stream` validates answer completeness after the instrumentor has finished the LLM span. An incomplete answer makes the example exit with an error and print no partial answer, but the span remains `OK`. Span success alone does not prove that a streamed answer is complete. A response model can differ from the gateway model id you requested. Gateway-side `tfy.*` fields, including routing and cost, are not added to these client spans. Instrument the client only once; adding another OpenAI instrumentor can create duplicate spans.