diff --git a/python/frameworks/openai/tests/test_chat_input_value.py b/python/frameworks/openai/tests/test_chat_input_value.py new file mode 100644 index 000000000..55fc50a98 --- /dev/null +++ b/python/frameworks/openai/tests/test_chat_input_value.py @@ -0,0 +1,272 @@ +"""Canonical chat input must retain the full conversation.""" + +import asyncio +from copy import deepcopy +import json + +import openai +import pytest +from fi_instrumentation import TraceConfig +from fi_instrumentation.fi_types import SpanAttributes +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +from opentelemetry.trace import StatusCode +from traceai_openai import OpenAIInstrumentor + +if int(openai.__version__.split(".")[0]) >= 3: + import httpx2 as httpx +else: + import httpx + + +@pytest.fixture +def tracing(): + exporter = InMemorySpanExporter() + provider = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + instrumentor = OpenAIInstrumentor() + try: + instrumentor.instrument(tracer_provider=provider, config=TraceConfig()) + yield exporter + finally: + instrumentor.uninstrument() + provider.shutdown() + + +@pytest.mark.parametrize("async_", [False, True], ids=["sync", "async"]) +@pytest.mark.parametrize("stream", [False, True], ids=["ordinary", "streaming"]) +@pytest.mark.parametrize( + "messages, expected_value, expected_mime", + [ + pytest.param( + [ + {"role": "system", "content": "SYSTEM_CONTEXT_MARKER"}, + {"role": "user", "content": "USER_QUESTION_MARKER"}, + ], + "SYSTEM_CONTEXT_MARKER \n USER_QUESTION_MARKER", + "text/plain", + id="system-user", + ), + pytest.param( + [ + {"role": "system", "content": "SYSTEM_CONTEXT_MARKER"}, + {"role": "user", "content": "USER_QUESTION_MARKER"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_synthetic", + "type": "function", + "function": {"name": "lookup", "arguments": "{}"}, + } + ], + }, + { + "role": "tool", + "tool_call_id": "call_synthetic", + "content": "TOOL_RESULT_MARKER", + }, + { + "role": "user", + "content": [ + {"type": "text", "text": "FOLLOWUP_MARKER café"}, + {"type": "text", "text": "SECOND_TEXT_PART_MARKER"}, + { + "type": "image_url", + "image_url": { + "url": "https://synthetic.invalid/example.png", + "detail": "low", + }, + }, + ], + }, + ], + " \n ".join([ + "SYSTEM_CONTEXT_MARKER", "USER_QUESTION_MARKER", "None", + "TOOL_RESULT_MARKER", "FOLLOWUP_MARKER café", + "SECOND_TEXT_PART_MARKER", "{{" + f"{SpanAttributes.INPUT_IMAGES}.0" + "}}", + ]), + "text/plain", + id="multiturn-multimodal", + ), + pytest.param( + [{"role": "system", "content": "SYSTEM_CONTEXT_MARKER"}] + [ + message + for i in range(40) + for message in ( + {"role": "user", "content": f"QUESTION_{i}"}, + { + "role": "assistant", + "content": None, + "tool_calls": [{ + "id": f"call_{i}", + "type": "function", + "function": { + "name": "lookup", + "arguments": json.dumps({"query": f"item_{i}"}), + }, + }], + }, + {"role": "tool", "tool_call_id": f"call_{i}", "content": f"RESULT_{i}"}, + {"role": "assistant", "content": f"ANSWER_{i}"}, + ) + ], + " \n ".join(["SYSTEM_CONTEXT_MARKER"] + [ + text for i in range(40) + for text in (f"QUESTION_{i}", "None", f"RESULT_{i}", f"ANSWER_{i}") + ]), + "text/plain", + id="conversation-exceeds-attribute-count", + ), + pytest.param( + [{"role": "user", "content": [ + {"type": "image_url", "image_url": {"url": "https://synthetic.invalid/a.png"}}, + {"type": "image_url", "image_url": {"url": "https://synthetic.invalid/b.png"}}, + ]}], + " \n ".join("{{" + f"{SpanAttributes.INPUT_IMAGES}.{i}" + "}}" for i in range(2)), + "text/plain", + id="images-only", + ), + pytest.param( + [{"role": "user", "content": [{ + "type": "input_audio", "input_audio": {"data": "AAECAw==", "format": "wav"}, + }]}], + "[non-text content]", + "text/plain", + id="audio-only", + ), + pytest.param( + [{"role": "user", "content": [{ + "type": "file", "file": {"file_data": "AAECAw==", "filename": "sample.bin"}, + }]}], + "[non-text content]", + "text/plain", + id="file-only", + ), + pytest.param( + [{"role": "user", "content": '{"task": "sum"}'}], + '{"task": "sum"}', + "application/json", + id="single-json-message", + ), + pytest.param( + [ + {"role": "user", "content": '{"task": "sum"}'}, + {"role": "user", "content": "now explain it"}, + ], + '{"task": "sum"} \n now explain it', + "text/plain", + id="json-followed-by-text", + ), + ], +) +def test_chat_input_value_preserves_full_conversation( + tracing, async_, stream, messages, expected_value, expected_mime, +): + original = deepcopy(messages) + requests = [] + + def handle(request): + body = json.loads(request.content) + requests.append(body) + assert body["messages"] == original + assert body["stream"] is stream + response = { + "id": "chatcmpl_synthetic", + "object": "chat.completion", + "created": 1, + "model": "gpt-4o-mini", + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "ANSWER_MARKER"}, + "finish_reason": "stop", + } + ], + } + if stream: + response["object"] = "chat.completion.chunk" + choice = response["choices"][0] + choice["delta"] = choice.pop("message") + return httpx.Response( + 200, + headers={"content-type": "text/event-stream"}, + content=f"data: {json.dumps(response)}\n\ndata: [DONE]\n\n", + ) + return httpx.Response(200, json=response) + + params = {"model": "gpt-4o-mini", "messages": messages, "stream": stream} + + async def run_async(): + async with openai.AsyncOpenAI( + api_key="synthetic-test-key", + base_url="https://synthetic.invalid/v1", + max_retries=0, + http_client=httpx.AsyncClient( + transport=httpx.MockTransport(handle), trust_env=False + ), + ) as client: + result = await client.chat.completions.create(**params) + if stream: + parts = [chunk.choices[0].delta.content or "" async for chunk in result] + return "".join(parts) + return result.choices[0].message.content + + if async_: + answer = asyncio.run(run_async()) + else: + with openai.OpenAI( + api_key="synthetic-test-key", + base_url="https://synthetic.invalid/v1", + max_retries=0, + http_client=httpx.Client( + transport=httpx.MockTransport(handle), trust_env=False + ), + ) as client: + result = client.chat.completions.create(**params) + if stream: + answer = "".join(chunk.choices[0].delta.content or "" for chunk in result) + else: + answer = result.choices[0].message.content + + assert answer == "ANSWER_MARKER" + assert len(requests) == 1 + assert messages == original + (span,) = tracing.get_finished_spans() + assert span.status.status_code is StatusCode.OK + attrs = span.attributes + assert attrs["input.value"] == expected_value + assert attrs["input.value"].strip() + assert attrs["input.mime_type"] == expected_mime + assert json.loads(attrs[SpanAttributes.GEN_AI_INPUT_MESSAGES]) == original + if len(messages) in (2, 5) and messages[0]["role"] == "system": + assert attrs["gen_ai.input.messages.1.message.content"] == "USER_QUESTION_MARKER" + image_urls = [ + part["image_url"]["url"] + for message in messages if isinstance(message.get("content"), list) + for part in message["content"] if part["type"] == "image_url" + ] + if image_urls: + assert json.loads(attrs[SpanAttributes.INPUT_IMAGES]) == image_urls + + +@pytest.mark.parametrize( + "flag", ["hide_inputs", "hide_input_messages", "hide_input_text", "hide_input_images"], +) +def test_aggregate_messages_respect_input_privacy(flag): + from types import SimpleNamespace + from traceai_openai._request import _WithOpenAI + + request = SimpleNamespace( + _config=TraceConfig(**{flag: True}), + _openai=openai, + _get_span_kind=lambda **kwargs: "LLM", + ) + attributes = dict(_WithOpenAI._get_attributes_from_request( + request, + openai.types.chat.ChatCompletion, + {"model": "gpt-4o-mini", "messages": [{"role": "user", "content": "PRIVATE_INPUT"}]}, + )) + assert SpanAttributes.GEN_AI_INPUT_MESSAGES not in attributes diff --git a/python/frameworks/openai/tests/test_chat_usage_details.py b/python/frameworks/openai/tests/test_chat_usage_details.py new file mode 100644 index 000000000..5e7439498 --- /dev/null +++ b/python/frameworks/openai/tests/test_chat_usage_details.py @@ -0,0 +1,209 @@ +"""Chat usage details reach spans without changing SDK results.""" + +import asyncio +import json + +import httpx +import openai +import pytest +from fi_instrumentation import TraceConfig +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter + +from traceai_openai import OpenAIInstrumentor + + +@pytest.fixture +def exporter(): + exporter = InMemorySpanExporter() + provider = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + instrumentor = OpenAIInstrumentor() + instrumentor.instrument(tracer_provider=provider, config=TraceConfig()) + try: + yield exporter + finally: + instrumentor.uninstrument() + provider.shutdown() + + +@pytest.mark.parametrize("use_async", [False, True], ids=["sync", "async"]) +@pytest.mark.parametrize("stream", [False, True], ids=["nonstream", "stream"]) +@pytest.mark.parametrize( + "details,cached,reasoning", + [ + pytest.param( + { + "prompt_tokens_details": {"cached_tokens": 80}, + "completion_tokens_details": {"reasoning_tokens": 10}, + }, + 80, + 10, + id="nonzero", + ), + pytest.param( + { + "prompt_tokens_details": {"cached_tokens": 0}, + "completion_tokens_details": {"reasoning_tokens": 0}, + }, + 0, + 0, + id="zero", + ), + pytest.param({}, None, None, id="missing-details"), + pytest.param( + {"prompt_tokens_details": None, "completion_tokens_details": None}, + None, + None, + id="null-details", + ), + pytest.param( + {"prompt_tokens_details": {}, "completion_tokens_details": {}}, + None, + None, + id="missing-counts", + ), + pytest.param( + { + "prompt_tokens_details": {"cached_tokens": None}, + "completion_tokens_details": {"reasoning_tokens": None}, + }, + None, + None, + id="null-counts", + ), + pytest.param( + {"prompt_tokens_details": {"cached_tokens": 80}}, + 80, + None, + id="cache-only", + ), + pytest.param( + {"completion_tokens_details": {"reasoning_tokens": 10}}, + None, + 10, + id="reasoning-only", + ), + ], +) +def test_chat_usage_details(exporter, use_async, stream, details, cached, reasoning): + usage = { + "prompt_tokens": 100, + "completion_tokens": 20, + "total_tokens": 120, + **details, + } + common = {"id": "chatcmpl-test", "created": 1, "model": "gpt-4o-mini"} + + def handler(_request): + if not stream: + return httpx.Response( + 200, + json={ + **common, + "object": "chat.completion", + "choices": [{ + "index": 0, + "finish_reason": "stop", + "message": {"role": "assistant", "content": "Hello"}, + }], + "usage": usage, + }, + ) + chunks = [ + { + **common, + "object": "chat.completion.chunk", + "choices": [{ + "index": 0, + "delta": {"role": "assistant", "content": "Hello"}, + "finish_reason": None, + }], + "usage": None, + }, + { + **common, + "object": "chat.completion.chunk", + "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}], + "usage": None, + }, + { + **common, + "object": "chat.completion.chunk", + "choices": [], + "usage": usage, + }, + ] + body = "".join(f"data: {json.dumps(chunk)}\n\n" for chunk in chunks) + return httpx.Response( + 200, + headers={"content-type": "text/event-stream"}, + content=(body + "data: [DONE]\n\n").encode(), + ) + + kwargs = { + "model": "gpt-4o-mini", + "messages": [{"role": "user", "content": "hello"}], + "stream": stream, + } + if stream: + kwargs["stream_options"] = {"include_usage": True} + client_kwargs = { + "api_key": "test-only-key", + "base_url": "https://api.openai.com/v1", + "max_retries": 0, + } + + async def run_async(): + async with openai.AsyncOpenAI( + **client_kwargs, + http_client=httpx.AsyncClient( + transport=httpx.MockTransport(handler), trust_env=False + ), + ) as client: + response = await client.chat.completions.create(**kwargs) + return [chunk async for chunk in response] if stream else response + + if use_async: + result = asyncio.run(run_async()) + else: + with openai.OpenAI( + **client_kwargs, + http_client=httpx.Client( + transport=httpx.MockTransport(handler), trust_env=False + ), + ) as client: + response = client.chat.completions.create(**kwargs) + result = list(response) if stream else response + + if stream: + assert len(result) == 3 + assert result[-1].choices == [] + assert "".join( + choice.delta.content or "" for chunk in result for choice in chunk.choices + ) == "Hello" + sdk_usage = result[-1].usage + else: + assert result.choices[0].message.content == "Hello" + sdk_usage = result.usage + assert sdk_usage.prompt_tokens == 100 + assert sdk_usage.completion_tokens == 20 + assert sdk_usage.total_tokens == 120 + assert getattr(sdk_usage.prompt_tokens_details, "cached_tokens", None) == cached + assert getattr(sdk_usage.completion_tokens_details, "reasoning_tokens", None) == reasoning + + (span,) = exporter.get_finished_spans() + expected = { + "gen_ai.usage.input_tokens": 100, + "gen_ai.usage.output_tokens": 20, + "gen_ai.usage.total_tokens": 120, + } + if cached is not None: + expected["gen_ai.usage.input_tokens.cache_read"] = cached + if reasoning is not None: + expected["gen_ai.usage.output_tokens.reasoning"] = reasoning + assert { + key: value for key, value in span.attributes.items() + if key.startswith("gen_ai.usage.") + } == expected diff --git a/python/frameworks/openai/traceai_openai/_request.py b/python/frameworks/openai/traceai_openai/_request.py index 2f87156b5..111cdc6e3 100644 --- a/python/frameworks/openai/traceai_openai/_request.py +++ b/python/frameworks/openai/traceai_openai/_request.py @@ -205,6 +205,16 @@ def _get_attributes_from_request( if isinstance(model := request_parameters.get("model"), str) and model: yield SpanAttributes.GEN_AI_REQUEST_MODEL, model try: + if ( + cast_to is self._openai.types.chat.ChatCompletion + and isinstance(messages := request_parameters.get("messages"), list) + and not ( + self._config.hide_inputs or self._config.hide_input_messages + or self._config.hide_input_text or self._config.hide_input_images + ) + ): + # Keep the complete conversation primary so indexed details cannot evict it. + yield SpanAttributes.GEN_AI_INPUT_MESSAGES, safe_json_dumps(messages) yield from _as_input_attributes( _io_value_and_type(request_parameters), ) diff --git a/python/frameworks/openai/traceai_openai/_response_accumulator.py b/python/frameworks/openai/traceai_openai/_response_accumulator.py index 820779055..01a6df777 100644 --- a/python/frameworks/openai/traceai_openai/_response_accumulator.py +++ b/python/frameworks/openai/traceai_openai/_response_accumulator.py @@ -257,6 +257,14 @@ def get_attributes(self) -> Iterator[Tuple[str, AttributeValue]]: yield SpanAttributes.GEN_AI_USAGE_INPUT_TOKENS, prompt_tokens if completion_tokens is not None: yield SpanAttributes.GEN_AI_USAGE_OUTPUT_TOKENS, completion_tokens + prompt_details = usage.get("prompt_tokens_details") + if isinstance(prompt_details, Mapping): + if (cached_tokens := prompt_details.get("cached_tokens")) is not None: + yield SpanAttributes.GEN_AI_USAGE_INPUT_TOKENS_CACHE_READ, cached_tokens + completion_details = usage.get("completion_tokens_details") + if isinstance(completion_details, Mapping): + if (reasoning_tokens := completion_details.get("reasoning_tokens")) is not None: + yield SpanAttributes.GEN_AI_USAGE_OUTPUT_TOKENS_REASONING, reasoning_tokens # A tool-only completion has no text, but still has useful output. if not output_value and (result := self._result()): diff --git a/python/frameworks/openai/traceai_openai/_response_attributes_extractor.py b/python/frameworks/openai/traceai_openai/_response_attributes_extractor.py index 39331b6b5..604198c7e 100644 --- a/python/frameworks/openai/traceai_openai/_response_attributes_extractor.py +++ b/python/frameworks/openai/traceai_openai/_response_attributes_extractor.py @@ -311,6 +311,12 @@ def _get_attributes_from_completion_usage( yield SpanAttributes.GEN_AI_USAGE_INPUT_TOKENS, prompt_tokens if (completion_tokens := getattr(usage, "completion_tokens", None)) is not None: yield SpanAttributes.GEN_AI_USAGE_OUTPUT_TOKENS, completion_tokens + prompt_details = getattr(usage, "prompt_tokens_details", None) + if (cached_tokens := getattr(prompt_details, "cached_tokens", None)) is not None: + yield SpanAttributes.GEN_AI_USAGE_INPUT_TOKENS_CACHE_READ, cached_tokens + completion_details = getattr(usage, "completion_tokens_details", None) + if (reasoning_tokens := getattr(completion_details, "reasoning_tokens", None)) is not None: + yield SpanAttributes.GEN_AI_USAGE_OUTPUT_TOKENS_REASONING, reasoning_tokens def _get_attributes_from_embedding_usage( self, diff --git a/python/frameworks/openai/traceai_openai/_span_io_handler.py b/python/frameworks/openai/traceai_openai/_span_io_handler.py index 6898f4b23..7f7c35e0d 100644 --- a/python/frameworks/openai/traceai_openai/_span_io_handler.py +++ b/python/frameworks/openai/traceai_openai/_span_io_handler.py @@ -22,6 +22,7 @@ def _process_input_data(input_data: Any, span: _WithSpan) -> None: input_content = [] input_images = [] eval_input = [] + has_non_text_content = False for msg in input_data: if isinstance(msg, dict): msg_content = msg.get("content", "") @@ -32,7 +33,9 @@ def _process_input_data(input_data: Any, span: _WithSpan) -> None: if item.get("type") == "text": filtered_content.append(item) eval_input.append(item.get("text", "")) - elif item.get("type") == "image_url": + else: + has_non_text_content = True + if item.get("type") == "image_url": url_data = item.get("image_url", {}) url = url_data.get("url") if url: @@ -58,8 +61,10 @@ def _process_input_data(input_data: Any, span: _WithSpan) -> None: if input_images: images_value = json.dumps(input_images, ensure_ascii=False) span.set_attribute(SpanAttributes.INPUT_IMAGES, images_value) - if eval_input: - eval_input_str = " \n ".join(map(str, eval_input)) + eval_input_str = " \n ".join(map(str, eval_input)) + if not eval_input_str.strip() and has_non_text_content: + eval_input_str = "[non-text content]" + if eval_input or has_non_text_content: span.set_attribute(SpanAttributes.INPUT_VALUE, eval_input_str) # Describe the value actually exported (all messages joined), not the first message. _set_mime_type(span, SpanAttributes.INPUT_MIME_TYPE, eval_input_str)