Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions python/frameworks/openai/CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,3 +1,13 @@
## [Unreleased]
### Fixed
- Embeddings respect `hide_inputs` for indexed input text and `hide_outputs` for both raw and indexed vectors, including with older fi-core releases.
- Chat streams used with `with` / `async with` retain the traced iterator and finish once on exhaustion or early context exit; context-body errors mark an unfinished span as ERROR.
- Streamed and failed Chat Completions spans now carry `gen_ai.request.model` (the requested model); a successful non-streamed response still records the served model.
- `gen_ai.provider.name` follows the client's `base_url` host for a small, explicit set of OpenAI-compatible vendors (Azure, Google, Vertex AI, DeepSeek, xAI, Mistral, Anthropic, Baseten, DashScope, Databricks, Lepton, Perplexity, OrcaRouter, Anannas, Doubleword). Unknown hosts, proxies and loopback servers keep `openai`. Previously the label was always `openai`, even for Azure and Google hosts.
- Responses API: non-streamed `output.value` is valid JSON with `output.mime_type=application/json` (it was Python `str(dict)`) and leads with `output_text`; a string `input` is no longer JSON-quoted; text-only assistant messages are a flat `message.content`, so trace views show the answer instead of `{}`.
- Embeddings: the span records the embedding model in `gen_ai.request.model`; `output.value` is a compact JSON summary (model, count, dimensions) instead of the vectors; `FI_HIDE_EMBEDDING_VECTORS` now also strips vectors from the raw `embedding.embeddings` JSON; items without `index` (e.g. Gemini's OpenAI-compatible API) are no longer skipped. Token attributes stay omitted when the provider returns no usage.
- `input.mime_type` / `output.mime_type` match the value actually recorded.

## [0.1.8] - 2025-07-08
### Feature
- Introduced compatibility with OpenAI's latest response method
Expand Down
Empty file.
365 changes: 365 additions & 0 deletions python/frameworks/openai/tests/test_openai_compatible_spans.py

Large diffs are not rendered by default.

75 changes: 75 additions & 0 deletions python/frameworks/openai/tests/test_stream_context_manager.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
"""The SDK context manager must never unwrap the traced iterator."""
import asyncio

import pytest
from opentelemetry.trace import StatusCode

from .test_openai_compatible_spans import (
REQUEST_MODEL, async_client, chat_stream, client, make_tracer, only_span,
)


@pytest.mark.parametrize("mode", ["full", "early", "empty", "error"])
def test_sync_stream_context_finishes_once(make_tracer, mode):
exporter = make_tracer()
with client(lambda request: chat_stream()) as sdk:
stream = sdk.chat.completions.create(
model=REQUEST_MODEL, messages=[{"role": "user", "content": "hi"}], stream=True,
)
try:
with stream as entered:
assert entered is stream
if mode != "empty":
for _ in entered:
if mode == "error":
raise ValueError("consumer stopped")
if mode == "early":
break
except ValueError as error:
assert str(error) == "consumer stopped"
assert stream.response.is_closed
span, attrs = only_span(exporter)
assert attrs["gen_ai.request.model"] == REQUEST_MODEL
assert attrs["gen_ai.provider.name"] == "baseten"
assert span.status.status_code == (StatusCode.ERROR if mode == "error" else StatusCode.OK)
if mode == "full":
assert attrs["output.value"] == "Tokyo"
if mode == "early":
assert attrs["output.value"] == "Tok"
if mode == "error":
assert any(event.name == "exception" for event in span.events)


@pytest.mark.parametrize("mode", ["full", "early", "empty", "error"])
def test_async_stream_context_finishes_once(make_tracer, mode):
exporter = make_tracer()

async def run():
async with async_client(lambda request: chat_stream()) as sdk:
stream = await sdk.chat.completions.create(
model=REQUEST_MODEL, messages=[{"role": "user", "content": "hi"}], stream=True,
)
try:
async with stream as entered:
assert entered is stream
if mode != "empty":
async for _ in entered:
if mode == "error":
raise ValueError("consumer stopped")
if mode == "early":
break
except ValueError as error:
assert str(error) == "consumer stopped"
assert stream.response.is_closed

asyncio.run(run())
span, attrs = only_span(exporter)
assert attrs["gen_ai.request.model"] == REQUEST_MODEL
assert attrs["gen_ai.provider.name"] == "baseten"
assert span.status.status_code == (StatusCode.ERROR if mode == "error" else StatusCode.OK)
if mode == "full":
assert attrs["output.value"] == "Tokyo"
if mode == "early":
assert attrs["output.value"] == "Tok"
if mode == "error":
assert any(event.name == "exception" for event in span.events)
4 changes: 2 additions & 2 deletions python/frameworks/openai/traceai_openai/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -55,12 +55,12 @@ def _instrument(self, **kwargs: Any) -> None:
wrap_function_wrapper(
module=_MODULE,
name="OpenAI.request",
wrapper=_Request(tracer=tracer, openai=openai),
wrapper=_Request(tracer=tracer, openai=openai, config=config),
)
wrap_function_wrapper(
module=_MODULE,
name="AsyncOpenAI.request",
wrapper=_AsyncRequest(tracer=tracer, openai=openai),
wrapper=_AsyncRequest(tracer=tracer, openai=openai, config=config),
)
if Protect is not None:
self._original_protect = Protect.protect
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -534,7 +534,17 @@ def _get_attributes_from_response_output_message(
prefix: str = "",
) -> Iterator[Tuple[str, AttributeValue]]:
yield f"{prefix}{MessageAttributes.MESSAGE_ROLE}", obj.role
yield from cls._get_attributes_from_output_message_content_list(obj.content, prefix)
from openai.types import responses

content = list(obj.content or [])
if content and all(isinstance(item, responses.ResponseOutputText) for item in content):
# Text-only answers are one flat message.content, the same shape Chat
# Completions use, so trace viewers render them as the assistant text.
yield f"{prefix}{MessageAttributes.MESSAGE_CONTENT}", "".join(
item.text for item in content
)
return
yield from cls._get_attributes_from_output_message_content_list(content, prefix)

@classmethod
@stop_on_exception
Expand Down
57 changes: 57 additions & 0 deletions python/frameworks/openai/traceai_openai/_providers.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
"""Map an OpenAI-compatible client's base_url host to gen_ai.provider.name.

The mapping is deliberately small and explicit: a host is only labelled with a
vendor when it is that vendor's documented API host. Everything else (proxies,
loopback servers, self-hosted endpoints, look-alike domains) keeps the
instrumentor's historical label, "openai", rather than a guess.
"""

from typing import Optional

from fi_instrumentation.fi_types import FiLLMProviderValues

DEFAULT_PROVIDER = FiLLMProviderValues.OPENAI.value

# Exact API hosts.
_HOSTS = {
"api.openai.com": FiLLMProviderValues.OPENAI.value,
"generativelanguage.googleapis.com": FiLLMProviderValues.GOOGLE.value,
"api.anthropic.com": FiLLMProviderValues.ANTHROPIC.value,
"api.mistral.ai": FiLLMProviderValues.MISTRALAI.value,
"api.deepseek.com": FiLLMProviderValues.DEEPSEEK.value,
"api.x.ai": FiLLMProviderValues.XAI.value,
"inference.baseten.co": "baseten",
"dashscope.aliyuncs.com": "dashscope",
"dashscope-intl.aliyuncs.com": "dashscope",
"dashscope-us.aliyuncs.com": "dashscope",
"api.perplexity.ai": "perplexity",
"api.orcarouter.ai": "orcarouter",
"api.anannas.ai": "anannas",
"anannas.ai": "anannas",
"api.doubleword.ai": "doubleword",
}

# Vendors that give every resource, workspace or endpoint its own subdomain.
_SUFFIXES = (
(".openai.azure.com", FiLLMProviderValues.AZURE.value),
(".services.ai.azure.com", FiLLMProviderValues.AZURE.value),
(".cognitiveservices.azure.com", FiLLMProviderValues.AZURE.value),
("-aiplatform.googleapis.com", FiLLMProviderValues.VERTEXAI.value),
(".maas.aliyuncs.com", "dashscope"),
(".cloud.databricks.com", "databricks"),
(".gcp.databricks.com", "databricks"),
(".azuredatabricks.net", "databricks"),
(".xenon.lepton.run", "lepton"),
)


def provider_from_host(host: Optional[str]) -> str:
if not isinstance(host, str) or not host:
return DEFAULT_PROVIDER
host = host.lower().rstrip(".")
if host in _HOSTS:
return _HOSTS[host]
for suffix, provider in _SUFFIXES:
if host.endswith(suffix):
return provider
return DEFAULT_PROVIDER
68 changes: 43 additions & 25 deletions python/frameworks/openai/traceai_openai/_request.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,10 @@
from contextlib import contextmanager
from itertools import chain
from types import ModuleType
from typing import Any, Awaitable, Callable, Iterable, Iterator, Mapping, Tuple
from typing import Any, Awaitable, Callable, Iterable, Iterator, Mapping, Optional, Tuple

from fi_instrumentation import get_attributes_from_context
from fi_instrumentation import TraceConfig, get_attributes_from_context
from fi_instrumentation.fi_types import (
FiLLMProviderValues,
FiLLMSystemValues,
FiSpanKindValues,
SpanAttributes,
)
Expand All @@ -18,6 +16,7 @@
from opentelemetry.context import _SUPPRESS_INSTRUMENTATION_KEY
from opentelemetry.trace import INVALID_SPAN
from opentelemetry.util.types import AttributeValue
from traceai_openai._providers import provider_from_host
from traceai_openai._request_attributes_extractor import _RequestAttributesExtractor
from traceai_openai._response_accumulator import (
_ChatCompletionAccumulator,
Expand Down Expand Up @@ -89,15 +88,23 @@ class _WithOpenAI(ABC):
"_request_attributes_extractor",
"_response_attributes_extractor",
"_response_accumulator_factories",
"_config",
)

def __init__(self, openai: ModuleType, *args: Any, **kwargs: Any) -> None:
def __init__(
self,
openai: ModuleType,
*args: Any,
config: Optional[TraceConfig] = None,
**kwargs: Any,
) -> None:
super().__init__(*args, **kwargs)
self._openai = openai
self._config = config if config is not None else TraceConfig()
self._stream_types = (openai.Stream, openai.AsyncStream)
self._request_attributes_extractor = _RequestAttributesExtractor(openai=openai)
self._response_attributes_extractor = _ResponseAttributesExtractor(
openai=openai
openai=openai, config=self._config
)

def responses_accumulator(request_parameters: _RequestParameters) -> Any:
Expand Down Expand Up @@ -133,26 +140,23 @@ def _get_span_kind(self, cast_to: type) -> str:
def _get_attributes_from_instance(
self, instance: Any
) -> Iterator[Tuple[str, AttributeValue]]:
if (
not (base_url := getattr(instance, "base_url", None))
or not (host := getattr(base_url, "host", None))
or not isinstance(host, str)
):
return
if host.endswith("api.openai.com"):
yield SpanAttributes.GEN_AI_PROVIDER_NAME, FiLLMProviderValues.OPENAI.value
elif host.endswith("openai.azure.com"):
yield SpanAttributes.GEN_AI_PROVIDER_NAME, FiLLMProviderValues.AZURE.value
elif host.endswith("googleapis.com"):
yield SpanAttributes.GEN_AI_PROVIDER_NAME, FiLLMProviderValues.GOOGLE.value
# The provider is the vendor behind the client's base_url, not the SDK:
# known OpenAI-compatible hosts get their own label, anything else "openai".
base_url = getattr(instance, "base_url", None)
yield SpanAttributes.GEN_AI_PROVIDER_NAME, provider_from_host(
getattr(base_url, "host", None)
)

def _get_attributes_from_request(
self,
cast_to: type,
request_parameters: Mapping[str, Any],
) -> Iterator[Tuple[str, AttributeValue]]:
yield SpanAttributes.GEN_AI_SPAN_KIND, self._get_span_kind(cast_to=cast_to)
yield SpanAttributes.GEN_AI_PROVIDER_NAME, FiLLMSystemValues.OPENAI.value
# Record the requested model up front so streamed and failed calls carry it;
# a successful non-streamed response overwrites it with the served model.
if isinstance(model := request_parameters.get("model"), str) and model:
yield SpanAttributes.GEN_AI_REQUEST_MODEL, model
try:
yield from _as_input_attributes(
_io_value_and_type(request_parameters),
Expand Down Expand Up @@ -243,6 +247,18 @@ def _finalize_response(
)
return response

def _raw_embeddings_json(self, response: CreateEmbeddingResponse) -> str:
"""The raw embeddings response; vectors are replaced by their dimension
count when FI_HIDE_EMBEDDING_VECTORS / hide_embedding_vectors is set."""
raw = response.to_dict()
if self._config.hide_embedding_vectors or self._config.hide_outputs:
for item in raw.get("data") or []:
if isinstance(item, dict) and "embedding" in item:
vector = item.pop("embedding")
if isinstance(vector, list):
item["dimensions"] = len(vector)
return safe_json_dumps(raw)

def response_to_dict(self, response: Any) -> Any:
if hasattr(response, "to_dict") and callable(response.to_dict):
return response.to_dict()
Expand Down Expand Up @@ -302,9 +318,9 @@ def __call__(
response = wrapped(*args, **kwargs)

if isinstance(response, CreateEmbeddingResponse):
embedding = response.to_dict()
with_span.set_attribute(
SpanAttributes.EMBEDDING_EMBEDDINGS, safe_json_dumps(embedding)
SpanAttributes.EMBEDDING_EMBEDDINGS,
self._raw_embeddings_json(response),
)

# Add output data to span attributes after getting response
Expand Down Expand Up @@ -344,8 +360,10 @@ async def __call__(
try:
cast_to, request_parameters = _parse_request_args(args)
# Extract input data more robustly
input_data = request_parameters.get(
"messages", request_parameters.get("prompt")
input_data = (
request_parameters.get("messages")
or request_parameters.get("prompt")
or request_parameters.get("input")
)
llm_tools = request_parameters.get("tools")

Expand Down Expand Up @@ -377,9 +395,9 @@ async def __call__(
response = await wrapped(*args, **kwargs)

if isinstance(response, CreateEmbeddingResponse):
embedding = response.to_dict()
with_span.set_attribute(
SpanAttributes.EMBEDDING_EMBEDDINGS, safe_json_dumps(embedding)
SpanAttributes.EMBEDDING_EMBEDDINGS,
self._raw_embeddings_json(response),
)

add_io_to_span_attributes(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
Type,
)

from fi_instrumentation import TraceConfig
from fi_instrumentation.fi_types import (
EmbeddingAttributes,
MessageAttributes,
Expand Down Expand Up @@ -49,15 +50,17 @@
class _ResponseAttributesExtractor:
__slots__ = (
"_openai",
"_config",
"_chat_completion_type",
"_completion_type",
"_create_embedding_response_type",
"_images_response_type",
"_responses_type",
)

def __init__(self, openai: ModuleType) -> None:
def __init__(self, openai: ModuleType, config: Optional[TraceConfig] = None) -> None:
self._openai = openai
self._config = config if config is not None else TraceConfig()
self._chat_completion_type: Type["ChatCompletion"] = (
openai.types.chat.ChatCompletion
)
Expand Down Expand Up @@ -163,13 +166,22 @@ def _get_attributes_from_create_embedding_response(
# See https://github.com/openai/openai-python/blob/f1c7d714914e3321ca2e72839fe2d132a8646e7f/src/openai/types/embedding_create_params.py#L23 # noqa: E501
if model := getattr(response, "model"):
yield f"{SpanAttributes.EMBEDDING_MODEL_NAME}", model
# Also the span's model, so trace views and cost lookup see it.
yield SpanAttributes.GEN_AI_REQUEST_MODEL, model
if (data := getattr(response, "data", None)) and isinstance(data, Iterable):
for embedding in data:
if (index := getattr(embedding, "index", None)) is None:
continue
for key, value in self._get_attributes_from_embedding(embedding):
yield f"{SpanAttributes.EMBEDDING_EMBEDDINGS}.{index}.{key}", value
for position, embedding in enumerate(data):
# Some OpenAI-compatible servers (e.g. Gemini's) omit `index`;
# fall back to the item's position instead of dropping it.
index = getattr(embedding, "index", None)
if index is None:
index = position
if not (self._config.hide_outputs or self._config.hide_embedding_vectors):
for key, value in self._get_attributes_from_embedding(embedding):
yield f"{SpanAttributes.EMBEDDING_EMBEDDINGS}.{index}.{key}", value

# Embedding text is input too; older fi-core releases only mask input.value.
if self._config.hide_inputs:
return
embedding_input = request_parameters.get("input")
for index, text in enumerate(_get_texts(embedding_input, model)):
# FIXME: this step should move to request attributes extractor if decoding is not necessary.# noqa: E501
Expand Down
Loading
Loading