diff --git a/.release-please-manifest.json b/.release-please-manifest.json index bf1f1d012..31ac60b24 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -5,82 +5,82 @@ "python/frameworks/agent-framework": "0.1.0", "python/frameworks/agentscope": "0.1.0", "python/frameworks/agno": "0.2.2", - "python/frameworks/anthropic": "0.1.13", - "python/frameworks/autogen": "0.2.1", - "python/frameworks/bedrock": "0.1.11", + "python/frameworks/anthropic": "0.1.14", + "python/frameworks/autogen": "0.2.2", + "python/frameworks/bedrock": "0.1.12", "python/frameworks/beeai": "0.1.0", - "python/frameworks/cerebras": "0.1.1", + "python/frameworks/cerebras": "0.1.2", "python/frameworks/chromadb": "0.2.0", - "python/frameworks/claude-agent-sdk": "0.1.1", - "python/frameworks/cohere": "0.1.1", + "python/frameworks/claude-agent-sdk": "0.1.2", + "python/frameworks/cohere": "0.1.2", "python/frameworks/cognee": "0.1.1", "python/frameworks/crewai": "0.1.11", - "python/frameworks/deepseek": "0.1.2", + "python/frameworks/deepseek": "0.1.3", "python/frameworks/discoveryengine": "0.1.1", - "python/frameworks/dspy": "0.1.9", + "python/frameworks/dspy": "0.1.10", "python/frameworks/elevenlabs-agents": "0.1.0", "python/frameworks/exa": "0.1.0", "python/frameworks/firecrawl": "0.1.1", - "python/frameworks/fireworks": "0.1.2", + "python/frameworks/fireworks": "0.1.3", "python/frameworks/google-adk": "0.1.7", - "python/frameworks/google-genai": "0.1.3", + "python/frameworks/google-genai": "0.1.4", "python/frameworks/google-genai-live": "0.1.0", - "python/frameworks/groq": "0.1.10", + "python/frameworks/groq": "0.1.11", "python/frameworks/guardrails": "0.2.2", "python/frameworks/haystack": "0.1.11", "python/frameworks/hermes-agent/hermes-plugin/futureagi": "0.1.1", "python/frameworks/hermes-agent": "0.1.1", - "python/frameworks/huggingface": "0.1.0", - "python/frameworks/instructor": "0.2.1", + "python/frameworks/huggingface": "0.1.1", + "python/frameworks/instructor": "0.2.2", "python/frameworks/lancedb": "0.1.2", - "python/frameworks/langchain": "0.2.2", - "python/frameworks/litellm": "0.1.11", + "python/frameworks/langchain": "0.2.3", + "python/frameworks/litellm": "0.1.12", "python/frameworks/livekit": "0.2.1", "python/frameworks/llama_index": "0.1.11", "python/frameworks/mcp": "0.1.4", "python/frameworks/mcp-use": "0.1.1", "python/frameworks/milvus": "0.1.2", - "python/frameworks/minimax": "0.1.0", - "python/frameworks/mistralai": "0.1.10", + "python/frameworks/minimax": "0.1.1", + "python/frameworks/mistralai": "0.1.11", "python/frameworks/mongodb-vector": "0.1.2", - "python/frameworks/ollama": "0.1.0", + "python/frameworks/ollama": "0.1.1", "python/frameworks/openwebui": "0.1.0", - "python/frameworks/openai": "0.1.14", - "python/frameworks/openai-agents": "0.1.7", + "python/frameworks/openai": "0.1.15", + "python/frameworks/openai-agents": "0.1.8", "python/frameworks/openai-realtime": "0.1.0", "python/frameworks/parallel": "0.1.1", "python/frameworks/pgvector": "0.1.1", "python/frameworks/pinecone": "0.1.1", - "python/frameworks/pipecat": "0.1.2", - "python/frameworks/portkey": "0.1.3", - "python/frameworks/pydantic-ai": "0.1.0", + "python/frameworks/pipecat": "0.1.3", + "python/frameworks/portkey": "0.1.4", + "python/frameworks/pydantic-ai": "0.2.0", "python/frameworks/qdrant": "0.1.1", "python/frameworks/redis-vector": "0.1.1", "python/frameworks/replicate": "0.1.0", "python/frameworks/semantic-kernel": "0.1.2", - "python/frameworks/smolagents": "0.1.6", + "python/frameworks/smolagents": "0.1.7", "python/frameworks/strands": "0.1.2", "python/frameworks/tavily": "0.1.1", "python/frameworks/temporal": "0.1.0", - "python/frameworks/together": "0.1.1", + "python/frameworks/together": "0.1.2", "python/frameworks/typesafe": "0.1.0", - "python/frameworks/vertexai": "0.1.11", + "python/frameworks/vertexai": "0.1.12", "python/frameworks/vllm": "0.1.1", "python/frameworks/voyage": "0.1.2", "python/frameworks/weaviate": "0.1.2", - "python/frameworks/xai": "0.1.1", + "python/frameworks/xai": "0.1.2", "python/traceai": "0.1.0", - "typescript/packages/fi-core": "1.2.3", - "typescript/packages/fi-semantic-conventions": "1.2.0", - "typescript/packages/traceai_anthropic": "0.1.5", - "typescript/packages/traceai_auto": "0.1.3", - "typescript/packages/traceai_bedrock": "0.1.3", + "typescript/packages/fi-core": "1.2.4", + "typescript/packages/fi-semantic-conventions": "1.3.0", + "typescript/packages/traceai_anthropic": "0.1.6", + "typescript/packages/traceai_auto": "0.1.4", + "typescript/packages/traceai_bedrock": "0.1.4", "typescript/packages/traceai_beeai": "0.1.2", "typescript/packages/traceai_cerebras": "0.1.2", "typescript/packages/traceai_chromadb": "0.1.2", "typescript/packages/traceai_claude_agent_sdk": "0.1.0", "typescript/packages/traceai_codex_sdk": "0.1.1", - "typescript/packages/traceai_cohere": "0.1.1", + "typescript/packages/traceai_cohere": "0.1.2", "typescript/packages/traceai_cursor_sdk": "0.1.1", "typescript/packages/traceai_deepseek": "0.1.2", "typescript/packages/traceai_fireworks": "0.1.2", @@ -93,7 +93,7 @@ "typescript/packages/traceai_huggingface": "0.1.1", "typescript/packages/traceai_instructor": "0.1.2", "typescript/packages/traceai_lancedb": "0.1.2", - "typescript/packages/traceai_langchain": "0.1.7", + "typescript/packages/traceai_langchain": "0.1.8", "typescript/packages/traceai_livekit": "0.1.1", "typescript/packages/traceai_llamaindex": "0.1.4", "typescript/packages/traceai_mastra": "0.2.2", @@ -102,7 +102,7 @@ "typescript/packages/traceai_mistral": "0.1.1", "typescript/packages/traceai_mongodb": "0.1.2", "typescript/packages/traceai_ollama": "0.1.2", - "typescript/packages/traceai_openai": "0.1.17", + "typescript/packages/traceai_openai": "0.1.18", "typescript/packages/traceai_openai_agents": "0.2.1", "typescript/packages/traceai_pgvector": "0.1.1", "typescript/packages/traceai_pi_coding_agent": "0.2.0", @@ -113,7 +113,7 @@ "typescript/packages/traceai_redis": "0.1.3", "typescript/packages/traceai_strands": "0.1.3", "typescript/packages/traceai_together": "0.1.1", - "typescript/packages/traceai_vercel": "0.1.4", + "typescript/packages/traceai_vercel": "0.1.5", "typescript/packages/traceai_vertexai": "0.1.1", "typescript/packages/traceai_vllm": "0.1.3", "typescript/packages/traceai_voltagent": "0.1.0", @@ -127,7 +127,7 @@ "go/lambda": "0.1.1", "go/openai": "0.1.0", "csharp/src/FIInstrumentation": "1.0.1", - "python": "1.3.1", + "python": "1.3.2", "typescript/packages/traceai_elevenlabs": "0.1.1", "go/bedrock": "0.1.1", "typescript/packages/traceai_huggingface_transformers": "0.1.0", @@ -153,10 +153,10 @@ "typescript/packages/traceai_opencode_plugin": "0.1.0", "typescript/packages/traceai_pi_plugin": "0.1.0", "typescript/packages/traceai_vscode_extension": "0.1.0", - "typescript/packages/traceai_openrouter": "0.1.0", - "typescript/packages/traceai_vitest_evals": "0.1.0", + "typescript/packages/traceai_openrouter": "0.1.1", + "typescript/packages/traceai_vitest_evals": "0.1.1", "python/frameworks/openrouter": "0.1.0", "go/langchaingo": "0.1.0", "ruby/roast": "0.1.0", - "typescript/packages/traceai_n8n": "0.1.0" + "typescript/packages/traceai_n8n": "0.1.1" } diff --git a/go/langchaingo/README.md b/go/langchaingo/README.md index e79df121f..106e0d2c9 100644 --- a/go/langchaingo/README.md +++ b/go/langchaingo/README.md @@ -12,7 +12,7 @@ Until the first `go/langchaingo/v0.1.0` tag is released, use `@dev` instead of ` ## Configure tracing -Set `FI_API_KEY`, `FI_SECRET_KEY`, `FI_PROJECT_NAME` and the credential for your chosen model provider. The [example](example/main.go) configures an OpenTelemetry OTLP/HTTP exporter with the full endpoint `https://api.futureagi.com/tracer/v1/traces`, headers `X-Api-Key` and `X-Secret-Key`, no compression, and the `project_name` and `project_type=observe` resource attributes. +Set `FI_API_KEY`, `FI_SECRET_KEY`, `FI_PROJECT_NAME` and the credential for your chosen model provider. The [example](example/main.go) configures an OpenTelemetry OTLP/HTTP exporter that sends to `{FI_BASE_URL}/tracer/v1/traces` (`FI_BASE_URL` is optional, defaults to `https://api.futureagi.com`, and may end with a slash), with headers `X-Api-Key` and `X-Secret-Key`, no compression, and the `project_name` and `project_type=observe` resource attributes. Set `FI_BASE_URL` to send to a self-hosted or local Future AGI instead. ```go handler := traceailangchaingo.NewHandler( diff --git a/go/langchaingo/example/main.go b/go/langchaingo/example/main.go index 76527376d..343358cf1 100644 --- a/go/langchaingo/example/main.go +++ b/go/langchaingo/example/main.go @@ -1,10 +1,14 @@ // Run with FI_API_KEY, FI_SECRET_KEY, FI_PROJECT_NAME and OPENAI_API_KEY set. +// Spans go to {FI_BASE_URL}/tracer/v1/traces; FI_BASE_URL is optional and +// defaults to https://api.futureagi.com. package main import ( + "cmp" "context" "fmt" "os" + "strings" traceailangchaingo "github.com/future-agi/traceAI/go/langchaingo" "github.com/tmc/langchaingo/llms" @@ -15,8 +19,16 @@ import ( sdktrace "go.opentelemetry.io/otel/sdk/trace" ) +const defaultBaseURL = "https://api.futureagi.com" + +// tracesEndpoint returns the OTLP/HTTP traces URL under FI_BASE_URL, ignoring +// trailing slashes, or under the hosted Future AGI API when FI_BASE_URL is unset. +func tracesEndpoint() string { + return cmp.Or(strings.TrimRight(os.Getenv("FI_BASE_URL"), "/"), defaultBaseURL) + "/tracer/v1/traces" +} + func run(ctx context.Context) error { - exp, err := otlptracehttp.New(ctx, otlptracehttp.WithEndpointURL("https://api.futureagi.com/tracer/v1/traces"), otlptracehttp.WithHeaders(map[string]string{"X-Api-Key": os.Getenv("FI_API_KEY"), "X-Secret-Key": os.Getenv("FI_SECRET_KEY")}), otlptracehttp.WithCompression(otlptracehttp.NoCompression)) + exp, err := otlptracehttp.New(ctx, otlptracehttp.WithEndpointURL(tracesEndpoint()), otlptracehttp.WithHeaders(map[string]string{"X-Api-Key": os.Getenv("FI_API_KEY"), "X-Secret-Key": os.Getenv("FI_SECRET_KEY")}), otlptracehttp.WithCompression(otlptracehttp.NoCompression)) if err != nil { return err } diff --git a/go/langchaingo/example/main_test.go b/go/langchaingo/example/main_test.go new file mode 100644 index 000000000..1208d4afd --- /dev/null +++ b/go/langchaingo/example/main_test.go @@ -0,0 +1,32 @@ +package main + +import ( + "os" + "testing" +) + +func TestTracesEndpointDefaultsToHostedAPI(t *testing.T) { + t.Setenv("FI_BASE_URL", "") + if got, want := tracesEndpoint(), "https://api.futureagi.com/tracer/v1/traces"; got != want { + t.Errorf("FI_BASE_URL empty: tracesEndpoint() = %q, want %q", got, want) + } + os.Unsetenv("FI_BASE_URL") + if got, want := tracesEndpoint(), "https://api.futureagi.com/tracer/v1/traces"; got != want { + t.Errorf("FI_BASE_URL unset: tracesEndpoint() = %q, want %q", got, want) + } +} + +func TestTracesEndpointUsesBaseURL(t *testing.T) { + for _, tc := range []struct{ base, want string }{ + {"http://127.0.0.1:8000", "http://127.0.0.1:8000/tracer/v1/traces"}, + {"http://127.0.0.1:8000/", "http://127.0.0.1:8000/tracer/v1/traces"}, + {"http://127.0.0.1:8000//", "http://127.0.0.1:8000/tracer/v1/traces"}, + {"https://observe.example.com/futureagi/", "https://observe.example.com/futureagi/tracer/v1/traces"}, + {"/", "https://api.futureagi.com/tracer/v1/traces"}, + } { + t.Setenv("FI_BASE_URL", tc.base) + if got := tracesEndpoint(); got != tc.want { + t.Errorf("FI_BASE_URL=%q: tracesEndpoint() = %q, want %q", tc.base, got, tc.want) + } + } +} diff --git a/python/CHANGELOG.md b/python/CHANGELOG.md index 2e1aa00f1..55fc2137e 100644 --- a/python/CHANGELOG.md +++ b/python/CHANGELOG.md @@ -1,3 +1,11 @@ +## [1.3.2] - pending release + +### Fixed +- using_session, using_user and the other context helpers keep the session and user on spans inside async functions and async generators when used as decorators. +- Generator functions traced with tracer.chain, tracer.agent or tracer.tool, sync or async, keep their span open until the generator finishes or is closed, so spans created inside them nest under it and the span's output defaults to the last yielded value. +- Eval tags for custom evaluations with no model set no longer make register fail. +- The get_session_attributes and get_context_attributes helpers set session.id alongside gen_ai.conversation.id. + ## [1.3.1] - 2026-11-10 ### Fixed - PII redaction (`pii_redaction` / `FI_PII_REDACTION`) no longer corrupts correlation identifiers (#195): diff --git a/python/fi_instrumentation/fi_types.py b/python/fi_instrumentation/fi_types.py index 6974f4c6e..05f3a565d 100644 --- a/python/fi_instrumentation/fi_types.py +++ b/python/fi_instrumentation/fi_types.py @@ -1794,12 +1794,13 @@ def validate_fagi_system_eval_mapping( def to_dict(self) -> Dict[str, Any]: """Convert EvalTag to dictionary format for API responses""" - if isinstance(self.model, str): + # Custom evaluations may use their template's model without an override. + if self.model is None or isinstance(self.model, str): model_name = self.model elif isinstance(self.model, ModelChoices): model_name = self.model.value else: - raise ValueError(f"Model must be a string or ModelChoices, got {type(self.model)}") + raise ValueError(f"Model must be None, a string or ModelChoices, got {type(self.model)}") return { "type": self.type.value, diff --git a/python/fi_instrumentation/instrumentation/_attributes.py b/python/fi_instrumentation/instrumentation/_attributes.py index 708f46427..18443032c 100644 --- a/python/fi_instrumentation/instrumentation/_attributes.py +++ b/python/fi_instrumentation/instrumentation/_attributes.py @@ -172,7 +172,10 @@ def get_context_attributes( def get_session_attributes(*, session_id: str) -> Dict[str, AttributeValue]: - return {GEN_AI_CONVERSATION_ID: session_id} + return { + SESSION_ID: session_id, + GEN_AI_CONVERSATION_ID: session_id, + } def get_tag_attributes(*, tags: List[str]) -> Dict[str, AttributeValue]: @@ -577,6 +580,7 @@ def get_llm_tool_attributes( OUTPUT_MIME_TYPE = SpanAttributes.OUTPUT_MIME_TYPE OUTPUT_VALUE = SpanAttributes.OUTPUT_VALUE RETRIEVAL_DOCUMENTS = SpanAttributes.RETRIEVAL_DOCUMENTS +SESSION_ID = SpanAttributes.SESSION_ID GEN_AI_CONVERSATION_ID = SpanAttributes.GEN_AI_CONVERSATION_ID TAG_TAGS = SpanAttributes.TAG_TAGS GEN_AI_TOOL_DESCRIPTION = SpanAttributes.GEN_AI_TOOL_DESCRIPTION diff --git a/python/fi_instrumentation/instrumentation/_tracers.py b/python/fi_instrumentation/instrumentation/_tracers.py index 779854676..46449edfc 100644 --- a/python/fi_instrumentation/instrumentation/_tracers.py +++ b/python/fi_instrumentation/instrumentation/_tracers.py @@ -1,18 +1,23 @@ import asyncio import collections import inspect +import sys from collections.abc import Sequence from contextlib import contextmanager from contextvars import ContextVar from datetime import datetime +from functools import partial from secrets import randbits from types import ModuleType from typing import ( TYPE_CHECKING, Any, # type: ignore[attr-defined] + AsyncGenerator, Awaitable, Callable, + ContextManager, Dict, + Generator, Iterator, Literal, Optional, @@ -27,7 +32,14 @@ import wrapt # type: ignore[import-untyped] from fi_instrumentation.fi_types import FiSpanKindValues, SpanAttributes from opentelemetry.attributes import BoundedAttributes -from opentelemetry.context import _SUPPRESS_INSTRUMENTATION_KEY, Context, get_value +from opentelemetry.context import ( + _SUPPRESS_INSTRUMENTATION_KEY, + Context, + attach, + detach, + get_current, + get_value, +) from opentelemetry.sdk.trace.id_generator import IdGenerator, RandomIdGenerator from opentelemetry.sdk.trace.sampling import Sampler, SamplingResult from opentelemetry.trace import ( @@ -40,6 +52,7 @@ Status, StatusCode, Tracer, + set_span_in_context, use_span, ) from opentelemetry.util.types import Attributes @@ -380,16 +393,27 @@ def sync_wrapper( args: Tuple[Any, ...], kwargs: Dict[str, Any], ) -> ReturnType: - tracer = self - with _chain_context( - tracer=tracer, + context = partial( + _chain_context, + tracer=self, name=name, kind=kind, wrapped=wrapped, instance=instance, args=args, kwargs=kwargs, - ) as chain_context: + ) + if inspect.isgeneratorfunction(wrapped): + return cast( + ReturnType, + _trace_generator(wrapped(*args, **kwargs), context(streaming=True)), + ) + if inspect.isasyncgenfunction(wrapped): + return cast( + ReturnType, + _trace_async_generator(wrapped(*args, **kwargs), context(streaming=True)), + ) + with context() as chain_context: output = wrapped(*args, **kwargs) return chain_context.process_output(output) @@ -466,9 +490,9 @@ def sync_wrapper( args: Tuple[Any, ...], kwargs: Dict[str, Any], ) -> ReturnType: - tracer = self - with _tool_context( - tracer=tracer, + context = partial( + _tool_context, + tracer=self, name=name, description=description, parameters=parameters, @@ -476,7 +500,18 @@ def sync_wrapper( instance=instance, args=args, kwargs=kwargs, - ) as tool_context: + ) + if inspect.isgeneratorfunction(wrapped): + return cast( + ReturnType, + _trace_generator(wrapped(*args, **kwargs), context(streaming=True)), + ) + if inspect.isasyncgenfunction(wrapped): + return cast( + ReturnType, + _trace_async_generator(wrapped(*args, **kwargs), context(streaming=True)), + ) + with context() as tool_context: output = wrapped(*args, **kwargs) return tool_context.process_output(output) @@ -532,6 +567,7 @@ def _chain_context( instance: Any, args: Tuple[Any, ...], kwargs: Dict[str, Any], + streaming: bool = False, ) -> Iterator[_ChainContext]: span_name = name or _infer_span_name(instance=instance, callable=wrapped) bound_args = inspect.signature(wrapped).bind(*args, **kwargs) @@ -544,7 +580,8 @@ def _chain_context( else: input_attributes = get_input_attributes(arguments) - with tracer.start_as_current_span( + start_span = partial(_stream_span, tracer) if streaming else tracer.start_as_current_span + with start_span( span_name, fi_span_kind=kind, attributes=input_attributes, @@ -576,6 +613,7 @@ def _tool_context( instance: Any, args: Tuple[Any, ...], kwargs: Dict[str, Any], + streaming: bool = False, ) -> Iterator[_ToolContext]: tool_name = name or _infer_span_name(instance=instance, callable=wrapped) bound_args = inspect.signature(wrapped).bind(*args, **kwargs) @@ -593,7 +631,8 @@ def _tool_context( description=tool_description, parameters=tool_parameters, ) - with tracer.start_as_current_span( + start_span = partial(_stream_span, tracer) if streaming else tracer.start_as_current_span + with start_span( tool_name, fi_span_kind=FiSpanKindValues.TOOL, attributes={ @@ -605,6 +644,114 @@ def _tool_context( yield context +@contextmanager +def _stream_span(tracer: FITracer, name: str, **kwargs: Any) -> Iterator[FiSpan]: + """Own the lifetime without leaving a current-span token across a yield.""" + span = tracer.start_span(name, **kwargs) + try: + yield span + except GeneratorExit: + # Explicit close/aclose is normal termination, not an application error. + raise + except BaseException as exception: + if span.is_recording(): + span.record_exception(exception, escaped=True) + span.set_status( + Status(StatusCode.ERROR, f"{type(exception).__name__}: {exception}") + ) + raise + finally: + span.end() + + +class _Stream: + """Resume only the producer under its saved OTel context. + + A child span may itself cross a yield. Save that context as well, but restore + the consumer's context after every operation. Do not copy the Python context: + application ContextVar writes must still behave like an undecorated generator. + Only the last yielded value is retained, so infinite streams are not buffered. + """ + + def __init__(self, generator: Any, span: FiSpan) -> None: + self.generator = generator + self.context = set_span_in_context(cast(Span, span)) + self.output: Any = None + + @contextmanager + def activate(self) -> Iterator[None]: + token = attach(self.context) + try: + yield + finally: + self.context = get_current() + detach(token) + + def __iter__(self) -> "_Stream": + return self + + def __next__(self) -> Any: + return self.send(None) + + def send(self, value: Any) -> Any: + with self.activate(): + self.output = self.generator.send(value) + return self.output + + def throw(self, *args: Any) -> Any: + with self.activate(): + self.output = self.generator.throw(*args) + return self.output + + def close(self) -> Any: + with self.activate(): + return self.generator.close() + + async def step(self, method: Callable[..., Awaitable[Any]], *args: Any) -> Any: + with self.activate(): + self.output = await method(*args) + return self.output + + +def _trace_generator( + generator: Any, + context: ContextManager[Union[_ChainContext, _ToolContext]], +) -> Generator[Any, Any, Any]: + with context as span_context: + stream = _Stream(generator, span_context._span) + try: + # Native delegation preserves send/throw/close and StopIteration.value. + return (yield from stream) + finally: + span_context.process_output(stream.output) + + +async def _trace_async_generator( + generator: Any, + context: ContextManager[Union[_ChainContext, _ToolContext]], +) -> AsyncGenerator[Any, Any]: + with context as span_context: + stream = _Stream(generator, span_context._span) + try: + try: + output = await stream.step(generator.__anext__) + while True: + try: + value = yield output + except GeneratorExit: + with stream.activate(): + await generator.aclose() + raise + except BaseException: + output = await stream.step(generator.athrow, *sys.exc_info()) + else: + output = await stream.step(generator.asend, value) + except StopAsyncIteration: + return + finally: + span_context.process_output(stream.output) + + def _infer_span_name(*, instance: Any, callable: Callable[..., Any]) -> str: """ Makes a best-effort attempt to infer a span name from the bound instance diff --git a/python/fi_instrumentation/instrumentation/context_attributes.py b/python/fi_instrumentation/instrumentation/context_attributes.py index a4e25353d..bbd221199 100644 --- a/python/fi_instrumentation/instrumentation/context_attributes.py +++ b/python/fi_instrumentation/instrumentation/context_attributes.py @@ -1,9 +1,12 @@ from contextlib import ContextDecorator -from typing import Any, Dict, Iterator, List, Optional, Tuple +from contextvars import Token +from functools import wraps +from inspect import isasyncgenfunction, iscoroutinefunction +from typing import Any, Callable, Dict, Iterator, List, Optional, Tuple from fi_instrumentation.fi_types import SimulatorAttributes, SpanAttributes from fi_instrumentation.instrumentation.helpers import safe_json_dumps -from opentelemetry.context import attach, detach, get_current, get_value, set_value +from opentelemetry.context import Context, attach, detach, get_current, get_value, set_value from opentelemetry.util.types import AttributeValue from typing_extensions import Self @@ -25,6 +28,8 @@ class _UsingAttributesContextManager(ContextDecorator): + """Use a fresh instance per with/async with block; helpers also decorate sync and async functions.""" + def __init__( self, *, @@ -48,7 +53,72 @@ def __init__( self._prompt_template_variables = prompt_template_variables self._simulator_attributes = simulator_attributes + def __call__(self, func: Callable[..., Any]) -> Callable[..., Any]: + if iscoroutinefunction(func): + @wraps(func) + async def async_wrapper(*args: Any, **kwargs: Any) -> Any: + token = self._attach_context() + try: + return await func(*args, **kwargs) + finally: + detach(token) + + return async_wrapper + + if isasyncgenfunction(func): + @wraps(func) + async def async_generator_wrapper(*args: Any, **kwargs: Any): + token = self._attach_context() + try: + generator_context = get_current() + finally: + detach(token) + generator = func(*args, **kwargs) + operation = generator.__anext__() + try: + while True: + token = attach(generator_context) + try: + try: + value = await operation + except StopAsyncIteration: + return + finally: + # Keep context changes made inside the body, but never + # leave its context attached while the consumer runs. + generator_context = get_current() + detach(token) + try: + sent = yield value + except GeneratorExit: + raise + except BaseException as exc: + operation = generator.athrow(exc) + else: + operation = generator.asend(sent) + finally: + token = attach(generator_context) + try: + await generator.aclose() + finally: + detach(token) + + return async_generator_wrapper + + @wraps(func) + def wrapper(*args: Any, **kwargs: Any) -> Any: + token = self._attach_context() + try: + return func(*args, **kwargs) + finally: + detach(token) + + return wrapper + def attach_context(self) -> None: + self._token = self._attach_context() + + def _attach_context(self) -> Token[Context]: ctx = get_current() if self._session_id: ctx = set_value(SpanAttributes.SESSION_ID, self._session_id, ctx) @@ -95,7 +165,7 @@ def attach_context(self) -> None: # Explicitly check for None, not truthiness, since False is a valid boolean value if is_simulator_trace is not None: ctx = set_value(SimulatorAttributes.IS_SIMULATOR_TRACE, is_simulator_trace, ctx) - self._token = attach(ctx) + return attach(ctx) def __enter__(self) -> Self: self.attach_context() diff --git a/python/fi_instrumentation/instrumentation/version.py b/python/fi_instrumentation/instrumentation/version.py index 17bd5af46..67e7c89a9 100644 --- a/python/fi_instrumentation/instrumentation/version.py +++ b/python/fi_instrumentation/instrumentation/version.py @@ -1 +1 @@ -__version__ = "1.3.1" # x-release-please-version +__version__ = "1.3.2" # x-release-please-version diff --git a/python/frameworks/anthropic/CHANGELOG.md b/python/frameworks/anthropic/CHANGELOG.md index a67f6224d..bb214498b 100644 --- a/python/frameworks/anthropic/CHANGELOG.md +++ b/python/frameworks/anthropic/CHANGELOG.md @@ -1,3 +1,8 @@ +## [0.1.14] - pending release + +### Fixed +- Traced messages.create and messages.count_tokens calls whose messages are a generator or other single-use iterable send those messages to Anthropic instead of an emptied iterable, and the span still records them. Spans also record tool call IDs from the model output, for streamed and non-streamed responses. + ## [0.1.12](https://github.com/future-agi/traceAI/compare/traceai-anthropic-v0.1.11...traceai-anthropic-v0.1.12) (2026-10-31) diff --git a/python/frameworks/anthropic/pyproject.toml b/python/frameworks/anthropic/pyproject.toml index b226cdff6..b4c02aa18 100644 --- a/python/frameworks/anthropic/pyproject.toml +++ b/python/frameworks/anthropic/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "traceAI-anthropic" -version = "0.1.13" +version = "0.1.14" description = "OpenTelemetry instrumentation for Anthropic" authors = ["Future AGI "] readme = "README.md" diff --git a/python/frameworks/anthropic/tests/test_message_iterables.py b/python/frameworks/anthropic/tests/test_message_iterables.py new file mode 100644 index 000000000..0e6ce7e30 --- /dev/null +++ b/python/frameworks/anthropic/tests/test_message_iterables.py @@ -0,0 +1,133 @@ +"""T2-ANTHROPIC-001: tracing must not exhaust messages before the SDK call.""" + +import asyncio + +import opentelemetry.context as context_api +import pytest +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter + +from traceai_anthropic import _wrappers + + +class _OneShotMessages: + """An iterable sharing one iterator, without itself implementing __next__.""" + + def __init__(self, messages): + self._iterator = iter(messages) + self.iterations = 0 + + def __iter__(self): + self.iterations += 1 + return self._iterator + + +class _FakeResponse: + model = "local-test" + role = "assistant" + content = () + usage = None + input_tokens = 3 + + def model_dump_json(self): + return "{}" + + def to_dict(self): + return {"input_tokens": self.input_tokens} + + +@pytest.fixture +def tracing(): + exporter = InMemorySpanExporter() + provider = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + try: + yield provider.get_tracer(__name__), exporter + finally: + provider.shutdown() + + +@pytest.mark.parametrize("operation", ["sync-create", "async-create", "count-tokens"]) +@pytest.mark.parametrize( + "message_factory", + [ + pytest.param(lambda messages: (message for message in messages), id="generator"), + pytest.param(iter, id="iterator"), + pytest.param(_OneShotMessages, id="one-shot-iterable"), + pytest.param(list, id="list"), + pytest.param(tuple, id="tuple"), + ], +) +@pytest.mark.parametrize("suppressed", [False, True], ids=["instrumented", "suppressed"]) +def test_messages_iterables_preserve_request_and_telemetry( + tracing, operation, message_factory, suppressed +): + tracer, exporter = tracing + expected = [ + {"role": "user", "content": "hello"}, + {"role": "assistant", "content": "hi"}, + {"role": "user", "content": [{"type": "text", "text": "follow-up"}]}, + ] + messages = message_factory(expected) + kwargs = {"model": "local-test", "messages": messages} + if operation != "count-tokens": + kwargs["max_tokens"] = 8 + response = _FakeResponse() + received = [] + + def original(**request): + # This is the SDK boundary: consume messages only after tracing has run. + received.append((request, list(request["messages"]))) + return response + + async def async_original(**request): + return original(**request) + + wrapper_type = { + "sync-create": _wrappers._MessagesWrapper, + "async-create": _wrappers._AsyncMessagesWrapper, + "count-tokens": _wrappers._MessagesCountTokensWrapper, + }[operation] + wrapper = wrapper_type(tracer=tracer) + token = context_api.attach( + context_api.set_value(context_api._SUPPRESS_INSTRUMENTATION_KEY, suppressed) + ) + try: + if operation == "async-create": + result = asyncio.run(wrapper(async_original, None, (), kwargs)) + else: + result = wrapper(original, None, (), kwargs) + finally: + context_api.detach(token) + + assert result is response + assert len(received) == 1 + request, sent_messages = received[0] + assert sent_messages == expected + assert all(sent is source for sent, source in zip(sent_messages, expected)) + assert kwargs["messages"] is messages # Do not mutate the caller's mapping. + assert {key: value for key, value in request.items() if key != "messages"} == { + key: value for key, value in kwargs.items() if key != "messages" + } + if isinstance(messages, _OneShotMessages): + assert messages.iterations == 1 + if suppressed or isinstance(messages, (list, tuple)): + assert request["messages"] is messages + + spans = exporter.get_finished_spans() + if suppressed: + assert not spans + else: + assert len(spans) == 1 + attributes = spans[0].attributes + for index, message in enumerate(expected): + prefix = f"{_wrappers.GEN_AI_INPUT_MESSAGES}.{index}" + assert attributes[f"{prefix}.{_wrappers.MESSAGE_ROLE}"] == message["role"] + assert attributes[ + f"{_wrappers.GEN_AI_INPUT_MESSAGES}.0.{_wrappers.MESSAGE_CONTENT}" + ] == "hello" + assert attributes[ + f"{_wrappers.GEN_AI_INPUT_MESSAGES}.2.{_wrappers.MESSAGE_CONTENT}.0." + f"{_wrappers.MESSAGE_CONTENT_TEXT}" + ] == "follow-up" diff --git a/python/frameworks/anthropic/tests/test_output_tool_call_ids.py b/python/frameworks/anthropic/tests/test_output_tool_call_ids.py new file mode 100644 index 000000000..69c8c80c6 --- /dev/null +++ b/python/frameworks/anthropic/tests/test_output_tool_call_ids.py @@ -0,0 +1,162 @@ +"""T2-ANTHROPIC-002: retain provider IDs for parallel calls to one tool.""" +import asyncio +import json + +import anthropic +import pytest +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter + +from traceai_anthropic import AnthropicInstrumentor + +try: + import httpx2 as httpx +except ImportError: + import httpx + + +TOOL_CALLS = [ + {"type": "tool_use", "id": "call_fake_first", "name": "lookup", "input": {"q": "hello"}}, + {"type": "tool_use", "id": "call_fake_second", "name": "lookup", "input": {"q": "goodbye"}}, +] +MESSAGE = { + "id": "msg_fake", "type": "message", "role": "assistant", "model": "local-test", + "content": [{"type": "text", "text": "Looking up both."}, *TOOL_CALLS], + "stop_reason": "tool_use", "stop_sequence": None, + "usage": {"input_tokens": 4, "output_tokens": 8}, +} +PARAMS = { + "model": "local-test", "max_tokens": 32, + "messages": [{"role": "user", "content": "Look up hello and goodbye."}], + "tools": [{ + "name": "lookup", "description": "Look up a query.", + "input_schema": {"type": "object", "properties": {"q": {"type": "string"}}}, + }], +} + + +def stream_events(): + yield { + "type": "message_start", + "message": {**MESSAGE, "content": [], "stop_reason": None, + "usage": {"input_tokens": 4, "output_tokens": 0}}, + } + yield { + "type": "content_block_start", "index": 0, + "content_block": {"type": "text", "text": ""}, + } + yield { + "type": "content_block_delta", "index": 0, + "delta": {"type": "text_delta", "text": "Looking up both."}, + } + yield {"type": "content_block_stop", "index": 0} + for index, call in enumerate(TOOL_CALLS, start=1): + yield { + "type": "content_block_start", "index": index, + "content_block": {**call, "input": {}}, + } + arguments = json.dumps(call["input"]) + # Two fragments ensure an ID is retained, not concatenated per delta. + for fragment in (arguments[:6], arguments[6:]): + yield { + "type": "content_block_delta", "index": index, + "delta": {"type": "input_json_delta", "partial_json": fragment}, + } + yield {"type": "content_block_stop", "index": index} + yield { + "type": "message_delta", + "delta": {"stop_reason": "tool_use", "stop_sequence": None}, + "usage": {"output_tokens": 8}, + } + yield {"type": "message_stop"} + + +def transport_handler(request): + if json.loads(request.content).get("stream"): + payload = "".join( + f"event: {event['type']}\ndata: {json.dumps(event)}\n\n" + for event in stream_events() + ) + return httpx.Response( + 200, headers={"content-type": "text/event-stream"}, content=payload.encode() + ) + return httpx.Response(200, json=MESSAGE) + + +def consume_sync(client, streaming): + response = client.messages.create(**PARAMS, stream=streaming) + if not streaming: + return response.model_dump() + try: + return [event.model_dump() for event in response] + finally: + response.close() + + +async def consume_async(client, streaming): + response = await client.messages.create(**PARAMS, stream=streaming) + if not streaming: + return response.model_dump() + try: + return [event.model_dump() async for event in response] + finally: + await response.close() + + +@pytest.mark.parametrize( + "async_, streaming", + [(False, False), (True, False), (False, True), (True, True)], + ids=["sync-message", "async-message", "sync-stream", "async-stream"], +) +def test_output_tool_call_ids(async_, streaming): + exporter = InMemorySpanExporter() + provider = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + instrumentor = AnthropicInstrumentor() + + async def run_async(): + async with anthropic.AsyncAnthropic( + api_key="fake-local-key", max_retries=0, + http_client=httpx.AsyncClient(transport=httpx.MockTransport(transport_handler)), + ) as client: + expected = await consume_async(client, streaming) + instrumentor.instrument(tracer_provider=provider) + return expected, await consume_async(client, streaming) + + try: + if async_: + expected, actual = asyncio.run(run_async()) + else: + with anthropic.Anthropic( + api_key="fake-local-key", max_retries=0, + http_client=httpx.Client(transport=httpx.MockTransport(transport_handler)), + ) as client: + expected = consume_sync(client, streaming) + instrumentor.instrument(tracer_provider=provider) + actual = consume_sync(client, streaming) + + # Instrumentation must leave every SDK response/event unchanged. + assert actual == expected + if streaming: + returned_ids = [ + event["content_block"]["id"] for event in actual + if event["type"] == "content_block_start" + and event["content_block"]["type"] == "tool_use" + ] + else: + returned_ids = [block["id"] for block in actual["content"] if block["type"] == "tool_use"] + assert returned_ids == [call["id"] for call in TOOL_CALLS] + + spans = exporter.get_finished_spans() + assert len(spans) == 1 + attributes = spans[0].attributes + assert attributes["gen_ai.output.messages.0.message.content"] == "Looking up both." + for index, call in enumerate(TOOL_CALLS): + prefix = f"gen_ai.output.messages.0.message.tool_calls.{index}.tool_call" + assert attributes[f"{prefix}.function.name"] == call["name"] + assert json.loads(attributes[f"{prefix}.function.arguments"]) == call["input"] + assert attributes[f"{prefix}.id"] == call["id"] + finally: + instrumentor.uninstrument() + provider.shutdown() diff --git a/python/frameworks/anthropic/traceai_anthropic/_stream.py b/python/frameworks/anthropic/traceai_anthropic/_stream.py index 159963d1a..85b57b096 100644 --- a/python/frameworks/anthropic/traceai_anthropic/_stream.py +++ b/python/frameworks/anthropic/traceai_anthropic/_stream.py @@ -533,11 +533,11 @@ class _MessageResponseAccumulator: def __init__(self) -> None: self._is_null = True self._current_message_idx = -1 - # (kind, type, tool name) of the open content block; kind is "text", - # "tool_use" or None for blocks whose deltas are not recorded. - self._current_block: Tuple[Optional[str], Optional[str], Optional[str]] = ( - None, None, None - ) + # (kind, type, tool name, tool ID) of the open content block; kind is + # "text", "tool_use" or None for blocks whose deltas are not recorded. + self._current_block: Tuple[ + Optional[str], Optional[str], Optional[str], Optional[str] + ] = (None, None, None, None) self._values = _ValuesAccumulator( messages=_IndexedAccumulator( lambda: _ValuesAccumulator( @@ -547,6 +547,7 @@ def __init__(self) -> None: type=_SimpleStringReplace(), text=_StringAccumulator(), tool_name=_SimpleStringReplace(), + tool_id=_SimpleStringReplace(), tool_input=_StringAccumulator(), ), ), @@ -580,10 +581,10 @@ def extract(self, chunk: "RawMessageStreamEvent") -> Tuple[Any, ...]: if isinstance(chunk, RawContentBlockStartEvent): block = chunk.content_block if isinstance(block, TextBlock): - return ("block_start", "text", block.type, None) + return ("block_start", "text", block.type, None, None) if isinstance(block, ToolUseBlock): - return ("block_start", "tool_use", block.type, block.name) - return ("block_start", None, None, None) + return ("block_start", "tool_use", block.type, block.name, block.id) + return ("block_start", None, None, None, None) if isinstance(chunk, RawContentBlockDeltaEvent): delta_type = getattr(chunk.delta, "type", None) if delta_type == "text_delta": @@ -614,7 +615,7 @@ def merge(self, event: Tuple[Any, ...]) -> None: self._current_block = event[1:] elif kind == "block_delta": _, index, delta_type, fragment = event - block_kind, block_type, tool_name = self._current_block + block_kind, block_type, tool_name, tool_id = self._current_block if block_kind == "text" and delta_type == "text_delta": content = {"index": index, "type": block_type, "text": fragment} elif block_kind == "tool_use" and delta_type == "input_json_delta": @@ -622,6 +623,7 @@ def merge(self, event: Tuple[Any, ...]) -> None: "index": index, "type": block_type, "tool_name": tool_name, + "tool_id": tool_id, "tool_input": fragment, } else: @@ -722,6 +724,11 @@ def get_extra_attributes(self) -> Iterator[Tuple[str, AttributeValue]]: content.get("text", ""), ) elif content_type == "tool_use": + if tool_id := content.get("tool_id"): + yield ( + f"{SpanAttributes.GEN_AI_OUTPUT_MESSAGES}.{idx}.{MessageAttributes.MESSAGE_TOOL_CALLS}.{tool_idx}.{ToolCallAttributes.TOOL_CALL_ID}", + tool_id, + ) yield ( f"{SpanAttributes.GEN_AI_OUTPUT_MESSAGES}.{idx}.{MessageAttributes.MESSAGE_TOOL_CALLS}.{tool_idx}.{ToolCallAttributes.TOOL_CALL_FUNCTION_NAME}", content.get("tool_name", ""), diff --git a/python/frameworks/anthropic/traceai_anthropic/_wrappers.py b/python/frameworks/anthropic/traceai_anthropic/_wrappers.py index 9815f9042..84fc5b84f 100644 --- a/python/frameworks/anthropic/traceai_anthropic/_wrappers.py +++ b/python/frameworks/anthropic/traceai_anthropic/_wrappers.py @@ -1,5 +1,6 @@ import logging from abc import ABC +from collections.abc import Iterable from contextlib import contextmanager from itertools import chain from typing import ( @@ -262,6 +263,16 @@ async def __call__( return response +def _snapshot_messages(kwargs: Mapping[str, Any]) -> Mapping[str, Any]: + """Share a reusable message sequence between telemetry and the SDK.""" + messages = kwargs.get("messages") + if isinstance(messages, Iterable) and not isinstance(messages, (list, tuple)): + # An iterable may share a single iterator without implementing __next__. + # Copy kwargs, not the message values, and retain ordinary sequences as-is. + return {**kwargs, "messages": list(messages)} + return kwargs + + class _MessagesWrapper(_BaseWrapper): """ Wrapper for the pipeline processing @@ -279,6 +290,7 @@ def __call__( return wrapped(*args, **kwargs) wrapped = self._without_native_span(wrapped, instance) + kwargs = _snapshot_messages(kwargs) arguments = kwargs llm_input_messages = dict(arguments).pop("messages", None) llm_prompt = dict(arguments).pop("system", None) @@ -337,6 +349,7 @@ async def __call__( return await wrapped(*args, **kwargs) wrapped = self._without_native_span_async(wrapped, instance) + kwargs = _snapshot_messages(kwargs) arguments = kwargs llm_prompt = dict(arguments).pop("system", None) llm_input_messages = dict(arguments).pop("messages", None) @@ -419,6 +432,7 @@ def __call__( return wrapped(*args, **kwargs) wrapped = self._without_native_span(wrapped, instance) + kwargs = _snapshot_messages(kwargs) arguments = kwargs llm_input_messages = arguments.get("messages", None) llm_prompt = dict(arguments).pop("system", None) @@ -779,6 +793,10 @@ def _get_output_messages(response: Any) -> Any: for block in response.content: yield f"{GEN_AI_OUTPUT_MESSAGES}.{0}.{MESSAGE_ROLE}", response.role if isinstance(block, ToolUseBlock): + yield ( + f"{GEN_AI_OUTPUT_MESSAGES}.{0}.{MESSAGE_TOOL_CALLS}.{tool_index}.{TOOL_CALL_ID}", + block.id, + ) yield ( f"{GEN_AI_OUTPUT_MESSAGES}.{0}.{MESSAGE_TOOL_CALLS}.{tool_index}.{TOOL_CALL_FUNCTION_NAME}", block.name, diff --git a/python/frameworks/anthropic/traceai_anthropic/version.py b/python/frameworks/anthropic/traceai_anthropic/version.py index 2ae9a1c63..af77e1197 100644 --- a/python/frameworks/anthropic/traceai_anthropic/version.py +++ b/python/frameworks/anthropic/traceai_anthropic/version.py @@ -1 +1 @@ -__version__ = "0.1.13" # x-release-please-version +__version__ = "0.1.14" # x-release-please-version diff --git a/python/frameworks/autogen/CHANGELOG.md b/python/frameworks/autogen/CHANGELOG.md index c3a431601..8b0001bbf 100644 --- a/python/frameworks/autogen/CHANGELOG.md +++ b/python/frameworks/autogen/CHANGELOG.md @@ -1,3 +1,11 @@ +## [0.2.2] - pending release + +### Changed +- traceai-autogen now requires fi-instrumentation-otel 1.0.0 or later. Action: Upgrade fi-instrumentation-otel if your environment pins an earlier version. + +### Fixed +- AgentChat spans use the standard content attributes, so the tracing hide-input and hide-output settings apply, and a team span records its last generated message as output, not the task. + ## [0.2.1] - 2026-10-18 ### Fixed - Agent, team and tool spans were created with a raw OpenTelemetry tracer, so the diff --git a/python/frameworks/autogen/README.md b/python/frameworks/autogen/README.md index d399d6a18..be0892bd0 100644 --- a/python/frameworks/autogen/README.md +++ b/python/frameworks/autogen/README.md @@ -22,6 +22,7 @@ pip install "pyautogen>=0.2.0,<0.3" **AutoGen v0.4 (AgentChat)**: ```bash pip install "autogen-agentchat>=0.4.0" +pip install "autogen-ext[openai]" ``` ## Quick Start diff --git a/python/frameworks/autogen/poetry.lock b/python/frameworks/autogen/poetry.lock index 5e357d3f2..38c151b6a 100644 --- a/python/frameworks/autogen/poetry.lock +++ b/python/frameworks/autogen/poetry.lock @@ -2013,4 +2013,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.1" python-versions = ">=3.10,<3.14" -content-hash = "41e5a5f94dac7618cc5f8d5afa11c8562ddd2b4257f79c9038d01f9340fbb739" +content-hash = "aca5df05809b9b66bac204f3bd3b8e56a79fe0a64c0c399d185643027741fb29" diff --git a/python/frameworks/autogen/pyproject.toml b/python/frameworks/autogen/pyproject.toml index 5bd71171f..ad3670d26 100644 --- a/python/frameworks/autogen/pyproject.toml +++ b/python/frameworks/autogen/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "traceAI-autogen" -version = "0.2.1" +version = "0.2.2" description = "OpenTelemetry instrumentation for AutoGen AgentChat and legacy v0.2" authors = ["Future AGI "] readme = "README.md" @@ -28,7 +28,7 @@ classifiers = [ [tool.poetry.dependencies] python = ">=3.10,<3.14" -fi-instrumentation-otel = ">=0.1.11" +fi-instrumentation-otel = ">=1.0.0" autogen-agentchat = ">=0.4.0" [tool.poetry.group.dev.dependencies] diff --git a/python/frameworks/autogen/tests/test_canonical_v04_attributes.py b/python/frameworks/autogen/tests/test_canonical_v04_attributes.py new file mode 100644 index 000000000..f4312f0ae --- /dev/null +++ b/python/frameworks/autogen/tests/test_canonical_v04_attributes.py @@ -0,0 +1,240 @@ +"""T5-PY-005: fake AgentChat calls export collector-readable attributes.""" + +import asyncio +import sys +from dataclasses import dataclass +from types import SimpleNamespace + +import pytest +from fi_instrumentation import FITracer, TraceConfig +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter + +from traceai_autogen._v04_wrapper import ( + _record_team_result, + _team_attributes, + wrap_agent_on_messages, + wrap_agent_on_messages_stream, + wrap_team_run, + wrap_tool_execution, +) + + +@dataclass +class Message: + content: str + source: str + + +@dataclass +class Response: + chat_message: Message + + +@dataclass +class TaskResult: + messages: list + stop_reason: str = "complete" + + +@pytest.fixture +def recording(): + exporter = InMemorySpanExporter() + provider = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + yield provider.get_tracer(__name__), exporter + provider.shutdown() + + +POLICIES = ( + "hide_inputs", "hide_outputs", "hide_input_messages", + "hide_output_messages", "hide_input_text", "hide_output_text", +) + + +@pytest.mark.parametrize("team", [False, True], ids=["agent", "team"]) +@pytest.mark.parametrize("streaming", [False, True], ids=["call", "stream"]) +@pytest.mark.parametrize("policy", [None, *POLICIES]) +def test_agentchat_exports_canonical_content_and_kind(recording, monkeypatch, team, streaming, policy): + # Only the wrapped library is fake; real FITracer/OTel exercise export and + # privacy. Streaming uses the same Response/TaskResult boundary as AgentChat. + monkeypatch.setitem(sys.modules, "autogen_agentchat.base", SimpleNamespace( + Response=Response, TaskResult=TaskResult, + )) + raw_tracer, exporter = recording + config = TraceConfig(**{flag: flag == policy for flag in POLICIES}) + tracer = FITracer(raw_tracer, config=config) + instance = SimpleNamespace(name="assistant") + messages = [Message("question", "user")] + answer = Message("answer", "assistant") + terminal = TaskResult([*messages, answer]) if team else Response(answer) + arguments = {"task": "question"} if team else {"messages": messages} + calls = [] + cleanup = [] + + async def original(self, **kwargs): + calls.append((self, kwargs)) + return terminal + + async def original_stream(self, **kwargs): + calls.append((self, kwargs)) + yield answer + yield terminal + cleanup.append(True) + + if team: + wrapped = wrap_team_run( + original_stream if streaming else original, tracer, + "run_stream" if streaming else "run", config, + ) + else: + factory = wrap_agent_on_messages_stream if streaming else wrap_agent_on_messages + wrapped = factory(original_stream if streaming else original, tracer, config) + + async def exercise(): + if not streaming: + assert await wrapped(instance, **arguments) is terminal + return + stream = wrapped(instance, **arguments) + assert await anext(stream) is answer + assert await anext(stream) is terminal + assert cleanup == [] # Recording must not prefetch post-result work. + assert len(exporter.get_finished_spans()) == 1 + with pytest.raises(StopAsyncIteration): + await anext(stream) + assert cleanup == [True] + await stream.aclose() + + asyncio.run(exercise()) + assert calls == [(instance, arguments)] + if not team: + assert calls[0][1]["messages"] is messages + assert messages[0].content == "question" + assert answer.content == "answer" + span, = exporter.get_finished_spans() + attributes = span.attributes + # These exact keys feed resolveObservationType/contentValue in the collector; + # autogen.* aliases alone do not populate its observation/content columns. + assert attributes["gen_ai.span.kind"] == ("CHAIN" if team else "AGENT") + assert attributes["autogen.span_kind"] == ("team_run" if team else "agent_run") + input_alias = "autogen.task.content" if team else "autogen.input.last_message" + output_alias = "autogen.task.result" if team else "autogen.output.content" + expected_input = "__REDACTED__" if policy in POLICIES[::2] else "question" + expected_output = "__REDACTED__" if policy in POLICIES[1::2] else "answer" + assert attributes["input.value"] == attributes[input_alias] == expected_input + assert attributes["output.value"] == attributes[output_alias] == expected_output + if expected_input == "__REDACTED__": + assert "question" not in span.to_json() + if expected_output == "__REDACTED__": + assert "answer" not in span.to_json() + + +@pytest.mark.parametrize("task,expected", [ + ("question", "question"), + (Message("question", "user"), "question"), + ([Message("first", "user"), Message("question", "user")], '["first", "question"]'), +]) +def test_team_canonical_input_extracts_message_content(task, expected): + attributes = _team_attributes(SimpleNamespace(), task, "run", TraceConfig()) + assert attributes["input.value"] == expected + + +@pytest.mark.parametrize("messages", [[], [Message("question", "user")]]) +def test_team_without_answer_does_not_export_task_as_output(recording, messages): + tracer, exporter = recording + with tracer.start_as_current_span("empty-result") as span: + _record_team_result(span, TaskResult(messages), TraceConfig(), input_message_count=len(messages)) + finished, = exporter.get_finished_spans() + assert "output.value" not in finished.attributes + assert "autogen.task.result" not in finished.attributes + + +@pytest.mark.parametrize("policy", ["hide_inputs", "hide_input_messages", "hide_input_text"]) +@pytest.mark.parametrize("streaming", [False, True], ids=["call", "stream"]) +@pytest.mark.parametrize("sequence", [False, True], ids=["message", "sequence"]) +def test_team_input_only_non_user_task_stays_hidden(recording, monkeypatch, policy, streaming, sequence): + monkeypatch.setitem(sys.modules, "autogen_agentchat.base", SimpleNamespace( + Response=Response, TaskResult=TaskResult, + )) + raw_tracer, exporter = recording + # Hide one input channel at a time; every output channel stays enabled. + config = TraceConfig(**{flag: flag == policy for flag in POLICIES}) + tracer = FITracer(raw_tracer, config=config) + messages = [Message("private-task-text", "operator")] + if sequence: + messages.insert(0, Message("private-earlier-text", "operator")) + task = messages if sequence else messages[0] + terminal = TaskResult(list(messages)) + instance = SimpleNamespace() + calls = [] + cleanup = [] + + async def original(self, *, task): + calls.append((self, task)) + return terminal + + async def original_stream(self, *, task): + calls.append((self, task)) + for message in messages: + yield message + yield terminal + cleanup.append(True) + + wrapped = wrap_team_run( + original_stream if streaming else original, tracer, + "run_stream" if streaming else "run", config, + ) + + async def exercise(): + if not streaming: + assert await wrapped(instance, task=task) is terminal + return + stream = wrapped(instance, task=task) + for message in messages: + assert await anext(stream) is message + assert await anext(stream) is terminal + assert cleanup == [] + assert len(exporter.get_finished_spans()) == 1 + with pytest.raises(StopAsyncIteration): + await anext(stream) + assert cleanup == [True] + await stream.aclose() + + asyncio.run(exercise()) + assert len(calls) == 1 + assert calls[0][0] is instance + assert calls[0][1] is task + assert terminal.messages == messages + assert [message.content for message in messages] == ( + ["private-earlier-text", "private-task-text"] if sequence else ["private-task-text"] + ) + finished, = exporter.get_finished_spans() + assert finished.attributes["input.value"] == "__REDACTED__" + assert finished.attributes["autogen.task.content"] == "__REDACTED__" + assert "output.value" not in finished.attributes + assert "autogen.task.result" not in finished.attributes + assert "private-task-text" not in finished.to_json() + assert "private-earlier-text" not in finished.to_json() + + +@pytest.mark.parametrize("asynchronous", [False, True], ids=["sync", "async"]) +def test_tool_exports_canonical_kind(recording, asynchronous): + tracer, exporter = recording + returned = {"answer": 42} + calls = [] + + def original(value): + calls.append(value) + return returned + + async def original_async(value): + return original(value) + + wrapped = wrap_tool_execution(original_async if asynchronous else original, tracer, "lookup") + result = asyncio.run(wrapped("question")) if asynchronous else wrapped("question") + assert result is returned + assert calls == ["question"] + span, = exporter.get_finished_spans() + assert span.attributes["gen_ai.span.kind"] == "TOOL" + assert span.attributes["autogen.span_kind"] == "tool_call" diff --git a/python/frameworks/autogen/tests/test_team_input_boundary.py b/python/frameworks/autogen/tests/test_team_input_boundary.py new file mode 100644 index 000000000..992bbc5c7 --- /dev/null +++ b/python/frameworks/autogen/tests/test_team_input_boundary.py @@ -0,0 +1,94 @@ +"""Real AgentChat task/result boundaries respect independent input privacy.""" + +import asyncio + +import pytest +from autogen_agentchat.agents import UserProxyAgent +from autogen_agentchat.base import TaskResult +from autogen_agentchat.conditions import MaxMessageTermination +from autogen_agentchat.messages import TextMessage +from autogen_agentchat.teams import RoundRobinGroupChat +from fi_instrumentation import FITracer, TraceConfig +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter + +from traceai_autogen._v04_wrapper import wrap_team_run + + +@pytest.mark.parametrize("policy", ["hide_inputs", "hide_input_messages", "hide_input_text"]) +@pytest.mark.parametrize("streaming", [False, True], ids=["call", "stream"]) +@pytest.mark.parametrize("task_form", ["message", "list", "string"]) +@pytest.mark.parametrize("input_only", [True, False], ids=["no-answer", "user-answer"]) +@pytest.mark.parametrize("output_task_messages", [True, False], ids=["echo-task", "omit-task"]) +def test_real_team_task_boundary(policy, streaming, task_form, input_only, output_task_messages): + exporter = InMemorySpanExporter() + provider = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + config = TraceConfig(**{ + flag: flag == policy for flag in ( + "hide_inputs", "hide_input_messages", "hide_input_text", + "hide_outputs", "hide_output_messages", "hide_output_text", + ) + }) + tracer = FITracer(provider.get_tracer(__name__), config=config) + inputs = [TextMessage(content="private-task-text", source="user" if task_form == "string" else "operator")] + if task_form == "list": + inputs.insert(0, TextMessage(content="private-earlier-text", source="operator")) + task = inputs if task_form == "list" else inputs[0].content if task_form == "string" else inputs[0] + calls = [] + + async def reply(prompt, cancellation_token=None): + calls.append(prompt) + return "public-answer" + + async def exercise(): + # Termination on the supplied task creates a genuine input-only result. + # A genuine answer may itself have source="user": source is not a role. + team = RoundRobinGroupChat( + [UserProxyAgent("user", input_func=reply)], + termination_condition=MaxMessageTermination(len(inputs)) if input_only else None, + max_turns=1, + ) + method = "run_stream" if streaming else "run" + wrapped = wrap_team_run(getattr(type(team), method), tracer, method, config) + arguments = {"task": task, "output_task_messages": output_task_messages} + if streaming: + items = [item async for item in wrapped(team, **arguments)] + result = items[-1] + assert isinstance(result, TaskResult) + assert items[:-1] == result.messages + else: + result = await wrapped(team, **arguments) + input_count = len(inputs) if output_task_messages else 0 + assert [message.content for message in result.messages[:input_count]] == [ + message.content for message in inputs[:input_count] + ] + if input_only: + assert len(result.messages) == input_count + else: + # UserProxyAgent can also emit a UserInputRequestedEvent. + assert len(result.messages) > input_count + assert result.messages[-1].source == "user" + assert result.messages[-1].content == "public-answer" + + try: + asyncio.run(exercise()) + finished, = exporter.get_finished_spans() + assert finished.attributes["input.value"] == "__REDACTED__" + assert finished.attributes["autogen.task.content"] == "__REDACTED__" + if input_only: + assert calls == [] + assert "output.value" not in finished.attributes + assert "autogen.task.result" not in finished.attributes + else: + assert len(calls) == 1 + assert finished.attributes["output.value"] == "public-answer" + assert finished.attributes["autogen.task.result"] == "public-answer" + assert "private-task-text" not in finished.to_json() + assert "private-earlier-text" not in finished.to_json() + assert [message.content for message in inputs] == ( + ["private-earlier-text", "private-task-text"] if task_form == "list" else ["private-task-text"] + ) + finally: + provider.shutdown() diff --git a/python/frameworks/autogen/traceai_autogen/__init__.py b/python/frameworks/autogen/traceai_autogen/__init__.py index 88dffa0c0..ec04731a6 100644 --- a/python/frameworks/autogen/traceai_autogen/__init__.py +++ b/python/frameworks/autogen/traceai_autogen/__init__.py @@ -19,7 +19,7 @@ _MODULE_V04_AGENTS = "autogen_agentchat.agents" _MODULE_V04_TEAMS = "autogen_agentchat.teams" -__version__ = "0.2.1" # x-release-please-version +__version__ = "0.2.2" # x-release-please-version _instruments = ("autogen-agentchat>=0.4.0",) diff --git a/python/frameworks/autogen/traceai_autogen/_v04_wrapper.py b/python/frameworks/autogen/traceai_autogen/_v04_wrapper.py index 7913444eb..55b923579 100644 --- a/python/frameworks/autogen/traceai_autogen/_v04_wrapper.py +++ b/python/frameworks/autogen/traceai_autogen/_v04_wrapper.py @@ -222,6 +222,7 @@ def _mask_content(config, value, output=False): def _agent_attributes(agent, messages, config): info = extract_agent_info(agent) attributes = { + SpanAttributes.GEN_AI_SPAN_KIND: "AGENT", "autogen.span_kind": "agent_run", "autogen.run.id": str(uuid.uuid4()), "autogen.agent.name": info["agent_name"], @@ -234,7 +235,9 @@ def _agent_attributes(agent, messages, config): if messages: attributes["autogen.input.message_count"] = len(messages) last = extract_message_info(messages[-1]) - attributes["autogen.input.last_message"] = _mask_content(config, last.get("content", "")) + content = _mask_content(config, last.get("content", "")) + attributes[SpanAttributes.INPUT_VALUE] = content + attributes["autogen.input.last_message"] = content attributes["autogen.input.last_source"] = last.get("source", "") return attributes @@ -243,7 +246,9 @@ def _record_agent_result(span, result, config): if result is not None and hasattr(result, "chat_message"): info = extract_message_info(result.chat_message) span.set_attribute("autogen.output.message_type", info.get("message_type", "")) - span.set_attribute("autogen.output.content", _mask_content(config, info.get("content", ""), output=True)) + content = _mask_content(config, info.get("content", ""), output=True) + span.set_attribute(SpanAttributes.OUTPUT_VALUE, content) + span.set_attribute("autogen.output.content", content) if getattr(result, "inner_messages", None): span.set_attribute("autogen.output.inner_message_count", len(result.inner_messages)) @@ -251,6 +256,7 @@ def _record_agent_result(span, result, config): def _team_attributes(team, task, method, config): info = extract_team_info(team) attributes = { + SpanAttributes.GEN_AI_SPAN_KIND: "CHAIN", "autogen.span_kind": "team_run", "autogen.run.id": str(uuid.uuid4()), "autogen.team.type": info["team_type"], @@ -263,14 +269,39 @@ def _team_attributes(team, task, method, config): attributes["autogen.team.participants"] = json.dumps(info["participants"]) if task: attributes["autogen.task.content"] = _mask_content(config, safe_serialize(task, 1000)) + # AgentChat accepts a string, a message, or a sequence of messages. + # Keep the historical alias format, but show content rather than a + # message object's repr in the collector's normal input column. + content = ([getattr(message, "content", message) for message in task] + if isinstance(task, (list, tuple)) else getattr(task, "content", task)) + attributes[SpanAttributes.INPUT_VALUE] = _mask_content(config, safe_serialize(content, 1000)) return attributes -def _record_team_result(span, result, config): +def _team_result_recorder(args, kwargs, config): + task = kwargs.get("task", args[0] if args else None) + # AgentChat puts task messages before generated messages in TaskResult. + # Snapshot the boundary per call, never infer roles from caller-set sources. + input_message_count = 0 + if task is not None and kwargs.get("output_task_messages", True): + input_message_count = len(task) if isinstance(task, (list, tuple)) else 1 + return functools.partial( + _record_team_result, config=config, input_message_count=input_message_count, + ) + + +def _record_team_result(span, result, config, input_message_count=0): if result is None: return if hasattr(result, "messages"): span.set_attribute("autogen.task.message_count", len(result.messages)) + if len(result.messages) > input_message_count: + last = result.messages[-1] + # An input-only partial TaskResult has no answer under any source. + if hasattr(last, "content"): + content = _mask_content(config, extract_message_info(last)["content"], output=True) + span.set_attribute(SpanAttributes.OUTPUT_VALUE, content) + span.set_attribute("autogen.task.result", content) if hasattr(result, "stop_reason"): # TextMentionTermination quotes model text in the stop reason too. span.set_attribute("autogen.task.stop_reason", _mask_content(config, str(result.stop_reason), output=True)) @@ -301,9 +332,10 @@ def _record_error(span, error, config): span.record_exception(error) -def _wrap_stream(original_method, tracer, name, attributes, record_result, kind, config): +def _wrap_stream(original_method, tracer, name, attributes, result_recorder, kind, config): @functools.wraps(original_method) async def stream_wrapper(self, *args, **kwargs): + record_result = result_recorder(args, kwargs) # Remain a true async generator, forwarding each consumer operation once. # No prefetch: Response may precede slow/failing SDK post-yield cleanup. span = tracer.start_span(name(self), kind=kind, attributes=attributes(self, args, kwargs)) @@ -435,7 +467,7 @@ def wrap_agent_on_messages_stream(original_method: Callable, tracer: trace_api.T original_method, tracer, lambda agent: f"autogen.agent.on_messages_stream.{getattr(agent, 'name', 'agent')}", lambda agent, args, kwargs: _agent_attributes(agent, kwargs.get("messages", args[0] if args else None), config), - lambda span, result: _record_agent_result(span, result, config), SpanKind.INTERNAL, config, + lambda args, kwargs: functools.partial(_record_agent_result, config=config), SpanKind.INTERNAL, config, ) @@ -446,12 +478,13 @@ def wrap_team_run(original_method: Callable, tracer: trace_api.Tracer, method_na original_method, tracer, lambda team: f"{TEAM_RUN_STREAM_SPAN_NAME}.{type(team).__name__}", lambda team, args, kwargs: _team_attributes(team, kwargs.get("task", args[0] if args else None), method_name, config), - lambda span, result: _record_team_result(span, result, config), SpanKind.CLIENT, config, + lambda args, kwargs: _team_result_recorder(args, kwargs, config), SpanKind.CLIENT, config, ) @functools.wraps(original_method) async def async_wrapper(self, *args, **kwargs): task = kwargs.get("task", args[0] if args else None) + record_result = _team_result_recorder(args, kwargs, config) start_time = time.time() with tracer.start_as_current_span( f"{TEAM_RUN_SPAN_NAME}.{type(self).__name__}", kind=SpanKind.CLIENT, @@ -462,7 +495,7 @@ async def async_wrapper(self, *args, **kwargs): # AgentChat's task is keyword-only and optional. Preserve the # entire call shape, including task-less continuation. result = await original_method(self, *args, **kwargs) - _record_team_result(span, result, config) + record_result(span, result) span.set_attribute("autogen.is_error", False) span.set_status(Status(StatusCode.OK)) return result @@ -496,6 +529,7 @@ def wrap_tool_execution( @functools.wraps(tool_func) async def async_wrapper(*args, **kwargs): attributes = { + SpanAttributes.GEN_AI_SPAN_KIND: "TOOL", "autogen.span_kind": "tool_call", "autogen.tool.name": tool_name, } @@ -539,6 +573,7 @@ async def async_wrapper(*args, **kwargs): @functools.wraps(tool_func) def sync_wrapper(*args, **kwargs): attributes = { + SpanAttributes.GEN_AI_SPAN_KIND: "TOOL", "autogen.span_kind": "tool_call", "autogen.tool.name": tool_name, } diff --git a/python/frameworks/autogen/traceai_autogen/version.py b/python/frameworks/autogen/traceai_autogen/version.py index 130276d81..210c13fdd 100644 --- a/python/frameworks/autogen/traceai_autogen/version.py +++ b/python/frameworks/autogen/traceai_autogen/version.py @@ -1 +1 @@ -__version__ = "0.2.1" # x-release-please-version +__version__ = "0.2.2" # x-release-please-version diff --git a/python/frameworks/bedrock/CHANGELOG.md b/python/frameworks/bedrock/CHANGELOG.md index 3333a9104..c08b9713d 100644 --- a/python/frameworks/bedrock/CHANGELOG.md +++ b/python/frameworks/bedrock/CHANGELOG.md @@ -1,5 +1,15 @@ +## [0.1.12] - pending release + +### Changed +- traceai-bedrock now requires fi-instrumentation-otel 1.0.0 or later. Action: Upgrade fi-instrumentation-otel if your environment pins an earlier version. + +### Fixed +- converse_stream calls are instrumented. +- invoke_model_with_response_stream spans for non-Anthropic models end when the caller finishes or closes the stream, and record the model, plus output and token usage when the stream includes them. +- invoke_model calls with a file-like request body are accepted, and tracing leaves the body unread for the SDK. + ## [0.1.11] - 2026-10-29 ### Fixed diff --git a/python/frameworks/bedrock/poetry.lock b/python/frameworks/bedrock/poetry.lock index d0ab2da10..d28f4e9aa 100644 --- a/python/frameworks/bedrock/poetry.lock +++ b/python/frameworks/bedrock/poetry.lock @@ -1009,4 +1009,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.1" python-versions = ">=3.10,<3.14" -content-hash = "514ec408c7f1cd4b795594dc97e242d8e82a0bfd3d7968f5a680383f0b7fe52b" +content-hash = "128d8e27db4e2a6cf5138640949368dfeabf1a74e6f8fb7b47c90f362f0390af" diff --git a/python/frameworks/bedrock/pyproject.toml b/python/frameworks/bedrock/pyproject.toml index d4151f756..921770c7a 100644 --- a/python/frameworks/bedrock/pyproject.toml +++ b/python/frameworks/bedrock/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "traceAI-bedrock" -version = "0.1.11" +version = "0.1.12" description = "OpenTelemetry instrumentation for Bedrock" authors = ["Future AGI "] readme = "README.md" @@ -29,7 +29,7 @@ classifiers = [ [tool.poetry.dependencies] python = ">=3.10,<3.14" dacite = ">=1.8.1" -fi-instrumentation-otel = ">=0.1.11" +fi-instrumentation-otel = ">=1.0.0" botocore = "*" boto3 = ">=1.38.17" diff --git a/python/frameworks/bedrock/tests/test_bedrock_converse_stream.py b/python/frameworks/bedrock/tests/test_bedrock_converse_stream.py new file mode 100644 index 000000000..ea6f58fc4 --- /dev/null +++ b/python/frameworks/bedrock/tests/test_bedrock_converse_stream.py @@ -0,0 +1,292 @@ +"""T4-PY-CLOUD-006: ConverseStream through the client-creation installation path. + +All provider operations and event streams are fakes; spans export only to memory. +""" + +from copy import deepcopy +from types import SimpleNamespace +from unittest.mock import Mock + +import botocore +import pytest +from botocore.client import ClientCreator +from botocore.eventstream import EventStream +from opentelemetry import context as context_api, trace +from opentelemetry.context import _SUPPRESS_INSTRUMENTATION_KEY +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +from opentelemetry.trace import StatusCode + +import traceai_bedrock as bedrock +from fi_instrumentation import TraceConfig +from fi_instrumentation.fi_types import ( + FiLLMProviderValues, + FiSpanKindValues, + MessageAttributes, + MessageContentAttributes, + SpanAttributes, + ToolCallAttributes, +) + + +REQUEST = { + "modelId": "meta.llama3-8b-instruct-v1:0", + "system": [{"text": "Be concise."}], + "messages": [{"role": "user", "content": [{"text": "hello"}]}], + "inferenceConfig": {"maxTokens": 64, "temperature": 0}, +} + + +class FakeEventStream(EventStream): + """An EventStream-shaped, lazy source with observable iteration and close.""" + + def __init__(self, events=(), error=None, close_error=None): + self.events = iter(events) + self.error = error + self.close_error = close_error + self.consumed = 0 + self.close_calls = 0 + self.active_spans = [] + self.initial_response = object() + self.close_result = object() + + def __iter__(self): + for event in self.events: + self.active_spans.append(trace.get_current_span().get_span_context()) + self.consumed += 1 + yield event + if self.error is not None: + raise self.error + + def get_initial_response(self): + return self.initial_response + + def close(self): + self.close_calls += 1 + if self.close_error is not None: + raise self.close_error + return self.close_result + + +@pytest.fixture +def installed(monkeypatch): + exporter = InMemorySpanExporter() + provider = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + pending = [] + + def create_client(_creator, service_name, **kwargs): + return pending.pop(0) + + # Exercise the actual wrap_function_wrapper installation without creating an + # AWS session, resolving credentials, or constructing an HTTP transport. + monkeypatch.setattr(ClientCreator, "create_client", create_client) + monkeypatch.setattr(botocore, "__version__", "1.39.3") + monkeypatch.setattr(bedrock, "Protect", None) + instrumentor = bedrock.BedrockInstrumentor() + instrumentor.instrument( + tracer_provider=provider, + skip_dep_check=True, + config=TraceConfig( + hide_inputs=False, + hide_outputs=False, + hide_input_messages=False, + hide_output_messages=False, + hide_input_text=False, + hide_output_text=False, + hide_llm_invocation_parameters=False, + ), + ) + + def make(stream=None, error=None, service="bedrock-runtime", has_stream=True): + if stream is None: + stream = FakeEventStream() + response = {"stream": stream, "ResponseMetadata": {"HTTPStatusCode": 200}} + operation = Mock(return_value=response, side_effect=error) + raw_client = SimpleNamespace( + invoke_model=Mock(), + invoke_model_with_response_stream=Mock(), + converse=Mock(), + ) + if has_stream: + raw_client.converse_stream = operation + pending.append(raw_client) + client = object.__new__(ClientCreator).create_client(service_name=service) + return SimpleNamespace( + client=client, operation=operation, response=response, stream=stream + ) + + yield SimpleNamespace(make=make, exporter=exporter, provider=provider) + instrumentor.uninstrument() + provider.shutdown() + + +def test_converse_stream_installation_text_tools_and_trailing_usage(installed): + events = [ + {"messageStart": {"role": "assistant"}}, + {"contentBlockDelta": {"contentBlockIndex": 0, "delta": {"text": "Hel"}}}, + {"contentBlockDelta": {"contentBlockIndex": 0, "delta": {"text": "lo"}}}, + {"contentBlockStop": {"contentBlockIndex": 0}}, + {"contentBlockStart": {"contentBlockIndex": 1, "start": { + "toolUse": {"toolUseId": "fake-tool-1", "name": "weather"} + }}}, + {"contentBlockDelta": {"contentBlockIndex": 1, "delta": { + "toolUse": {"input": '{"city":'} + }}}, + {"contentBlockDelta": {"contentBlockIndex": 1, "delta": { + "toolUse": {"input": '"Paris"}'} + }}}, + {"contentBlockStop": {"contentBlockIndex": 1}}, + {"messageStop": {"stopReason": "tool_use"}}, + {"metadata": {"usage": {"inputTokens": 0, "outputTokens": 4, "totalTokens": 4}}}, + ] + original_events = deepcopy(events) + case = installed.make(FakeEventStream(events)) + request = deepcopy(REQUEST) + parent_tracer = installed.provider.get_tracer(__name__) + with parent_tracer.start_as_current_span("parent") as parent: + response = case.client.converse_stream(**request) + assert response is case.response + assert case.client.converse_stream is not case.operation + assert isinstance(response["stream"], EventStream) + assert response["stream"].get_initial_response() is case.stream.initial_response + assert case.stream.consumed == 0 + assert installed.exporter.get_finished_spans() == () + assert trace.get_current_span() is parent + iterator = iter(response["stream"]) + # messageStop is not terminal: usage arrives in the following metadata. + for event in events[:-1]: + assert next(iterator) is event + assert trace.get_current_span() is parent + assert installed.exporter.get_finished_spans() == () + assert next(iterator) is events[-1] + with pytest.raises(StopIteration): + next(iterator) + spans = installed.exporter.get_finished_spans() + assert len(spans) == 1 + span = spans[0] + assert span.parent.span_id == parent.get_span_context().span_id + assert all(context.span_id == span.context.span_id for context in case.stream.active_spans) + assert response["stream"].close() is case.stream.close_result + assert len(installed.exporter.get_finished_spans()) == 1 + + case.operation.assert_called_once_with(**REQUEST) + assert request == REQUEST + assert events == original_events + assert span.name == "bedrock.converse_stream" + assert span.status.status_code is StatusCode.OK + attrs = span.attributes + assert attrs[SpanAttributes.GEN_AI_SPAN_KIND] == FiSpanKindValues.LLM.value + assert attrs[SpanAttributes.GEN_AI_PROVIDER_NAME] == FiLLMProviderValues.AWS.value + assert attrs[SpanAttributes.GEN_AI_REQUEST_MODEL] == REQUEST["modelId"] + assert attrs[SpanAttributes.INPUT_VALUE] == "hello" + assert attrs[f"{SpanAttributes.GEN_AI_INPUT_MESSAGES}.0.{MessageAttributes.MESSAGE_ROLE}"] == "system" + assert attrs[f"{SpanAttributes.GEN_AI_INPUT_MESSAGES}.1.{MessageAttributes.MESSAGE_CONTENTS}.0.{MessageContentAttributes.MESSAGE_CONTENT_TEXT}"] == "hello" + assert attrs[SpanAttributes.OUTPUT_VALUE] == "Hello" + output = f"{SpanAttributes.GEN_AI_OUTPUT_MESSAGES}.0" + assert attrs[f"{output}.{MessageAttributes.MESSAGE_ROLE}"] == "assistant" + assert attrs[f"{output}.{MessageAttributes.MESSAGE_CONTENTS}.0.{MessageContentAttributes.MESSAGE_CONTENT_TEXT}"] == "Hello" + tool = f"{output}.{MessageAttributes.MESSAGE_TOOL_CALLS}.0" + assert attrs[f"{tool}.{ToolCallAttributes.TOOL_CALL_ID}"] == "fake-tool-1" + assert attrs[f"{tool}.{ToolCallAttributes.TOOL_CALL_FUNCTION_NAME}"] == "weather" + assert attrs[f"{tool}.{ToolCallAttributes.TOOL_CALL_FUNCTION_ARGUMENTS_JSON}"] == '{"city":"Paris"}' + assert attrs[SpanAttributes.GEN_AI_USAGE_INPUT_TOKENS] == 0 + assert attrs[SpanAttributes.GEN_AI_USAGE_OUTPUT_TOKENS] == 4 + assert attrs[SpanAttributes.GEN_AI_USAGE_TOTAL_TOKENS] == 4 + + +@pytest.mark.parametrize("phase", ["request", "iteration", "close"]) +def test_converse_stream_errors_keep_original_exception_and_finish_once(installed, phase): + error = RuntimeError("fake provider failure") + stream = FakeEventStream( + [{"contentBlockDelta": {"contentBlockIndex": 0, "delta": {"text": "partial"}}}], + error=error if phase == "iteration" else None, + close_error=error if phase == "close" else None, + ) + case = installed.make(stream, error=error if phase == "request" else None) + with pytest.raises(RuntimeError) as caught: + response = case.client.converse_stream(**deepcopy(REQUEST)) + if phase == "iteration": + list(response["stream"]) + elif phase == "close": + response["stream"].close() + assert caught.value is error + spans = installed.exporter.get_finished_spans() + assert len(spans) == 1 + assert spans[0].status.status_code is StatusCode.ERROR + assert len([event for event in spans[0].events if event.name == "exception"]) == 1 + if phase == "iteration": + assert spans[0].attributes[SpanAttributes.OUTPUT_VALUE] == "partial" + response["stream"].close() + assert len(installed.exporter.get_finished_spans()) == 1 + + +@pytest.mark.parametrize("consume_first", [False, True]) +def test_converse_stream_early_close_preserves_close_and_partial_output(installed, consume_first): + event = {"contentBlockDelta": {"contentBlockIndex": 0, "delta": {"text": "partial"}}} + case = installed.make(FakeEventStream([event])) + response = case.client.converse_stream(**deepcopy(REQUEST)) + if consume_first: + assert next(iter(response["stream"])) is event + assert response["stream"].close() is case.stream.close_result + assert response["stream"].close() is case.stream.close_result + assert case.stream.close_calls == 2 + assert case.stream.consumed == int(consume_first) + spans = installed.exporter.get_finished_spans() + assert len(spans) == 1 + assert spans[0].status.status_code is StatusCode.UNSET + if consume_first: + assert spans[0].attributes[SpanAttributes.OUTPUT_VALUE] == "partial" + + +def test_converse_stream_error_event_is_returned_unchanged(installed): + event = {"modelStreamErrorException": {"message": "fake stream failure"}} + case = installed.make(FakeEventStream([event])) + stream = case.client.converse_stream(**deepcopy(REQUEST))["stream"] + assert next(iter(stream)) is event + stream.close() + spans = installed.exporter.get_finished_spans() + assert len(spans) == 1 + assert spans[0].status.status_code is StatusCode.ERROR + + +@pytest.mark.parametrize("during_creation", [False, True]) +def test_converse_stream_suppression_is_passthrough(installed, during_creation): + case = None if during_creation else installed.make() + suppression = context_api.attach(context_api.set_value(_SUPPRESS_INSTRUMENTATION_KEY, True)) + try: + if during_creation: + case = installed.make() + response = case.client.converse_stream(**deepcopy(REQUEST)) + assert response is case.response + assert response["stream"] is case.stream + list(response["stream"]) + finally: + context_api.detach(suppression) + assert installed.exporter.get_finished_spans() == () + + +def test_converse_stream_absent_on_older_client_is_not_required(installed): + case = installed.make(has_stream=False) + assert not hasattr(case.client, "converse_stream") + assert installed.exporter.get_finished_spans() == () + + +def test_converse_stream_other_services_are_unchanged(installed): + case = installed.make(service="s3") + assert case.client.converse_stream is case.operation + assert case.client.converse_stream(**deepcopy(REQUEST))["stream"] is case.stream + assert installed.exporter.get_finished_spans() == () + + +def test_converse_stream_invalid_request_reaches_provider(installed): + error = ValueError("fake provider validation error") + case = installed.make(error=error) + with pytest.raises(ValueError) as caught: + case.client.converse_stream(modelId=REQUEST["modelId"], messages=[]) + assert caught.value is error + case.operation.assert_called_once_with(modelId=REQUEST["modelId"], messages=[]) + spans = installed.exporter.get_finished_spans() + assert len(spans) == 1 + assert spans[0].status.status_code is StatusCode.ERROR diff --git a/python/frameworks/bedrock/tests/test_bedrock_invoke_model_body.py b/python/frameworks/bedrock/tests/test_bedrock_invoke_model_body.py new file mode 100644 index 000000000..44d99c4ba --- /dev/null +++ b/python/frameworks/bedrock/tests/test_bedrock_invoke_model_body.py @@ -0,0 +1,109 @@ +"""T4-PY-CLOUD-005: request capture must not reject or consume SDK bodies.""" + +import io +from types import SimpleNamespace +from unittest.mock import MagicMock, Mock, patch + +import pytest +from botocore.response import StreamingBody + +from fi_instrumentation.fi_types import SpanAttributes +from traceai_bedrock import _model_invocation_wrapper + + +MODEL_ID = "meta.llama3-8b-instruct-v1:0" +REQUEST_BODY = b'{"prompt":"hello"}' +RESPONSE_BODY = b'{"generation":"hello back"}' + + +@pytest.fixture +def invocation(): + """Exercise the real invocation wrapper without a client, exporter, or network.""" + tracer = MagicMock() + sdk = Mock( + return_value={ + "body": StreamingBody(io.BytesIO(RESPONSE_BODY), len(RESPONSE_BODY)), + "ResponseMetadata": {"HTTPStatusCode": 200}, + } + ) + client = SimpleNamespace(invoke_model=sdk, _unwrapped_invoke_model=sdk) + return SimpleNamespace( + invoke=_model_invocation_wrapper(tracer)(client), + sdk=sdk, + span=tracer.start_as_current_span.return_value.__enter__.return_value, + ) + + +@pytest.mark.parametrize("position", [0, 5], ids=["start", "nonzero"]) +def test_invoke_model_preserves_file_like_body(invocation, position): + with io.BytesIO(REQUEST_BODY) as body: + body.seek(position) + with ( + patch.object(body, "read", wraps=body.read) as read, + patch.object(body, "seek", wraps=body.seek) as seek, + ): + def sdk_invoke(**kwargs): + assert kwargs["body"] is body + read.assert_not_called() + seek.assert_not_called() + assert body.tell() == position + assert body.read() == REQUEST_BODY[position:] + return invocation.sdk.return_value + + invocation.sdk.side_effect = sdk_invoke + response = invocation.invoke( + modelId=MODEL_ID, contentType="application/json", body=body + ) + + invocation.sdk.assert_called_once_with( + modelId=MODEL_ID, contentType="application/json", body=body + ) + read.assert_called_once_with() + seek.assert_not_called() + assert not body.closed + + assert response is invocation.sdk.return_value + assert response["body"].read() == RESPONSE_BODY + + +@pytest.mark.parametrize( + "body", + [REQUEST_BODY.decode("utf-8"), REQUEST_BODY, bytearray(REQUEST_BODY)], + ids=["string", "bytes", "bytearray"], +) +def test_invoke_model_still_captures_inline_json(invocation, body): + response = invocation.invoke(modelId=MODEL_ID, body=body) + + invocation.sdk.assert_called_once_with(modelId=MODEL_ID, body=body) + invocation.span.set_attribute.assert_any_call(SpanAttributes.INPUT_VALUE, "hello") + assert response["body"].read() == RESPONSE_BODY + + +@pytest.mark.parametrize( + "body_factory", + [ + lambda: io.BytesIO(REQUEST_BODY), + lambda: b"not-json", + lambda: b"\xff", + lambda: b"[]", + lambda: b"null", + lambda: None, + ], + ids=["file-like", "invalid-json", "invalid-encoding", "array", "null", "none"], +) +def test_invoke_model_preserves_sdk_error_when_capture_is_unavailable( + invocation, body_factory +): + body = body_factory() + sdk_error = RuntimeError("fake SDK rejection") + invocation.sdk.side_effect = sdk_error + + try: + with pytest.raises(RuntimeError) as exc_info: + invocation.invoke(modelId=MODEL_ID, body=body) + + assert exc_info.value is sdk_error + invocation.sdk.assert_called_once_with(modelId=MODEL_ID, body=body) + finally: + if isinstance(body, io.BytesIO): + body.close() diff --git a/python/frameworks/bedrock/tests/test_framework_bedrock_streaming.py b/python/frameworks/bedrock/tests/test_framework_bedrock_streaming.py new file mode 100644 index 000000000..240ae182f --- /dev/null +++ b/python/frameworks/bedrock/tests/test_framework_bedrock_streaming.py @@ -0,0 +1,314 @@ +"""T4-PY-CLOUD-007: InvokeModel streams retain their spans until consumption. + +These tests use a fake botocore EventStream and an in-memory OTel exporter only. +""" + +import io +import json +from unittest.mock import Mock + +import pytest +from botocore.eventstream import EventStream +from opentelemetry import context as context_api +from opentelemetry.context import _SUPPRESS_INSTRUMENTATION_KEY +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +from opentelemetry.trace import StatusCode + +from traceai_bedrock._wrappers import _InvokeModelWithResponseStream + + +class FakeEventStream(EventStream): + def __init__(self, events, error=None, close_error=None): + # No HTTP response, parser, client, credentials, or network is required. + self.events = events + self.error = error + self.close_error = close_error + self.consumed = 0 + self.close_calls = 0 + + def __iter__(self): + for event in self.events: + self.consumed += 1 + yield event + if self.error is not None: + raise self.error + + def close(self): + self.close_calls += 1 + if self.close_error is not None: + raise self.close_error + return "closed" + + +def chunk(payload): + return {"chunk": {"bytes": json.dumps(payload).encode()}} + + +@pytest.fixture +def tracing(): + provider = TracerProvider() + exporter = InMemorySpanExporter() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + yield provider.get_tracer(__name__), exporter + provider.shutdown() + + +def invoke(tracing, stream, model="meta.llama3-8b-instruct-v1:0", body=None): + tracer, exporter = tracing + if body is None: + body = '{"prompt":"hello","max_gen_len":16}' + kwargs = {"modelId": model, "contentType": "application/json", "body": body} + response = {"body": stream, "contentType": "application/json"} + wrapped = Mock(return_value=response) + result = _InvokeModelWithResponseStream(tracer)(wrapped)(**kwargs) + wrapped.assert_called_once_with(**kwargs) + assert result is response + assert result["contentType"] == "application/json" + return result["body"], exporter + + +@pytest.mark.parametrize( + "model,payloads", + [ + ( + "meta.llama3-8b-instruct-v1:0", + [ + {"generation": "Hello", "prompt_token_count": 3, "generation_token_count": 1}, + {"generation": " world", "prompt_token_count": None, "generation_token_count": 2}, + {"generation": "", "stop_reason": "stop"}, + ], + ), + ( + "us.meta.llama3-2-3b-instruct-v1:0", + [ + {"generation": "Hello", "prompt_token_count": 3, "generation_token_count": 1}, + {"generation": " world", "generation_token_count": 2}, + ], + ), + ( + "amazon.titan-text-express-v1", + [ + {"outputText": "Hello", "inputTextTokenCount": 3, "totalOutputTextTokenCount": 1}, + {"outputText": " world", "totalOutputTextTokenCount": 2}, + ], + ), + ( + "cohere.command-text-v14", + [ + {"text": "Hello", "is_finished": False}, + {"text": " world", "is_finished": False}, + { + "is_finished": True, + "amazon-bedrock-invocationMetrics": {"inputTokenCount": 3, "outputTokenCount": 2}, + }, + ], + ), + ( + "mistral.mistral-7b-instruct-v0:2", + [ + {"outputs": [{"text": "Hello"}]}, + { + "outputs": [{"text": " world", "stop_reason": "stop"}], + "amazon-bedrock-invocationMetrics": {"inputTokenCount": 3, "outputTokenCount": 2}, + }, + ], + ), + ], + ids=["llama", "llama-profile", "titan", "cohere", "mistral"], +) +def test_non_anthropic_stream_defers_and_captures_output_and_usage(tracing, model, payloads): + events = [chunk(payload) for payload in payloads] + source = FakeEventStream(events) + stream, exporter = invoke(tracing, source, model) + assert source.consumed == 0 + assert exporter.get_finished_spans() == () + + iterator = iter(stream) + assert next(iterator) is events[0] + assert exporter.get_finished_spans() == () + remaining = list(iterator) + assert all(actual is expected for actual, expected in zip(remaining, events[1:])) + assert len(remaining) == len(events) - 1 + span, = exporter.get_finished_spans() + attrs = span.attributes + assert attrs["gen_ai.request.model"] == model + assert attrs["output.value"] == "Hello world" + assert attrs["gen_ai.usage.input_tokens"] == 3 + assert attrs["gen_ai.usage.output_tokens"] == 2 + assert attrs["gen_ai.usage.total_tokens"] == 5 + assert json.loads(attrs["input.value"]) == {"prompt": "hello", "max_gen_len": 16} + assert json.loads(attrs["gen_ai.request.parameters"])["max_gen_len"] == 16 + assert span.status.status_code == StatusCode.OK + # Both another exhaustion and a close must be safe after finalization. + with pytest.raises(StopIteration): + next(iterator) + assert stream.close() == "closed" + assert source.close_calls == 1 + assert len(exporter.get_finished_spans()) == 1 + + +def test_stream_error_is_reraised_and_records_partial_output(tracing): + error = RuntimeError("fake stream failure") + event = chunk({"generation": "partial", "prompt_token_count": 3}) + source = FakeEventStream([event], error=error) + stream, exporter = invoke(tracing, source) + assert exporter.get_finished_spans() == () + iterator = iter(stream) + assert next(iterator) is event + with pytest.raises(RuntimeError) as caught: + next(iterator) + assert caught.value is error + span, = exporter.get_finished_spans() + assert span.status.status_code == StatusCode.ERROR + assert span.attributes["output.value"] == "partial" + assert span.attributes["gen_ai.usage.input_tokens"] == 3 + assert span.events[0].attributes["exception.type"] == "RuntimeError" + stream.close() + assert len(exporter.get_finished_spans()) == 1 + + +@pytest.mark.parametrize("consume_first", [False, True], ids=["unread", "partial"]) +def test_explicit_close_finishes_without_draining(tracing, consume_first): + events = [chunk({"generation": "first"}), chunk({"generation": "second"})] + source = FakeEventStream(events) + stream, exporter = invoke(tracing, source) + assert exporter.get_finished_spans() == () + if consume_first: + assert next(iter(stream)) is events[0] + assert stream.close() == "closed" + assert source.consumed == int(consume_first) + assert source.close_calls == 1 + span, = exporter.get_finished_spans() + assert span.attributes["gen_ai.request.model"] == "meta.llama3-8b-instruct-v1:0" + if consume_first: + assert span.attributes["output.value"] == "first" + stream.close() + assert len(exporter.get_finished_spans()) == 1 + + +def test_close_error_is_preserved_and_recorded(tracing): + error = OSError("fake close failure") + stream, exporter = invoke(tracing, FakeEventStream([], close_error=error)) + assert exporter.get_finished_spans() == () + with pytest.raises(OSError) as caught: + stream.close() + assert caught.value is error + span, = exporter.get_finished_spans() + assert span.status.status_code == StatusCode.ERROR + assert span.events + + +def test_empty_stream_records_request_and_finishes(tracing): + stream, exporter = invoke(tracing, FakeEventStream([])) + assert exporter.get_finished_spans() == () + assert list(stream) == [] + span, = exporter.get_finished_spans() + assert span.attributes["gen_ai.request.model"] == "meta.llama3-8b-instruct-v1:0" + assert json.loads(span.attributes["input.value"])["prompt"] == "hello" + + +def test_zero_counts_and_terminal_metrics_are_not_summed(tracing): + source = FakeEventStream([ + chunk({"generation": "", "prompt_token_count": 0, "generation_token_count": 0}), + chunk({"amazon-bedrock-invocationMetrics": {"inputTokenCount": 0, "outputTokenCount": 0}}), + ]) + stream, exporter = invoke(tracing, source) + list(stream) + span, = exporter.get_finished_spans() + assert span.attributes["output.value"] == "" + for name in ("input_tokens", "output_tokens", "total_tokens"): + assert span.attributes[f"gen_ai.usage.{name}"] == 0 + + +def test_bad_chunks_do_not_change_stream_results(tracing): + events = [ + {"chunk": {"bytes": b"not JSON"}}, + chunk(["unexpected", "schema"]), + chunk({"generation": "valid"}), + ] + stream, exporter = invoke(tracing, FakeEventStream(events)) + actual = list(stream) + assert len(actual) == len(events) + assert all(left is right for left, right in zip(actual, events)) + span, = exporter.get_finished_spans() + assert span.attributes["output.value"] == "valid" + + +@pytest.mark.parametrize("body", [b'{"prompt":"hello"}', io.BytesIO(b'{"prompt":"hello"}')]) +def test_bytes_and_file_request_bodies_are_not_consumed_by_instrumentation(tracing, body): + stream, exporter = invoke(tracing, FakeEventStream([chunk({"generation": "ok"})]), body=body) + assert exporter.get_finished_spans() == () + if hasattr(body, "tell"): + assert body.tell() == 0 + list(stream) + span, = exporter.get_finished_spans() + assert span.attributes["output.value"] == "ok" + + +def test_invocation_error_ends_span_and_preserves_exception(tracing): + tracer, exporter = tracing + error = RuntimeError("fake invocation failure") + wrapped = Mock(side_effect=error) + with pytest.raises(RuntimeError) as caught: + _InvokeModelWithResponseStream(tracer)(wrapped)( + modelId="meta.llama3-8b-instruct-v1:0", body='{"prompt":"hello"}' + ) + assert caught.value is error + span, = exporter.get_finished_spans() + assert span.status.status_code == StatusCode.ERROR + assert span.attributes["gen_ai.request.model"] == "meta.llama3-8b-instruct-v1:0" + assert len(span.events) == 1 + + +def test_unknown_schema_output_is_serialized(tracing): + payload = {"result": {"content": [{"text": "hello"}]}} + stream, exporter = invoke(tracing, FakeEventStream([chunk(payload)]), model="custom-model") + assert exporter.get_finished_spans() == () + list(stream) + span, = exporter.get_finished_spans() + assert json.loads(span.attributes["output.value"]) == [payload] + assert span.attributes["output.mime_type"] == "application/json" + + +def test_existing_messages_stream_remains_deferred(tracing): + events = [ + chunk({ + "type": "message_start", + "message": { + "id": "fake-message", + "type": "message", + "role": "assistant", + "model": "fake-model", + "content": [], + "usage": {"input_tokens": 3, "output_tokens": 1}, + }, + }), + chunk({"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": "hello"}}), + chunk({"type": "message_stop"}), + ] + body = json.dumps({ + "anthropic_version": "bedrock-2023-05-31", + "max_tokens": 16, + "messages": [{"role": "user", "content": "hello"}], + }) + stream, exporter = invoke(tracing, FakeEventStream(events), model="anthropic.fake-model", body=body) + assert exporter.get_finished_spans() == () + assert list(stream) == events + span, = exporter.get_finished_spans() + output = json.loads(span.attributes["output.value"]) + assert output["content"][0]["text"] == "hello" + assert span.attributes["gen_ai.request.model"] == "anthropic.fake-model" + assert span.attributes["gen_ai.usage.input_tokens"] == 3 + + +def test_suppressed_instrumentation_keeps_original_stream(tracing): + source = FakeEventStream([]) + token = context_api.attach(context_api.set_value(_SUPPRESS_INSTRUMENTATION_KEY, True)) + try: + stream, exporter = invoke(tracing, source) + finally: + context_api.detach(token) + assert stream is source + assert exporter.get_finished_spans() == () diff --git a/python/frameworks/bedrock/traceai_bedrock/__init__.py b/python/frameworks/bedrock/traceai_bedrock/__init__.py index 0dcca08a9..ee49cf294 100644 --- a/python/frameworks/bedrock/traceai_bedrock/__init__.py +++ b/python/frameworks/bedrock/traceai_bedrock/__init__.py @@ -39,6 +39,7 @@ get_attributes_from_context, safe_json_dumps, ) +from traceai_bedrock._converse_stream import _ConverseStream from traceai_bedrock._rag_wrappers import ( _retrieve_and_generate_wrapper, _retrieve_wrapper, @@ -91,6 +92,9 @@ class InstrumentedClient(BaseClient): # type: ignore converse: Callable[..., Any] _unwrapped_converse: Callable[..., Any] + converse_stream: Callable[..., Any] + _unwrapped_converse_stream: Callable[..., Any] + invoke_agent: Callable[..., Any] _unwrapped_invoke_agent: Callable[..., Any] @@ -171,6 +175,11 @@ def _client_wrapper( if module_version >= _MINIMUM_CONVERSE_BOTOCORE_VERSION: client._unwrapped_converse = client.converse client.converse = _model_converse_wrapper(tracer)(client) + # ConverseStream is an independent operation, not a call to converse. + # Feature-detect it so older botocore service models remain supported. + if hasattr(client, "converse_stream"): + client._unwrapped_converse_stream = client.converse_stream + client.converse_stream = _ConverseStream(tracer)(client.converse_stream) return client return _client_wrapper # type: ignore @@ -186,18 +195,28 @@ def instrumented_response(*args: Any, **kwargs: Any) -> Dict[str, Any]: return wrapped_client._unwrapped_invoke_model(*args, **kwargs) # type: ignore with tracer.start_as_current_span("bedrock.invoke_model") as span: - request_body = json.loads(kwargs["body"]) + # Leave file-like bodies and their positions untouched for the SDK. + request_body = None + body = kwargs.get("body") + if isinstance(body, (str, bytes, bytearray)): + try: + request_body = json.loads(body) + except Exception: + # Optional capture must not replace SDK validation. + pass + model_id = str(kwargs.get("modelId")) # Determine if this is a Claude Messages API model is_claude_message_api = _extract_invoke_model_attributes.is_claude_message_api( model_id ) - # Set input attributes based on model type - if is_claude_message_api: - anthropic_attributes.set_input_attributes(span, request_body, model_id) - else: - _extract_invoke_model_attributes.set_input_attributes(span, request_body) + # Set input attributes only when the request can be captured. + if isinstance(request_body, dict): + if is_claude_message_api: + anthropic_attributes.set_input_attributes(span, request_body, model_id) + else: + _extract_invoke_model_attributes.set_input_attributes(span, request_body) # Execute the model invocation with proper error handling try: diff --git a/python/frameworks/bedrock/traceai_bedrock/_converse_stream.py b/python/frameworks/bedrock/traceai_bedrock/_converse_stream.py new file mode 100644 index 000000000..9a4beb46e --- /dev/null +++ b/python/frameworks/bedrock/traceai_bedrock/_converse_stream.py @@ -0,0 +1,217 @@ +"""Instrumentation for Bedrock's native ConverseStream event schema.""" + +from __future__ import annotations + +import logging +from typing import Any, Callable + +import wrapt +from opentelemetry import context as context_api +from opentelemetry.context import _SUPPRESS_INSTRUMENTATION_KEY +from opentelemetry.trace import Span, Status, StatusCode, Tracer + +from fi_instrumentation import get_attributes_from_context, safe_json_dumps +from fi_instrumentation.fi_types import ( + FiLLMProviderValues, + FiSpanKindValues, + MessageAttributes, + SpanAttributes, + ToolCallAttributes, +) +from traceai_bedrock.utils import _EventStream, _use_span + +logger = logging.getLogger(__name__) + + +class _ConverseStream: + def __init__(self, tracer: Tracer) -> None: + self._tracer = tracer + + @wrapt.decorator # type: ignore[misc] + def __call__( + self, + wrapped: Callable[..., Any], + instance: Any, + args: tuple[Any, ...], + kwargs: dict[str, Any], + ) -> Any: + if context_api.get_value(_SUPPRESS_INSTRUMENTATION_KEY): + return wrapped(*args, **kwargs) + with self._tracer.start_as_current_span( + "bedrock.converse_stream", + end_on_exit=False, + record_exception=False, + set_status_on_exception=False, + ) as span: + callback = _ConverseStreamCallback(span) + try: + _set_request_attributes(span, kwargs) + except Exception: + # Invalid requests must still reach botocore's own validation. + logger.warning("Could not extract ConverseStream request attributes") + try: + response = wrapped(*args, **kwargs) + except BaseException as exc: + callback.finish(error=exc) + raise + try: + if (stream := response.get("stream")) is not None: + response["stream"] = _ConverseEventStream(stream, callback, _use_span(span)) + else: + callback.finish(closed=True) + except Exception: + # Telemetry must not prevent the caller from receiving a response. + callback.finish(closed=True) + return response + + +class _ConverseEventStream(_EventStream): + def __iter__(self): + try: + return super().__iter__() + except BaseException as exc: + self._self_callback.finish(error=exc) + raise + + def close(self, *args: Any, **kwargs: Any) -> Any: + try: + with self._self_context_manager_factory(): + result = self.__wrapped__.close(*args, **kwargs) + except BaseException as exc: + self._self_callback.finish(error=exc) + raise + self._self_callback.finish(closed=True) + return result + + +class _ConverseStreamCallback: + def __init__(self, span: Span) -> None: + self._span = span + self._role: str | None = None + self._blocks: dict[int, dict[str, Any]] = {} + self._finished = False + + def __call__(self, event: Any) -> Any: + if self._finished: + return event + if isinstance(event, StopIteration): + self.finish() + elif isinstance(event, BaseException): + self.finish(error=event) + elif isinstance(event, dict): + # _EventStream isolates callback errors and returns the original event. + if start := event.get("messageStart"): + self._role = start.get("role") + if start := event.get("contentBlockStart"): + if tool := start.get("start", {}).get("toolUse"): + block = self._blocks.setdefault(start["contentBlockIndex"], {}) + block["toolUse"] = dict(tool) + if delta := event.get("contentBlockDelta"): + block = self._blocks.setdefault(delta["contentBlockIndex"], {}) + content = delta.get("delta", {}) + if isinstance(text := content.get("text"), str): + block["text"] = block.get("text", "") + text + if isinstance(fragment := content.get("toolUse", {}).get("input"), str): + tool = block.setdefault("toolUse", {}) + tool["input"] = tool.get("input", "") + fragment + if stop := event.get("messageStop"): + if reason := stop.get("stopReason"): + self._span.set_attribute(SpanAttributes.GEN_AI_RESPONSE_FINISH_REASONS, [reason]) + # Do not end here: metadata (including usage) follows messageStop. + if metadata := event.get("metadata"): + usage = metadata.get("usage", {}) + for field, attribute in ( + ("inputTokens", SpanAttributes.GEN_AI_USAGE_INPUT_TOKENS), + ("outputTokens", SpanAttributes.GEN_AI_USAGE_OUTPUT_TOKENS), + ("totalTokens", SpanAttributes.GEN_AI_USAGE_TOTAL_TOKENS), + ("cacheReadInputTokens", SpanAttributes.GEN_AI_USAGE_CACHE_READ_TOKENS), + ("cacheWriteInputTokens", SpanAttributes.GEN_AI_USAGE_CACHE_WRITE_TOKENS), + ): + if isinstance(count := usage.get(field), int): + self._span.set_attribute(attribute, count) + # Botocore normally raises EventStreamError; also handle error union + # members without changing what a custom stream yields to its caller. + for name in event: + if name.endswith("Exception"): + self.finish(error=RuntimeError(f"Bedrock ConverseStream {name}")) + break + return event + + def finish(self, error: BaseException | None = None, closed: bool = False) -> None: + if self._finished: + return + self._finished = True + try: + blocks = [block for _, block in sorted(self._blocks.items())] + message = {"role": self._role, "content": blocks} + prefix = f"{SpanAttributes.GEN_AI_OUTPUT_MESSAGES}.0" + _set_message_attributes(self._span, prefix, message) + texts = [block["text"] for block in blocks if "text" in block] + if texts: + text = "\n".join(texts) + self._span.set_attribute(SpanAttributes.OUTPUT_VALUE, text) + self._span.set_attribute(f"{prefix}.{MessageAttributes.MESSAGE_CONTENT}", text) + except Exception: + logger.warning("Could not extract ConverseStream output attributes") + try: + if error is not None: + self._span.record_exception(error) + self._span.set_status(Status(StatusCode.ERROR, type(error).__name__)) + elif not closed: + self._span.set_status(Status(StatusCode.OK)) + # An explicit close before exhaustion is not evidence of success. + except Exception: + logger.warning("Could not set ConverseStream terminal status") + finally: + try: + self._span.end() + except Exception: + logger.warning("Could not end ConverseStream span") + + +def _set_message_attributes(span: Span, prefix: str, message: dict[str, Any]) -> None: + # Reuse the Converse text/image schema. Import lazily because __init__ installs + # this wrapper before defining its existing message helpers. + from traceai_bedrock import _get_attributes_from_message_param + + for key, value in _get_attributes_from_message_param(message): + span.set_attribute(f"{prefix}.{key}", value) + tool_index = 0 + for block in message.get("content", []): + if not isinstance(block, dict) or not isinstance(tool := block.get("toolUse"), dict): + continue + tool_prefix = f"{prefix}.{MessageAttributes.MESSAGE_TOOL_CALLS}.{tool_index}" + tool_index += 1 + for field, attribute in ( + ("toolUseId", ToolCallAttributes.TOOL_CALL_ID), + ("name", ToolCallAttributes.TOOL_CALL_FUNCTION_NAME), + ("input", ToolCallAttributes.TOOL_CALL_FUNCTION_ARGUMENTS_JSON), + ): + if (value := tool.get(field)) is not None: + if field == "input" and not isinstance(value, str): + value = safe_json_dumps(value) + span.set_attribute(f"{tool_prefix}.{attribute}", value) + + +def _set_request_attributes(span: Span, request: dict[str, Any]) -> None: + span.set_attribute(SpanAttributes.GEN_AI_SPAN_KIND, FiSpanKindValues.LLM.value) + span.set_attribute(SpanAttributes.GEN_AI_PROVIDER_NAME, FiLLMProviderValues.AWS.value) + span.set_attributes(dict(get_attributes_from_context())) + if model := request.get("modelId"): + span.set_attribute(SpanAttributes.GEN_AI_REQUEST_MODEL, model) + if config := request.get("inferenceConfig"): + span.set_attribute(SpanAttributes.GEN_AI_REQUEST_PARAMETERS, safe_json_dumps(config)) + messages = [] + if system := request.get("system"): + messages.append({"role": "system", "content": system}) + messages.extend(request.get("messages", [])) + for index, message in enumerate(messages): + if isinstance(message, dict): + _set_message_attributes(span, f"{SpanAttributes.GEN_AI_INPUT_MESSAGES}.{index}", message) + if messages and isinstance(messages[-1], dict): + text = "\n".join( + block.get("text", "") for block in messages[-1].get("content", []) + if isinstance(block, dict) + ).strip("\n") + if text: + span.set_attribute(SpanAttributes.INPUT_VALUE, text) diff --git a/python/frameworks/bedrock/traceai_bedrock/_wrappers.py b/python/frameworks/bedrock/traceai_bedrock/_wrappers.py index 6a0452bf9..2ced9c138 100644 --- a/python/frameworks/bedrock/traceai_bedrock/_wrappers.py +++ b/python/frameworks/bedrock/traceai_bedrock/_wrappers.py @@ -12,6 +12,10 @@ from traceai_bedrock._rag_wrappers import _RagEventStream from traceai_bedrock._response_accumulator import _ResponseAccumulator from traceai_bedrock.utils import _EventStream, _use_span +from traceai_bedrock.utils._invoke_model_stream import ( + _InvokeModelEventStream, + _InvokeModelStreamCallback, +) from traceai_bedrock.utils.anthropic._messages import ( _AnthropicMessagesCallback, ) @@ -47,24 +51,37 @@ def __call__( with self._tracer.start_as_current_span( self._name, end_on_exit=False, + record_exception=False, + set_status_on_exception=False, ) as span: - response = wrapped(*args, **kwargs) from botocore.eventstream import EventStream - kwargs["body"] = json.loads(kwargs["body"]) + # Do not mutate the caller's kwargs or read a file-like request body. + # Parsing for telemetry must not prevent a valid provider invocation. + body = None + if isinstance(raw_body := kwargs.get("body"), (str, bytes, bytearray)): + try: + body = json.loads(raw_body) + except (TypeError, ValueError): + pass + request = {**kwargs, "body": body} + callback: Callable[[Any], Any] + stream_type: type[_EventStream] + if isinstance(body, dict) and "anthropic_version" in body: + callback = _AnthropicMessagesCallback(span, request) + stream_type = _EventStream + else: + callback = _InvokeModelStreamCallback(span, request) + stream_type = _InvokeModelEventStream + try: + response = wrapped(*args, **kwargs) + except BaseException as exc: + callback(exc) + raise if isinstance(response["body"], EventStream): - if "anthropic_version" in kwargs["body"]: - response["body"] = _EventStream( - response["body"], - _AnthropicMessagesCallback(span, kwargs), - _use_span(span), - ) - return response - span.set_attribute(GEN_AI_REQUEST_PARAMETERS, kwargs["body"]) - span.set_attribute(INPUT_MIME_TYPE, JSON) - span.set_attribute(INPUT_VALUE, kwargs["body"]) - span.set_attribute(GEN_AI_SPAN_KIND, LLM) - span.end() + response["body"] = stream_type(response["body"], callback, _use_span(span)) + else: + callback(StopIteration()) return response diff --git a/python/frameworks/bedrock/traceai_bedrock/utils/_invoke_model_stream.py b/python/frameworks/bedrock/traceai_bedrock/utils/_invoke_model_stream.py new file mode 100644 index 000000000..1bdb1ba8f --- /dev/null +++ b/python/frameworks/bedrock/traceai_bedrock/utils/_invoke_model_stream.py @@ -0,0 +1,115 @@ +"""Deferred tracing for Bedrock InvokeModel streams outside the Messages API.""" + +from __future__ import annotations + +import json +from typing import Any, Mapping, TypeVar + +from opentelemetry.trace import Span +from opentelemetry.util.types import AttributeValue + +from fi_instrumentation import safe_json_dumps +from fi_instrumentation.fi_types import ( + FiLLMProviderValues, + FiMimeTypeValues, + FiSpanKindValues, + SpanAttributes, +) +from traceai_bedrock._proxy import _no_err +from traceai_bedrock.utils import _EventStream, _finish + +_AnyT = TypeVar("_AnyT") + + +class _InvokeModelEventStream(_EventStream): + def close(self) -> Any: + # Closing is not iteration: explicitly finish without draining the provider. + callback = _no_err(self._self_callback) + try: + result = self.__wrapped__.close() + except BaseException as exc: + callback(exc) + raise + callback(StopIteration()) + return result + + +class _InvokeModelStreamCallback: + def __init__(self, span: Span, request: Mapping[str, Any]) -> None: + self._span = span + self._finished = False + self._text: list[str] = [] + self._payloads: list[dict[str, Any]] = [] + self._attributes: dict[str, AttributeValue] = { + SpanAttributes.GEN_AI_SPAN_KIND: FiSpanKindValues.LLM.value, + SpanAttributes.GEN_AI_PROVIDER_NAME: FiLLMProviderValues.AWS.value, + } + if isinstance(model := request.get("modelId"), str): + self._attributes[SpanAttributes.GEN_AI_REQUEST_MODEL] = model + if (body := request.get("body")) is not None: + serialized = safe_json_dumps(body) + self._attributes[SpanAttributes.INPUT_VALUE] = serialized + self._attributes[SpanAttributes.INPUT_MIME_TYPE] = FiMimeTypeValues.JSON.value + self._attributes[SpanAttributes.GEN_AI_REQUEST_PARAMETERS] = serialized + + def __call__(self, obj: _AnyT) -> _AnyT: + if self._finished: + return obj + if isinstance(obj, dict): + chunk = obj.get("chunk") + if not isinstance(chunk, dict): + return obj + try: + payload = json.loads(chunk.get("bytes")) + except (TypeError, ValueError): + return obj + if isinstance(payload, dict): + self._accumulate(payload) + elif isinstance(obj, BaseException): + self._finished = True + if self._text: + self._attributes[SpanAttributes.OUTPUT_VALUE] = "".join(self._text) + self._attributes[SpanAttributes.OUTPUT_MIME_TYPE] = FiMimeTypeValues.TEXT.value + elif self._payloads: + self._attributes[SpanAttributes.OUTPUT_VALUE] = safe_json_dumps(self._payloads) + self._attributes[SpanAttributes.OUTPUT_MIME_TYPE] = FiMimeTypeValues.JSON.value + input_tokens = self._attributes.get(SpanAttributes.GEN_AI_USAGE_INPUT_TOKENS) + output_tokens = self._attributes.get(SpanAttributes.GEN_AI_USAGE_OUTPUT_TOKENS) + if isinstance(input_tokens, int) and isinstance(output_tokens, int): + self._attributes[SpanAttributes.GEN_AI_USAGE_TOTAL_TOKENS] = input_tokens + output_tokens + error = None if isinstance(obj, (StopIteration, StopAsyncIteration)) else obj + _finish(self._span, error, self._attributes) + return obj + + def _accumulate(self, payload: dict[str, Any]) -> None: + # Match response schemas rather than model-id prefixes: inference-profile + # IDs and ARNs also carry the same Meta, Titan, Cohere and Mistral chunks. + for key in ("generation", "outputText", "completion", "text"): + if isinstance(text := payload.get(key), str): + self._text.append(text) + break + else: + outputs = payload.get("outputs") + if isinstance(outputs, list): + self._text.extend( + output["text"] for output in outputs + if isinstance(output, dict) and isinstance(output.get("text"), str) + ) + else: + # Preserve structured output for schemas without a text adapter. + self._payloads.append(payload) + + # Native counts and invocation metrics are running/final totals, not + # per-chunk deltas. Keep the latest non-null value, including zero. + for key in ("prompt_token_count", "inputTextTokenCount"): + self._set_count(SpanAttributes.GEN_AI_USAGE_INPUT_TOKENS, payload.get(key)) + for key in ("generation_token_count", "totalOutputTextTokenCount"): + self._set_count(SpanAttributes.GEN_AI_USAGE_OUTPUT_TOKENS, payload.get(key)) + metrics = payload.get("amazon-bedrock-invocationMetrics") + if isinstance(metrics, dict): + self._set_count(SpanAttributes.GEN_AI_USAGE_INPUT_TOKENS, metrics.get("inputTokenCount")) + self._set_count(SpanAttributes.GEN_AI_USAGE_OUTPUT_TOKENS, metrics.get("outputTokenCount")) + + def _set_count(self, name: str, value: Any) -> None: + if isinstance(value, int) and not isinstance(value, bool) and value >= 0: + self._attributes[name] = value diff --git a/python/frameworks/bedrock/traceai_bedrock/version.py b/python/frameworks/bedrock/traceai_bedrock/version.py index 8c39d1009..d22c42508 100644 --- a/python/frameworks/bedrock/traceai_bedrock/version.py +++ b/python/frameworks/bedrock/traceai_bedrock/version.py @@ -1 +1 @@ -__version__ = "0.1.11" # x-release-please-version +__version__ = "0.1.12" # x-release-please-version diff --git a/python/frameworks/beeai/tests/test_real_core_names.py b/python/frameworks/beeai/tests/test_real_core_names.py new file mode 100644 index 000000000..71096f345 --- /dev/null +++ b/python/frameworks/beeai/tests/test_real_core_names.py @@ -0,0 +1,45 @@ +"""Check the core names this package uses against the real installed core. + +tests/conftest.py replaces fi_instrumentation with mock modules, so the other tests here +cannot notice a core name that the declared fi-instrumentation-otel floor lacks. This test +reads the package source in a clean interpreter that imports the real core. +""" + +import pathlib +import subprocess +import sys + +PACKAGE = pathlib.Path(__file__).resolve().parents[1] / "traceai_beeai" + +CHECK = r""" +import ast, importlib, pathlib, sys, types +missing = [] +for path in sorted(pathlib.Path(sys.argv[1]).rglob("*.py")): + tree = ast.parse(path.read_text(encoding="utf-8")) + names = {} + for node in ast.walk(tree): + if isinstance(node, ast.ImportFrom) and node.module and node.module.split(".")[0] == "fi_instrumentation": + module = importlib.import_module(node.module) + for alias in node.names: + if hasattr(module, alias.name): + names[alias.asname or alias.name] = getattr(module, alias.name) + else: + missing.append(f"{path.name}: {node.module}.{alias.name}") + for node in ast.walk(tree): + if isinstance(node, ast.Attribute) and isinstance(node.value, ast.Name) and node.value.id in names: + owner = names[node.value.id] + if isinstance(owner, (type, types.ModuleType)) and not hasattr(owner, node.attr): + missing.append(f"{path.name}: {node.value.id}.{node.attr}") +print("\n".join(sorted(set(missing)))) +sys.exit(1 if missing else 0) +""" + + +def test_every_core_name_the_package_uses_exists_in_the_installed_core(): + result = subprocess.run( + [sys.executable, "-I", "-c", CHECK, str(PACKAGE)], + capture_output=True, + text=True, + check=False, + ) + assert result.returncode == 0, "missing in the installed core:\n" + result.stdout + result.stderr diff --git a/python/frameworks/cerebras/CHANGELOG.md b/python/frameworks/cerebras/CHANGELOG.md index fa441c5eb..fda595bc8 100644 --- a/python/frameworks/cerebras/CHANGELOG.md +++ b/python/frameworks/cerebras/CHANGELOG.md @@ -1,5 +1,10 @@ # Changelog +## [0.1.2] - pending release + +### Changed +- traceai-cerebras now requires fi-instrumentation-otel 1.0.0 or later. Action: Upgrade fi-instrumentation-otel if your environment pins an earlier version. + ## [0.1.1] - 2026-10-29 ### Fixed diff --git a/python/frameworks/cerebras/pyproject.toml b/python/frameworks/cerebras/pyproject.toml index 0b34fb625..6651f6837 100644 --- a/python/frameworks/cerebras/pyproject.toml +++ b/python/frameworks/cerebras/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "traceAI-cerebras" -version = "0.1.1" +version = "0.1.2" description = "OpenTelemetry instrumentation for Cerebras Cloud SDK" authors = ["Future AGI "] readme = "README.md" @@ -27,7 +27,7 @@ classifiers = [ [tool.poetry.dependencies] python = ">=3.10,<3.14" -fi-instrumentation-otel = ">=0.1.11" +fi-instrumentation-otel = ">=1.0.0" cerebras-cloud-sdk = ">=1.0.0" [tool.poetry.group.dev.dependencies] diff --git a/python/frameworks/cerebras/traceai_cerebras/version.py b/python/frameworks/cerebras/traceai_cerebras/version.py index 34d76ee9e..0d33eef3f 100644 --- a/python/frameworks/cerebras/traceai_cerebras/version.py +++ b/python/frameworks/cerebras/traceai_cerebras/version.py @@ -1 +1 @@ -__version__ = "0.1.1" # x-release-please-version +__version__ = "0.1.2" # x-release-please-version diff --git a/python/frameworks/claude-agent-sdk/CHANGELOG.md b/python/frameworks/claude-agent-sdk/CHANGELOG.md index 46fbf9388..a0273824a 100644 --- a/python/frameworks/claude-agent-sdk/CHANGELOG.md +++ b/python/frameworks/claude-agent-sdk/CHANGELOG.md @@ -1,6 +1,10 @@ # Changelog -## 0.1.1 +## [0.1.2] - pending release + +### Changed +- traceai-claude-agent-sdk now requires fi-instrumentation-otel 0.1.11 or later. Action: Upgrade fi-instrumentation-otel if your environment pins an earlier version. ### Fixed - Subagent tool calls nest under the call that launched them again on current Claude Agent SDK releases: the launch tool `Agent` is recognised alongside `Task`, and tool calls are attributed by `agent_id` through the `SubagentStart` hook, with `parent_tool_use_id` still honoured for older releases. + diff --git a/python/frameworks/claude-agent-sdk/pyproject.toml b/python/frameworks/claude-agent-sdk/pyproject.toml index 723838784..9bc45d4f2 100644 --- a/python/frameworks/claude-agent-sdk/pyproject.toml +++ b/python/frameworks/claude-agent-sdk/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "traceai-claude-agent-sdk" -version = "0.1.1" +version = "0.1.2" description = "OpenTelemetry instrumentation for Claude Agent SDK" authors = ["FutureAGI "] license = "Apache-2.0" @@ -35,7 +35,7 @@ python = ">=3.10,<3.14" opentelemetry-api = ">=1.0.0" opentelemetry-sdk = ">=1.0.0" opentelemetry-instrumentation = ">=0.40b0" -fi-instrumentation-otel = ">=0.1.0" +fi-instrumentation-otel = ">=0.1.11" [tool.poetry.group.dev.dependencies] pytest = "^8.0.0" diff --git a/python/frameworks/claude-agent-sdk/traceai_claude_agent_sdk/__init__.py b/python/frameworks/claude-agent-sdk/traceai_claude_agent_sdk/__init__.py index 26ffedf24..e7fdf9330 100644 --- a/python/frameworks/claude-agent-sdk/traceai_claude_agent_sdk/__init__.py +++ b/python/frameworks/claude-agent-sdk/traceai_claude_agent_sdk/__init__.py @@ -36,7 +36,7 @@ async def main(): from traceai_claude_agent_sdk._session_tracker import SessionTracker from traceai_claude_agent_sdk._mcp_tracker import MCPTracker -__version__ = "0.1.1" # x-release-please-version +__version__ = "0.1.2" # x-release-please-version __all__ = [ # Main API diff --git a/python/frameworks/cohere/CHANGELOG.md b/python/frameworks/cohere/CHANGELOG.md index fa441c5eb..4fe843ced 100644 --- a/python/frameworks/cohere/CHANGELOG.md +++ b/python/frameworks/cohere/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +## [0.1.2] - pending release + +### Changed +- traceai-cohere now requires fi-instrumentation-otel 1.0.0 or later. Action: Upgrade fi-instrumentation-otel if your environment pins an earlier version. + +### Fixed +- AsyncClient chat streams are traced; chat calls are also traced when the history holds SDK message objects or when history, tools, documents or preamble is set to None; and typed embedding responses other than base64 record a preview of the first vector. + ## [0.1.1] - 2026-10-29 ### Fixed diff --git a/python/frameworks/cohere/pyproject.toml b/python/frameworks/cohere/pyproject.toml index 5e857a798..2ed01545e 100644 --- a/python/frameworks/cohere/pyproject.toml +++ b/python/frameworks/cohere/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "traceAI-cohere" -version = "0.1.1" +version = "0.1.2" description = "OpenTelemetry instrumentation for Cohere - embeddings, chat, and rerank" authors = ["Future AGI "] readme = "README.md" @@ -27,7 +27,7 @@ classifiers = [ [tool.poetry.dependencies] python = ">=3.10,<3.14" -fi-instrumentation-otel = ">=0.1.11" +fi-instrumentation-otel = ">=1.0.0" cohere = ">=5.0.0" [tool.poetry.urls] diff --git a/python/frameworks/cohere/tests/test_stream_and_embeddings_sdk.py b/python/frameworks/cohere/tests/test_stream_and_embeddings_sdk.py new file mode 100644 index 000000000..621deea7e --- /dev/null +++ b/python/frameworks/cohere/tests/test_stream_and_embeddings_sdk.py @@ -0,0 +1,320 @@ +"""Public SDK contracts with real models and only the HTTP transport replaced.""" + +import asyncio +import json +from contextlib import contextmanager + +import cohere +import httpx +import pytest +from fi_instrumentation import TraceConfig +from fi_instrumentation.fi_types import EmbeddingAttributes, SpanAttributes +from fi_instrumentation.instrumentation.config import REDACTED_VALUE, suppress_tracing +from opentelemetry import trace +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter + +from traceai_cohere import CohereInstrumentor + + +PROMPT = "private synthetic input" +REPLY = "private synthetic output" +KEY = "synthetic-not-a-credential" +FINAL = { + "text": REPLY, + "generation_id": "synthetic-generation", + "finish_reason": "COMPLETE", + "meta": {"billed_units": {"input_tokens": 2, "output_tokens": 3}}, + "tool_calls": [{"name": "synthetic_tool", "parameters": {"text": REPLY}}], +} +EVENTS = [ + {"event_type": "text-generation", "text": REPLY, "is_finished": False}, + {"event_type": "stream-end", "is_finished": True, + "finish_reason": "COMPLETE", "response": FINAL}, +] +VECTOR_KEY = ( + f"{SpanAttributes.EMBEDDING_EMBEDDINGS}.0.{EmbeddingAttributes.EMBEDDING_VECTOR}" +) + + +def encode(event): + return (json.dumps(event) + "\n").encode() + + +@contextmanager +def capture(config=None, enabled=True): + exporter = InMemorySpanExporter() + provider = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + instrumentor = CohereInstrumentor() + if enabled: + instrumentor.instrument(tracer_provider=provider, config=config) + try: + yield exporter, provider + finally: + if enabled: + instrumentor.uninstrument() + provider.shutdown() + + +class EventBody(httpx.AsyncByteStream): + def __init__(self, mode="complete"): + self.mode = mode + self.closed = False + self.waiting = asyncio.Event() + self.error = httpx.ReadError("synthetic stream failure") + + async def __aiter__(self): + if self.mode == "empty": + return + yield encode(EVENTS[0]) + if self.mode == "error": + raise self.error + if self.mode == "cancel": + self.waiting.set() + await asyncio.Event().wait() + yield encode(EVENTS[1]) + + async def aclose(self): + self.closed = True + + +async def consume_async_stream(mode, enabled, config=None): + body = EventBody(mode) + requests = [] + + def respond(request): + requests.append(json.loads(request.content)) + return httpx.Response( + 200, headers={"content-type": "application/x-ndjson"}, stream=body + ) + + with capture(config, enabled) as (exporter, provider): + async with httpx.AsyncClient(transport=httpx.MockTransport(respond)) as http: + client = cohere.AsyncClient(api_key=KEY, httpx_client=http) + with provider.get_tracer(__name__).start_as_current_span("parent") as parent: + stream = client.chat_stream(model="command-r", message=PROMPT) + assert hasattr(stream, "__aiter__") + assert requests == [] # The public SDK does not read ahead. + if mode == "empty": + assert [event async for event in stream] == [] + else: + first = await stream.__anext__() + assert first.event_type == "text-generation" + assert first.text == REPLY + assert trace.get_current_span() is parent + if mode == "close": + await stream.aclose() + elif mode == "error": + with pytest.raises(httpx.ReadError) as caught: + await stream.__anext__() + assert caught.value is body.error + elif mode == "cancel": + task = asyncio.create_task(stream.__anext__()) + await asyncio.wait_for(body.waiting.wait(), timeout=2) + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + else: + remaining = [event async for event in stream] + assert len(remaining) == 1 + assert remaining[0].event_type == "stream-end" + assert remaining[0].response.text == REPLY + await stream.aclose() # Repeated close must not export twice. + assert body.closed + assert trace.get_current_span() is parent + parent_id = parent.get_span_context().span_id + provider.force_flush() + assert len(requests) == 1 + assert requests[0]["message"] == PROMPT + spans = [s for s in exporter.get_finished_spans() if s.name != "parent"] + return spans, parent_id + + +@pytest.mark.asyncio +@pytest.mark.parametrize("mode", ["complete", "empty", "close", "error", "cancel"]) +async def test_async_stream_regression(mode): + spans, parent_id = await consume_async_stream(mode, enabled=True) + assert len(spans) == 1, "AsyncClient.chat_stream must export one span" + span = spans[0] + assert span.name == "cohere.chat_stream" + assert span.parent.span_id == parent_id + assert span.attributes[SpanAttributes.GEN_AI_REQUEST_MODEL] == "command-r" + if mode in ("complete", "empty"): + assert span.status.status_code == trace.StatusCode.OK + if mode == "complete": + assert span.attributes[SpanAttributes.GEN_AI_USAGE_INPUT_TOKENS] == 2 + assert span.attributes[SpanAttributes.GEN_AI_USAGE_OUTPUT_TOKENS] == 3 + assert json.loads(span.attributes[SpanAttributes.OUTPUT_VALUE])["text"] == REPLY + if mode == "error": + assert span.status.status_code == trace.StatusCode.ERROR + assert len(span.events) == 1 + assert KEY not in json.dumps(dict(span.attributes)) + + +@pytest.mark.asyncio +async def test_async_stream_privacy_regression(): + spans, _ = await consume_async_stream( + "complete", enabled=True, config=TraceConfig(hide_inputs=True, hide_outputs=True) + ) + assert len(spans) == 1 + attributes = dict(spans[0].attributes) + assert attributes[SpanAttributes.INPUT_VALUE] == REDACTED_VALUE + assert attributes[SpanAttributes.OUTPUT_VALUE] == REDACTED_VALUE + assert not any(k.startswith(SpanAttributes.GEN_AI_INPUT_MESSAGES) for k in attributes) + assert not any(k.startswith(SpanAttributes.GEN_AI_OUTPUT_MESSAGES) for k in attributes) + for value in (PROMPT, REPLY, KEY): + assert value not in json.dumps(attributes) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("mode", ["complete", "empty", "close", "error", "cancel"]) +async def test_uninstrumented_async_stream_control(mode): + spans, _ = await consume_async_stream(mode, enabled=False) + assert spans == [] + + +@pytest.mark.asyncio +async def test_suppressed_async_stream_control(): + with suppress_tracing(): + spans, _ = await consume_async_stream("complete", enabled=True) + assert spans == [] + + +@pytest.mark.asyncio +async def test_async_stream_instrument_uninstrument_regression(): + original = cohere.AsyncClient.chat_stream + for _ in range(2): + spans, _ = await consume_async_stream("complete", enabled=True) + assert cohere.AsyncClient.chat_stream is original + assert len(spans) == 1 + spans, _ = await consume_async_stream("complete", enabled=False) + assert spans == [] + + +@pytest.mark.asyncio +async def test_existing_chat_paths_control(): + requests = [] + + def respond(request): + request_body = json.loads(request.content) + requests.append(request_body) + if request_body.get("stream"): + return httpx.Response( + 200, headers={"content-type": "application/x-ndjson"}, + content=b"".join(encode(event) for event in EVENTS), + ) + return httpx.Response(200, json=FINAL) + + with capture() as (exporter, provider): + with httpx.Client(transport=httpx.MockTransport(respond)) as http: + client = cohere.Client(api_key=KEY, httpx_client=http) + events = list(client.chat_stream(model="command-r", message=PROMPT)) + assert len(events) == 2 + assert events[0].text == REPLY + async with httpx.AsyncClient(transport=httpx.MockTransport(respond)) as http: + client = cohere.AsyncClient(api_key=KEY, httpx_client=http) + response = await client.chat(model="command-r", message=PROMPT) + assert response.text == REPLY + provider.force_flush() + assert [s.name for s in exporter.get_finished_spans()] == [ + "cohere.chat_stream", "cohere.chat" + ] + assert len(requests) == 2 + + +async def embed_call(asynchronous, typed, vectors, enabled=True, config=None): + requests = [] + + def respond(request): + requests.append(json.loads(request.content)) + return httpx.Response(200, json={ + "id": "synthetic-embedding", + "texts": [PROMPT] * len(vectors), + "response_type": "embeddings_by_type" if typed else "embeddings_floats", + "embeddings": {"float": vectors} if typed else vectors, + "meta": {"billed_units": {"input_tokens": 2}}, + }) + + kwargs = dict(texts=[PROMPT] * max(1, len(vectors)), model="embed-english-v3.0", + input_type="search_document") + if typed: + kwargs["embedding_types"] = ["float"] + with capture(config, enabled) as (exporter, provider): + if asynchronous: + async with httpx.AsyncClient(transport=httpx.MockTransport(respond)) as http: + client = cohere.AsyncClient(api_key=KEY, httpx_client=http) + response = await client.embed(**kwargs) + else: + with httpx.Client(transport=httpx.MockTransport(respond)) as http: + client = cohere.Client(api_key=KEY, httpx_client=http) + response = client.embed(**kwargs) + data = response.model_dump() if hasattr(response, "model_dump") else response.dict() + actual = data["embeddings"] + if typed: + actual = actual.get("float", actual.get("float_")) + assert actual == vectors + assert len(requests) == 1 + assert requests[0]["texts"] == kwargs["texts"] + provider.force_flush() + return exporter.get_finished_spans() + + +def assert_embeddings(spans, vectors, caplog, hidden=False): + assert len(spans) == 1 + assert not [r for r in caplog.records if r.name.startswith("traceai_cohere") and r.exc_info] + span = spans[0] + assert span.name == "cohere.embed" + attributes = dict(span.attributes) + assert span.status.status_code == trace.StatusCode.OK + assert attributes[SpanAttributes.GEN_AI_USAGE_INPUT_TOKENS] == 2 + if vectors: + assert attributes["cohere.embeddings_count"] == len(vectors) + assert attributes["cohere.embedding_dimensions"] == len(vectors[0]) + if hidden: + assert VECTOR_KEY not in attributes + assert attributes[SpanAttributes.INPUT_VALUE] == REDACTED_VALUE + assert attributes[SpanAttributes.OUTPUT_VALUE] == REDACTED_VALUE + assert PROMPT not in json.dumps(attributes) + else: + if vectors: + assert json.loads(attributes[VECTOR_KEY]) == vectors[0][:10] + else: + assert VECTOR_KEY not in attributes + assert json.loads(attributes[SpanAttributes.OUTPUT_VALUE]) == {"embeddings_count": len(vectors)} + assert KEY not in json.dumps(attributes) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("asynchronous", [False, True]) +@pytest.mark.parametrize("vectors", [[], [[0.25, 0.5, 0.75]], [[0.25, 0.5, 0.75]] * 2]) +async def test_typed_embeddings_regression(asynchronous, vectors, caplog): + spans = await embed_call(asynchronous, True, vectors) + assert_embeddings(spans, vectors, caplog) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("asynchronous", [False, True]) +async def test_typed_embeddings_privacy_regression(asynchronous, caplog): + vectors = [[0.25, 0.5, 0.75]] + spans = await embed_call(asynchronous, True, vectors, config=TraceConfig( + hide_inputs=True, hide_outputs=True, hide_embedding_vectors=True + )) + assert_embeddings(spans, vectors, caplog, hidden=True) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("asynchronous", [False, True]) +@pytest.mark.parametrize("hidden", [False, True]) +async def test_legacy_embeddings_control(asynchronous, hidden, caplog): + vectors = [[0.25, 0.5, 0.75]] + config = TraceConfig(hide_inputs=hidden, hide_outputs=hidden, hide_embedding_vectors=hidden) + spans = await embed_call(asynchronous, False, vectors, config=config) + assert_embeddings(spans, vectors, caplog, hidden) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("asynchronous", [False, True]) +async def test_uninstrumented_typed_embeddings_control(asynchronous): + assert not await embed_call(asynchronous, True, [[0.25, 0.5, 0.75]], enabled=False) diff --git a/python/frameworks/cohere/tests/test_stream_request_shapes_sdk.py b/python/frameworks/cohere/tests/test_stream_request_shapes_sdk.py new file mode 100644 index 000000000..77262298e --- /dev/null +++ b/python/frameworks/cohere/tests/test_stream_request_shapes_sdk.py @@ -0,0 +1,186 @@ +"""Stream request shapes through real SDK clients with HTTP-only stubs.""" + +import json + +import cohere +import httpx +import pytest +from fi_instrumentation import TraceConfig +from fi_instrumentation.fi_types import MessageAttributes, SpanAttributes +from fi_instrumentation.instrumentation.config import REDACTED_VALUE +from opentelemetry import trace + +from .test_stream_and_embeddings_sdk import EVENTS, KEY, PROMPT, REPLY, capture, encode + + +HISTORY = "synthetic private history" +PREAMBLE = "synthetic private instructions" +REGRESSION_SHAPES = [ + "sdk_history", "none_history", "none_tools", "none_documents", + "all_none", "mixed_history", +] +CONTROL_SHAPES = ["simple", "dict_history"] + + +def request_arguments(shape): + kwargs = {"model": "command-r", "message": PROMPT} + history = {"role": "USER", "message": HISTORY} + if shape in ("sdk_history", "mixed_history"): + # Older SDKs expose one ChatMessage model; newer ones split it by role. + factory = getattr(cohere, "UserMessage", None) or cohere.ChatMessage + kwargs["chat_history"] = [factory(**history)] + if shape == "mixed_history": + kwargs["preamble"] = PREAMBLE + kwargs["chat_history"].append(dict(history)) + elif shape == "dict_history": + kwargs["chat_history"] = [history] + elif shape == "none_history": + kwargs["chat_history"] = None + elif shape == "none_tools": + kwargs["tools"] = None + elif shape == "none_documents": + kwargs["documents"] = None + elif shape == "all_none": + kwargs.update(chat_history=None, tools=None, documents=None, preamble=None) + else: + assert shape == "simple" + return kwargs + + +async def stream_call(asynchronous, kwargs, enabled=True, config=None): + requests = [] + + def respond(request): + requests.append(json.loads(request.content)) + return httpx.Response( + 200, headers={"content-type": "application/x-ndjson"}, + content=b"".join(encode(event) for event in EVENTS), + ) + + with capture(config, enabled) as (exporter, provider): + if asynchronous: + async with httpx.AsyncClient(transport=httpx.MockTransport(respond)) as http: + client = cohere.AsyncClient(api_key=KEY, httpx_client=http) + events = [event async for event in client.chat_stream(**kwargs)] + else: + with httpx.Client(transport=httpx.MockTransport(respond)) as http: + client = cohere.Client(api_key=KEY, httpx_client=http) + events = list(client.chat_stream(**kwargs)) + provider.force_flush() + assert [event.event_type for event in events] == ["text-generation", "stream-end"] + assert events[0].text == REPLY + assert events[1].response.text == REPLY + assert len(requests) == 1 + assert requests[0]["message"] == PROMPT + return exporter.get_finished_spans(), requests[0] + + +def assert_stream(spans): + assert len(spans) == 1, "chat_stream must export exactly one span" + span = spans[0] + assert span.name == "cohere.chat_stream" + assert span.status.status_code == trace.StatusCode.OK + assert span.attributes[SpanAttributes.GEN_AI_REQUEST_MODEL] == "command-r" + assert span.attributes[SpanAttributes.GEN_AI_USAGE_INPUT_TOKENS] == 2 + assert span.attributes[SpanAttributes.GEN_AI_USAGE_OUTPUT_TOKENS] == 3 + assert KEY not in json.dumps(dict(span.attributes)) + return span.attributes + + +def assert_messages(attributes, shape): + expected = [] + if shape == "mixed_history": + expected.append(("system", PREAMBLE)) + if shape in ("sdk_history", "mixed_history", "dict_history"): + expected.append(("USER", HISTORY)) + if shape == "mixed_history": + expected.append(("USER", HISTORY)) + expected.append(("user", PROMPT)) + prefix = SpanAttributes.GEN_AI_INPUT_MESSAGES + for index, (role, message) in enumerate(expected): + assert attributes[f"{prefix}.{index}.{MessageAttributes.MESSAGE_ROLE}"] == role + assert attributes[f"{prefix}.{index}.{MessageAttributes.MESSAGE_CONTENT}"] == message + assert sum(key.startswith(prefix + ".") for key in attributes) == 2 * len(expected) + + +def assert_no_extraction_errors(caplog): + assert not [ + record for record in caplog.records + if record.name.startswith("traceai_cohere") and record.exc_info + ] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("asynchronous", [False, True], ids=["sync", "async"]) +@pytest.mark.parametrize("shape", REGRESSION_SHAPES) +async def test_stream_request_shape_regression(asynchronous, shape, caplog): + kwargs = request_arguments(shape) + original_history = kwargs.get("chat_history") + spans, body = await stream_call(asynchronous, kwargs) + # The telemetry layer must not alter the arguments or their wire encoding. + baseline, expected_body = await stream_call(asynchronous, kwargs, enabled=False) + assert not baseline + assert body == expected_body + assert kwargs.get("chat_history") is original_history + if shape in ("sdk_history", "mixed_history"): + assert not isinstance(original_history[0], dict) + attributes = assert_stream(spans) + assert_messages(attributes, shape) + for key in ("documents", "tools"): + if key in kwargs: + assert attributes[f"cohere.{key}_count"] == 0 + assert_no_extraction_errors(caplog) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("asynchronous", [False, True], ids=["sync", "async"]) +@pytest.mark.parametrize("shape", CONTROL_SHAPES) +async def test_stream_request_shape_control(asynchronous, shape, caplog): + spans, _ = await stream_call(asynchronous, request_arguments(shape)) + assert_messages(assert_stream(spans), shape) + assert_no_extraction_errors(caplog) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("asynchronous", [False, True], ids=["sync", "async"]) +@pytest.mark.parametrize("shape", REGRESSION_SHAPES + CONTROL_SHAPES) +async def test_uninstrumented_request_shape_control(asynchronous, shape): + spans, _ = await stream_call(asynchronous, request_arguments(shape), enabled=False) + assert not spans + + +@pytest.mark.asyncio +@pytest.mark.parametrize("asynchronous", [False, True], ids=["sync", "async"]) +@pytest.mark.parametrize("shape", ["sdk_history", "mixed_history"]) +async def test_stream_request_shape_privacy_regression(asynchronous, shape, caplog): + spans, _ = await stream_call( + asynchronous, request_arguments(shape), config=TraceConfig(hide_inputs=True) + ) + attributes = assert_stream(spans) + assert attributes[SpanAttributes.INPUT_VALUE] == REDACTED_VALUE + assert not any(key.startswith(SpanAttributes.GEN_AI_INPUT_MESSAGES) for key in attributes) + serialized = json.dumps(dict(attributes)) + for value in (HISTORY, PREAMBLE, PROMPT, KEY): + assert value not in serialized + assert_no_extraction_errors(caplog) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("asynchronous", [False, True], ids=["sync", "async"]) +@pytest.mark.parametrize("source", ["primary", "extra", "context"]) +async def test_stream_extraction_failure_regression(asynchronous, source, monkeypatch): + from traceai_cohere import _wrappers + from traceai_cohere._request_attributes_extractor import _ChatRequestAttributesExtractor + + def broken(*args, **kwargs): + yield SpanAttributes.GEN_AI_REQUEST_MODEL, "command-r" + raise RuntimeError("synthetic telemetry extraction failure") + + if source == "context": + monkeypatch.setattr(_wrappers, "get_attributes_from_context", broken) + else: + method = ("get_attributes_from_request" if source == "primary" + else "get_extra_attributes_from_request") + monkeypatch.setattr(_ChatRequestAttributesExtractor, method, broken) + spans, _ = await stream_call(asynchronous, request_arguments("simple")) + assert_stream(spans) diff --git a/python/frameworks/cohere/traceai_cohere/__init__.py b/python/frameworks/cohere/traceai_cohere/__init__.py index 630cb9c9f..2b07f0a70 100644 --- a/python/frameworks/cohere/traceai_cohere/__init__.py +++ b/python/frameworks/cohere/traceai_cohere/__init__.py @@ -17,6 +17,7 @@ from wrapt import wrap_function_wrapper from traceai_cohere._wrappers import ( + _AsyncChatStreamWrapper, _AsyncChatWrapper, _AsyncEmbedWrapper, _AsyncRerankWrapper, @@ -114,6 +115,14 @@ def _instrument(self, **kwargs: Any) -> None: wrapper=_AsyncChatWrapper(tracer=self._tracer), ) + if hasattr(cohere_module.AsyncClient, "chat_stream"): + self._original_async_chat_stream = cohere_module.AsyncClient.chat_stream + wrap_function_wrapper( + module="cohere", + name="AsyncClient.chat_stream", + wrapper=_AsyncChatStreamWrapper(tracer=self._tracer), + ) + if hasattr(cohere_module.AsyncClient, "embed"): self._original_async_embed = cohere_module.AsyncClient.embed wrap_function_wrapper( @@ -156,6 +165,8 @@ def _uninstrument(self, **kwargs: Any) -> None: if hasattr(cohere_module, "AsyncClient"): if hasattr(self, "_original_async_chat"): cohere_module.AsyncClient.chat = self._original_async_chat + if hasattr(self, "_original_async_chat_stream"): + cohere_module.AsyncClient.chat_stream = self._original_async_chat_stream if hasattr(self, "_original_async_embed"): cohere_module.AsyncClient.embed = self._original_async_embed if hasattr(self, "_original_async_rerank"): diff --git a/python/frameworks/cohere/traceai_cohere/_request_attributes_extractor.py b/python/frameworks/cohere/traceai_cohere/_request_attributes_extractor.py index fae95f291..ccc3f7e92 100644 --- a/python/frameworks/cohere/traceai_cohere/_request_attributes_extractor.py +++ b/python/frameworks/cohere/traceai_cohere/_request_attributes_extractor.py @@ -11,6 +11,8 @@ ) from opentelemetry.util.types import AttributeValue +from traceai_cohere._utils import _to_dict + logger = logging.getLogger(__name__) logger.addHandler(logging.NullHandler()) @@ -44,26 +46,29 @@ def get_extra_attributes_from_request( # Keep system/history/current messages in order without overwriting # indices. Canonical input-message keys also honor FI_HIDE_INPUTS. messages = [] - if "preamble" in request_parameters: + if request_parameters.get("preamble") is not None: messages.append({"role": "system", "message": request_parameters["preamble"]}) - messages.extend(request_parameters.get("chat_history", [])) + messages.extend(request_parameters.get("chat_history") or []) if "message" in request_parameters: message = request_parameters.get("message", "") messages.append({"role": "user", "message": message}) yield SpanAttributes.INPUT_VALUE, message for i, msg in enumerate(messages): - yield f"{SpanAttributes.GEN_AI_INPUT_MESSAGES}.{i}.{MessageAttributes.MESSAGE_ROLE}", msg.get("role", "") - yield f"{SpanAttributes.GEN_AI_INPUT_MESSAGES}.{i}.{MessageAttributes.MESSAGE_CONTENT}", msg.get("message", "") + # SDK histories accept model instances as well as dictionaries. + # Convert only for telemetry; leave the actual SDK arguments intact. + message = _to_dict(msg) + yield f"{SpanAttributes.GEN_AI_INPUT_MESSAGES}.{i}.{MessageAttributes.MESSAGE_ROLE}", message.get("role") or "" + yield f"{SpanAttributes.GEN_AI_INPUT_MESSAGES}.{i}.{MessageAttributes.MESSAGE_CONTENT}", message.get("message") or "" # Documents for RAG if "documents" in request_parameters: - docs = request_parameters.get("documents", []) + docs = request_parameters.get("documents") or [] yield "cohere.documents_count", len(docs) # Tools if "tools" in request_parameters: - tools = request_parameters.get("tools", []) + tools = request_parameters.get("tools") or [] yield "cohere.tools_count", len(tools) def _extract_invocation_parameters( diff --git a/python/frameworks/cohere/traceai_cohere/_response_attributes_extractor.py b/python/frameworks/cohere/traceai_cohere/_response_attributes_extractor.py index 212832b66..2baafa4d1 100644 --- a/python/frameworks/cohere/traceai_cohere/_response_attributes_extractor.py +++ b/python/frameworks/cohere/traceai_cohere/_response_attributes_extractor.py @@ -26,20 +26,20 @@ def get_attributes( return # Token counts from meta - meta = response.get("meta", {}) + meta = response.get("meta") or {} if "billed_units" in meta: - billed = meta["billed_units"] + billed = meta["billed_units"] or {} if "input_tokens" in billed: yield SpanAttributes.GEN_AI_USAGE_INPUT_TOKENS, billed["input_tokens"] if "output_tokens" in billed: yield SpanAttributes.GEN_AI_USAGE_OUTPUT_TOKENS, billed["output_tokens"] - total = billed.get("input_tokens", 0) + billed.get("output_tokens", 0) + total = (billed.get("input_tokens") or 0) + (billed.get("output_tokens") or 0) if total: yield SpanAttributes.GEN_AI_USAGE_TOTAL_TOKENS, total if "tokens" in meta: - tokens = meta["tokens"] + tokens = meta["tokens"] or {} if "input_tokens" in tokens: yield SpanAttributes.GEN_AI_USAGE_INPUT_TOKENS, tokens["input_tokens"] if "output_tokens" in tokens: @@ -113,6 +113,15 @@ def get_extra_attributes( # Embeddings embeddings = response.get("embeddings", []) + if isinstance(embeddings, Mapping): + # Typed responses use float/float_ depending on SDK version. + # Prefer floats when multiple representations are present; count + # vectors, not the number of representation keys in the mapping. + embeddings = next( + (embeddings[key] for key in ("float", "float_", "int8", "uint8", "binary", "ubinary") + if embeddings.get(key) is not None), + [], + ) if embeddings: yield "cohere.embeddings_count", len(embeddings) # First embedding info diff --git a/python/frameworks/cohere/traceai_cohere/_wrappers.py b/python/frameworks/cohere/traceai_cohere/_wrappers.py index 31e20a82c..3b414c12a 100644 --- a/python/frameworks/cohere/traceai_cohere/_wrappers.py +++ b/python/frameworks/cohere/traceai_cohere/_wrappers.py @@ -37,6 +37,18 @@ def __init__(self, tracer: trace_api.Tracer, *args: Any, **kwargs: Any) -> None: config = getattr(tracer, "_self_config", None) self._config = config if isinstance(config, TraceConfig) else TraceConfig() + @staticmethod + def _collect_attributes( + attributes: Iterable[Tuple[str, AttributeValue]], + ) -> Dict[str, AttributeValue]: + collected: Dict[str, AttributeValue] = {} + try: + for key, value in attributes: + collected[key] = value + except Exception: + logger.exception("Failed to extract request attributes") + return collected + @contextmanager def _start_as_current_span( self, @@ -45,9 +57,14 @@ def _start_as_current_span( context_attributes: Iterable[Tuple[str, AttributeValue]], extra_attributes: Iterable[Tuple[str, AttributeValue]], ) -> Iterator[_WithSpan]: + # Attribute extraction is best effort, not a prerequisite for a span. + # Keep each group independent and retain values yielded before a failure. + initial_attributes = self._collect_attributes(extra_attributes) + final_attributes = self._collect_attributes(attributes) + span_context_attributes = self._collect_attributes(context_attributes) try: span = self._tracer.start_span( - name=span_name, attributes=dict(extra_attributes) + name=span_name, attributes=initial_attributes ) except Exception: span = INVALID_SPAN @@ -59,8 +76,8 @@ def _start_as_current_span( ) as span: yield _WithSpan( span=span, - context_attributes=dict(context_attributes), - extra_attributes=dict(attributes), + context_attributes=span_context_attributes, + extra_attributes=final_attributes, ) @@ -272,6 +289,99 @@ def streaming_wrapper(): return streaming_wrapper() +class _AsyncChatStreamWrapper(_ChatStreamWrapper): + """Preserve the SDK's lazy async iterator and close its HTTP stream.""" + + def __call__( + self, + wrapped: Callable[..., Any], + instance: Any, + args: Tuple[Any, ...], + kwargs: Mapping[str, Any], + ) -> Any: + response = wrapped(*args, **kwargs) + if context_api.get_value(context_api._SUPPRESS_INSTRUMENTATION_KEY): + return response + return self._handle_async_stream(response, dict(kwargs)) + + async def _handle_async_stream( + self, response: Any, request_parameters: Dict[str, Any] + ): + # Start on first consumption, not construction. Closing an unstarted + # stream therefore neither sends a request nor leaves an open span. + with self._start_as_current_span( + span_name="cohere.chat_stream", + attributes=self._request_extractor.get_attributes_from_request(request_parameters), + context_attributes=get_attributes_from_context(), + extra_attributes=self._request_extractor.get_extra_attributes_from_request(request_parameters), + ) as span: + pass + content_parts: List[str] = [] + final_response: Dict[str, Any] = {} + try: + try: + iterator = response.__aiter__() + while True: + # Never keep our context attached across a yield into + # application code (or across iteration in another task). + with trace_api.use_span( + span._span, end_on_exit=False, + record_exception=False, set_status_on_exception=False, + ): + try: + event = await iterator.__anext__() + except StopAsyncIteration: + break + try: + event_dict = _to_dict(event) + if event_dict.get("event_type") == "text-generation": + if text := event_dict.get("text"): + content_parts.append(text) + elif event_dict.get("event_type") == "stream-end": + final_response = dict(event_dict.get("response") or {}) + except Exception: + logger.exception("Failed to extract streaming response") + yield event + finally: + # Explicit close, cancellation and upstream failures must all + # release the actual SDK generator's HTTP context immediately. + with trace_api.use_span( + span._span, end_on_exit=False, + record_exception=False, set_status_on_exception=False, + ): + await response.aclose() + except Exception as exception: + span.record_exception(exception) + span.finish_tracing(status=trace_api.Status( + status_code=trace_api.StatusCode.ERROR, + description=f"{type(exception).__name__}: {exception}", + )) + raise + else: + try: + final_response["text"] = "".join(content_parts) + # Legacy provider-specific tool fields do not pass through + # the core's canonical output-message masking rules. + if (self._config.hide_outputs or self._config.hide_output_messages + or self._config.hide_output_text): + final_response.pop("tool_calls", None) + _finish_tracing( + status=trace_api.Status(status_code=trace_api.StatusCode.OK), + with_span=span, + attributes=self._response_extractor.get_attributes(final_response, is_streaming=True), + extra_attributes=self._response_extractor.get_extra_attributes( + final_response, request_parameters, is_streaming=True + ), + ) + except Exception: + logger.exception("Failed to finalize streaming response") + span.finish_tracing() + finally: + # GeneratorExit and CancelledError also take this path. They do not + # claim that a partially consumed stream completed successfully. + span.finish_tracing() + + class _EmbedWrapper(_WithTracer): """Wrapper for Cohere embed.""" diff --git a/python/frameworks/cohere/traceai_cohere/version.py b/python/frameworks/cohere/traceai_cohere/version.py index 34d76ee9e..0d33eef3f 100644 --- a/python/frameworks/cohere/traceai_cohere/version.py +++ b/python/frameworks/cohere/traceai_cohere/version.py @@ -1 +1 @@ -__version__ = "0.1.1" # x-release-please-version +__version__ = "0.1.2" # x-release-please-version diff --git a/python/frameworks/deepseek/CHANGELOG.md b/python/frameworks/deepseek/CHANGELOG.md index 770966c4f..a979c75db 100644 --- a/python/frameworks/deepseek/CHANGELOG.md +++ b/python/frameworks/deepseek/CHANGELOG.md @@ -1,5 +1,10 @@ # Changelog +## [0.1.3] - pending release + +### Changed +- traceai-deepseek now requires fi-instrumentation-otel 1.0.0 or later. Action: Upgrade fi-instrumentation-otel if your environment pins an earlier version. + ## [0.1.2](https://github.com/future-agi/traceAI/compare/traceai-deepseek-v0.1.1...traceai-deepseek-v0.1.2) (2026-11-05) diff --git a/python/frameworks/deepseek/pyproject.toml b/python/frameworks/deepseek/pyproject.toml index fcf41a251..ae1f7c87b 100644 --- a/python/frameworks/deepseek/pyproject.toml +++ b/python/frameworks/deepseek/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "traceAI-deepseek" -version = "0.1.2" +version = "0.1.3" description = "OpenTelemetry instrumentation for DeepSeek - chat completions and reasoning models" authors = ["Future AGI "] readme = "README.md" @@ -27,7 +27,7 @@ classifiers = [ [tool.poetry.dependencies] python = ">=3.10,<3.14" -fi-instrumentation-otel = ">=0.1.11" +fi-instrumentation-otel = ">=1.0.0" openai = ">=1.0.0" [tool.poetry.urls] diff --git a/python/frameworks/deepseek/traceai_deepseek/version.py b/python/frameworks/deepseek/traceai_deepseek/version.py index 0d33eef3f..97c339acc 100644 --- a/python/frameworks/deepseek/traceai_deepseek/version.py +++ b/python/frameworks/deepseek/traceai_deepseek/version.py @@ -1 +1 @@ -__version__ = "0.1.2" # x-release-please-version +__version__ = "0.1.3" # x-release-please-version diff --git a/python/frameworks/dspy/CHANGELOG.md b/python/frameworks/dspy/CHANGELOG.md index a9db4fc56..9b9e403ec 100644 --- a/python/frameworks/dspy/CHANGELOG.md +++ b/python/frameworks/dspy/CHANGELOG.md @@ -1,5 +1,13 @@ +## [0.1.10] - pending release + +### Changed +- traceai-dspy now requires fi-instrumentation-otel 1.0.0 or later. Action: Upgrade fi-instrumentation-otel if your environment pins an earlier version. + +### Fixed +- Calls with multipart message content reach the model and are traced instead of failing before the call. + ## [0.1.9] - 2026-10-29 ### Fixed diff --git a/python/frameworks/dspy/poetry.lock b/python/frameworks/dspy/poetry.lock index 625a08968..30d3cb43d 100644 --- a/python/frameworks/dspy/poetry.lock +++ b/python/frameworks/dspy/poetry.lock @@ -3351,4 +3351,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.1" python-versions = ">=3.10, <3.13" -content-hash = "7a1593d5d6f6d805fc4c1556c33cc07aa2fbccbefcc568c7c2a6c8c9a6dcea16" +content-hash = "2d9e227676badace17b40d6798e9907cfb83429a7341439acb1e0a826edfac0d" diff --git a/python/frameworks/dspy/pyproject.toml b/python/frameworks/dspy/pyproject.toml index 7791185fc..ad596d322 100644 --- a/python/frameworks/dspy/pyproject.toml +++ b/python/frameworks/dspy/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "traceAI-DSPy" -version = "0.1.9" +version = "0.1.10" description = "OpenTelemetry instrumentation for DSPy" authors = ["Future AGI "] readme = "README.md" @@ -28,7 +28,7 @@ classifiers = [ [tool.poetry.dependencies] python = ">=3.10, <3.13" dspy = ">=2.5.0" -fi-instrumentation-otel = ">=0.1.11" +fi-instrumentation-otel = ">=1.0.0" [tool.poetry.urls] diff --git a/python/frameworks/dspy/tests/test_lm_multipart.py b/python/frameworks/dspy/tests/test_lm_multipart.py new file mode 100644 index 000000000..b629869a6 --- /dev/null +++ b/python/frameworks/dspy/tests/test_lm_multipart.py @@ -0,0 +1,198 @@ +"""Offline public LM calls through the real model SDK and HTTP response parser.""" + +import importlib +import importlib.util +import inspect +import json +import socket +from types import SimpleNamespace + +import pytest +from fi_instrumentation import TraceConfig +from fi_instrumentation.fi_types import SpanAttributes +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +from opentelemetry.trace import StatusCode + +from traceai_dspy import DSPyInstrumentor, _eval_input + + +TEXT = "synthetic question" +ANSWER = "synthetic answer" +PARTS = [{"type": "text", "text": TEXT}, {"type": "text", "text": "second part"}] + + +@pytest.fixture +def public_lm(monkeypatch, tmp_path): + # No application credentials, disk cache, retries, telemetry, or real sockets. + monkeypatch.setenv("OPENAI_API_KEY", "synthetic-not-a-credential") + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + monkeypatch.setenv("DSPY_CACHEDIR", str(tmp_path / "cache")) + monkeypatch.setenv("DO_NOT_TRACK", "1") + + def no_network(*args, **kwargs): + raise AssertionError("unexpected network access in offline test") + + monkeypatch.setattr(socket.socket, "connect", no_network) + monkeypatch.setattr(socket.socket, "connect_ex", no_network) + monkeypatch.setattr(socket, "getaddrinfo", no_network) + + import dspy + + requests = [] + + def transport(module): + def respond(self, request): + assert request.url.host == "offline.invalid" + assert request.url.path == "/v1/chat/completions" + requests.append(json.loads(request.content)) + return module.Response( + 200, + request=request, + json={ + "id": "chatcmpl_synthetic", + "object": "chat.completion", + "created": 1, + "model": "gpt-4o", + "choices": [{"index": 0, "finish_reason": "stop", "message": { + "role": "assistant", "content": ANSWER, + }}], + "usage": {"prompt_tokens": 2, "completion_tokens": 2, "total_tokens": 4}, + }, + ) + return respond + + # SDK releases use either HTTP transport; no model method or parser is mocked. + for name in ("httpx", "httpx2"): + if importlib.util.find_spec(name) is not None: + module = importlib.import_module(name) + monkeypatch.setattr(module.HTTPTransport, "handle_request", transport(module)) + + kwargs = {"engine": "litellm"} if "engine" in inspect.signature(dspy.LM).parameters else {} + lm = dspy.LM( + "openai/gpt-4o", cache=False, num_retries=0, + api_base="https://offline.invalid/v1", **kwargs, + ) + exporter = InMemorySpanExporter() + provider = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + instrumentor = DSPyInstrumentor() + missing = object() + targets = [ + (dspy.LM, "__call__"), (dspy.Predict, "forward"), + (dspy.Retrieve, "forward"), (dspy.Module, "__call__"), + (dspy.ColBERTv2, "__call__"), (dspy.Adapter, "__call__"), + (dspy.Embedder, "__call__"), + *((cls, "forward") for cls in dspy.Predict.__subclasses__()), + ] + originals = [(cls, name, vars(cls).get(name, missing)) for cls, name in targets] + + def enable(hidden=False): + instrumentor.instrument( + tracer_provider=provider, + config=TraceConfig(hide_inputs=hidden, hide_outputs=hidden, + hide_llm_invocation_parameters=True), + ) + assert instrumentor.is_instrumented_by_opentelemetry + + try: + yield SimpleNamespace(lm=lm, requests=requests, exporter=exporter, enable=enable) + finally: + if instrumentor.is_instrumented_by_opentelemetry: + instrumentor.uninstrument() + # Keep cases isolated despite older releases' partial public teardown. + for cls, name, original in originals: + if original is missing: + if name in vars(cls): + delattr(cls, name) + else: + setattr(cls, name, original) + provider.shutdown() + + +def assert_call(public_lm, messages, instrumented): + public_lm.requests.clear() + public_lm.exporter.clear() + assert public_lm.lm(messages=messages) == [ANSWER] + assert len(public_lm.requests) == 1 + assert public_lm.requests[0]["messages"] == messages + spans = public_lm.exporter.get_finished_spans() + assert len(spans) == int(instrumented) + if instrumented: + assert spans[0].name == "LM.__call__" + assert spans[0].status.status_code == StatusCode.OK + assert spans[0].end_time is not None + assert "synthetic-not-a-credential" not in json.dumps(dict(spans[0].attributes)) + return spans + + +@pytest.mark.parametrize("content", [TEXT, PARTS], ids=["string", "multipart"]) +def test_public_lm_native_messages_reach_http(public_lm, content): + assert_call(public_lm, [{"role": "user", "content": content}], False) + + +def test_public_lm_string_messages_reach_http_when_instrumented(public_lm): + public_lm.enable() + assert_call(public_lm, [{"role": "user", "content": TEXT}], True) + + +@pytest.mark.parametrize("hidden", [False, True], ids=["visible", "hidden"]) +def test_public_lm_multipart_messages_reach_http_when_instrumented(public_lm, hidden): + messages = [{"role": "user", "content": PARTS}] + assert_call(public_lm, messages, False) + public_lm.enable(hidden) + spans = assert_call(public_lm, messages, True) + attributes = dict(spans[0].attributes) + if hidden: + encoded = json.dumps(attributes) + for value in (TEXT, "second part", ANSWER): + assert value not in encoded + assert not any(key.startswith((SpanAttributes.GEN_AI_INPUT_MESSAGES, + SpanAttributes.GEN_AI_OUTPUT_MESSAGES)) + for key in attributes) + else: + assert json.loads(attributes[SpanAttributes.INPUT_VALUE])["messages"] == messages + assert json.loads(attributes[SpanAttributes.OUTPUT_VALUE]) == [ANSWER] + + +def test_public_lm_string_messages_respect_hidden_io(public_lm): + public_lm.enable(hidden=True) + spans = assert_call(public_lm, [{"role": "user", "content": TEXT}], True) + encoded = json.dumps(dict(spans[0].attributes)) + assert TEXT not in encoded + assert ANSWER not in encoded + + +class UnconvertibleMessage(dict): + """JSON-valid input whose optional tracing conversion is unavailable.""" + + def to_dict(self): + raise ValueError("synthetic tracing conversion failure") + + +def test_public_lm_input_attribute_failure_does_not_abort_dispatch(public_lm): + messages = [UnconvertibleMessage(role="user", content=TEXT)] + assert_call(public_lm, messages, False) + public_lm.enable() + assert_call(public_lm, messages, True) + + +@pytest.mark.parametrize( + "messages, expected", + [ + ([{"content": "first"}, {"content": "second"}], "first\nsecond"), + ([{"content": PARTS}], json.dumps(PARTS)), + ([{"content": "first"}, {"content": PARTS}], "first\n" + json.dumps(PARTS)), + ([{"content": None}], "null"), + ([{"role": "assistant"}], ""), + ], + ids=["strings", "multipart", "mixed", "null", "missing"], +) +def test_eval_input_serializes_message_content(messages, expected): + assert json.loads(dict(_eval_input(messages))[SpanAttributes.INPUT_VALUE]) == expected + + +def test_eval_input_without_messages(): + assert dict(_eval_input(None)) == {} + assert dict(_eval_input([])) == {} diff --git a/python/frameworks/dspy/traceai_dspy/__init__.py b/python/frameworks/dspy/traceai_dspy/__init__.py index cc4323344..fd03f7855 100644 --- a/python/frameworks/dspy/traceai_dspy/__init__.py +++ b/python/frameworks/dspy/traceai_dspy/__init__.py @@ -221,27 +221,35 @@ def __call__( ) -> Any: if context_api.get_value(context_api._SUPPRESS_INSTRUMENTATION_KEY): return wrapped(*args, **kwargs) - arguments = _bind_arguments(wrapped, *args, **kwargs) - messages = arguments.get("messages") - prompts = arguments.get("prompt") span_name = instance.__class__.__name__ + ".__call__" + attributes: Dict[str, AttributeValue] = {GEN_AI_SPAN_KIND: LLM.value} + try: + arguments = _bind_arguments(wrapped, *args, **kwargs) + messages = arguments.get("messages") + prompts = arguments.get("prompt") + attributes.update( + dict( + _flatten( + { + **dict(_input_value_and_mime_type(messages)), + **dict(_eval_input(messages)), + **dict(_llm_prompt(prompts)), + **dict(_raw_input(arguments)), + **dict(_llm_model_name(instance)), + **dict(_llm_invocation_parameters(instance, arguments)), + **dict(_llm_input_messages(arguments)), + **dict(get_attributes_from_context()), + } + ) + ) + ) + except Exception: + # Attribute extraction must not prevent the application model call. + # Do not include payloads or exception details in this diagnostic. + logger.warning("Failed to extract model input attributes") with self._tracer.start_as_current_span( span_name, - attributes=dict( - _flatten( - { - GEN_AI_SPAN_KIND: LLM, - **dict(_input_value_and_mime_type(messages)), - **dict(_eval_input(messages)), - **dict(_llm_prompt(prompts)), - **dict(_raw_input(arguments)), - **dict(_llm_model_name(instance)), - **dict(_llm_invocation_parameters(instance, arguments)), - **dict(_llm_input_messages(arguments)), - **dict(get_attributes_from_context()), - } - ) - ), + attributes=attributes, ) as span: response = wrapped(*args, **kwargs) span.set_status(StatusCode.OK) @@ -748,13 +756,12 @@ def _eval_input(messages: list) -> Iterator[Tuple[str, Any]]: eval_input = [] for message in messages: if isinstance(message, dict) and "content" in message: - eval_input.append(message["content"]) - - if eval_input and len(eval_input) > 0: - yield INPUT_VALUE, safe_json_dumps(eval_input[0]) + content = message["content"] + eval_input.append( + content if isinstance(content, str) else safe_json_dumps(content) + ) - eval_input = "\n".join(eval_input) - yield INPUT_VALUE, safe_json_dumps(eval_input) + yield INPUT_VALUE, safe_json_dumps("\n".join(eval_input)) def _convert_to_dict(obj: Any) -> Any: diff --git a/python/frameworks/dspy/traceai_dspy/version.py b/python/frameworks/dspy/traceai_dspy/version.py index a99028c1f..b2b8e3d08 100644 --- a/python/frameworks/dspy/traceai_dspy/version.py +++ b/python/frameworks/dspy/traceai_dspy/version.py @@ -1 +1 @@ -__version__ = "0.1.9" # x-release-please-version +__version__ = "0.1.10" # x-release-please-version diff --git a/python/frameworks/fireworks/CHANGELOG.md b/python/frameworks/fireworks/CHANGELOG.md index 2a69edb36..e2b15b6a5 100644 --- a/python/frameworks/fireworks/CHANGELOG.md +++ b/python/frameworks/fireworks/CHANGELOG.md @@ -1,5 +1,10 @@ # Changelog +## [0.1.3] - pending release + +### Changed +- traceai-fireworks now requires fi-instrumentation-otel 1.3.0 or later. Action: Upgrade fi-instrumentation-otel if your environment pins an earlier version. + ## [0.1.2](https://github.com/future-agi/traceAI/compare/traceai-fireworks-v0.1.1...traceai-fireworks-v0.1.2) (2026-11-05) diff --git a/python/frameworks/fireworks/pyproject.toml b/python/frameworks/fireworks/pyproject.toml index 8021a0075..2ffe3da18 100644 --- a/python/frameworks/fireworks/pyproject.toml +++ b/python/frameworks/fireworks/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "traceAI-fireworks" -version = "0.1.2" +version = "0.1.3" description = "OpenTelemetry instrumentation for Fireworks AI" authors = ["Future AGI "] readme = "README.md" @@ -27,7 +27,7 @@ classifiers = [ [tool.poetry.dependencies] python = ">=3.10,<3.14" -fi-instrumentation-otel = ">=1.2.2" +fi-instrumentation-otel = ">=1.3.0" openai = ">=1.0.0" [tool.poetry.group.dev.dependencies] diff --git a/python/frameworks/fireworks/traceai_fireworks/version.py b/python/frameworks/fireworks/traceai_fireworks/version.py index 0d33eef3f..97c339acc 100644 --- a/python/frameworks/fireworks/traceai_fireworks/version.py +++ b/python/frameworks/fireworks/traceai_fireworks/version.py @@ -1 +1 @@ -__version__ = "0.1.2" # x-release-please-version +__version__ = "0.1.3" # x-release-please-version diff --git a/python/frameworks/google-genai/CHANGELOG.md b/python/frameworks/google-genai/CHANGELOG.md index 606a9879d..1a8a192fb 100644 --- a/python/frameworks/google-genai/CHANGELOG.md +++ b/python/frameworks/google-genai/CHANGELOG.md @@ -1,11 +1,11 @@ -## [Unreleased] +## [0.1.4] - pending release + +### Changed +- traceai-google-genai now requires fi-instrumentation-otel 1.0.0 or later. Action: Upgrade fi-instrumentation-otel if your environment pins an earlier version. ### Fixed -- Fix offline sync/async streaming tests on `google-genai` 3.x using public HTTP - transport options, retaining the 1.8.0 fallback and blocking network access. -- Preserve distinct parts in streaming responses so parallel function calls, - repeated calls to the same tool, and multimedia parts are not overwritten or - merged into text parts in exported traces. +- Preserve distinct parts in streaming responses so parallel function calls, repeated calls to the same tool, and multimedia parts are not overwritten or merged into text parts in exported traces. +- Streamed traces combine adjacent plain-text fragments without merging function calls or multimedia parts into text. ## [0.1.3] - 2026-10-29 diff --git a/python/frameworks/google-genai/poetry.lock b/python/frameworks/google-genai/poetry.lock index fe29db88f..25ef4876d 100644 --- a/python/frameworks/google-genai/poetry.lock +++ b/python/frameworks/google-genai/poetry.lock @@ -1211,4 +1211,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.1" python-versions = ">=3.10,<3.14" -content-hash = "5e1a878a25dca160839ffa3a6a075f5bc64ae6ae499803ae8a775ced02777c77" +content-hash = "134afd7cd4479afc6ed6a4521b2044fbd592e72c09f84a4970b259b0ce2211b6" diff --git a/python/frameworks/google-genai/pyproject.toml b/python/frameworks/google-genai/pyproject.toml index d0272939a..89d854c1d 100644 --- a/python/frameworks/google-genai/pyproject.toml +++ b/python/frameworks/google-genai/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "traceAI-google-genai" -version = "0.1.3" +version = "0.1.4" description = "OpenTelemetry instrumentation for Google Gen AI" authors = ["Future AGI "] readme = "README.md" @@ -29,7 +29,7 @@ classifiers = [ [tool.poetry.dependencies] python = ">=3.10,<3.14" google-genai = ">=1.8.0" -fi-instrumentation-otel = ">=0.1.11" +fi-instrumentation-otel = ">=1.0.0" [tool.poetry.urls] diff --git a/python/frameworks/google-genai/tests/test_stream_accumulation.py b/python/frameworks/google-genai/tests/test_stream_accumulation.py new file mode 100644 index 000000000..b4fe18d3e --- /dev/null +++ b/python/frameworks/google-genai/tests/test_stream_accumulation.py @@ -0,0 +1,159 @@ +"""Stream text deltas coalesce without erasing call or media boundaries.""" + +import asyncio +import json +from copy import deepcopy +from unittest.mock import Mock + +import pytest +from fi_instrumentation.fi_types import SpanAttributes +from opentelemetry import trace as trace_api +from traceai_google_genai._stream import _Stream +from traceai_google_genai._with_span import _WithSpan + + +class _Chunk: + def __init__(self, parts, index=0): + self.values = { + "candidates": [ + {"index": index, "content": {"role": "model", "parts": parts}} + ] + } + + def model_dump(self, **kwargs): + return self.values + + +@pytest.mark.parametrize("asynchronous", [False, True], ids=["sync", "async"]) +@pytest.mark.parametrize( + "groups, expected", + [ + ( + [[{"text": "hello"}], [{"text": " world"}]], + [{"text": "hello world"}], + ), + ( + [[{"text": "hello"}, {"text": " world"}]], + [{"text": "hello world"}], + ), + ( + [ + [{"text": "hello"}], + [{"function_call": {"name": "weather", "args": {"city": "Paris"}}}], + [{"text": " world"}], + ], + [ + {"text": "hello"}, + {"function_call": {"name": "weather", "args": {"city": "Paris"}}}, + {"text": " world"}, + ], + ), + ( + [[ + {"function_call": {"name": "weather", "args": {"city": "Paris"}}}, + {"function_call": {"name": "time", "args": {"zone": "UTC"}}}, + ]], + [ + {"function_call": {"name": "weather", "args": {"city": "Paris"}}}, + {"function_call": {"name": "time", "args": {"zone": "UTC"}}}, + ], + ), + ( + [ + [{"function_call": {"name": "weather", "args": {"city": "Paris"}}}], + [{"function_call": {"name": "weather", "args": {"city": "Paris"}}}], + ], + [ + {"function_call": {"name": "weather", "args": {"city": "Paris"}}}, + {"function_call": {"name": "weather", "args": {"city": "Paris"}}}, + ], + ), + ( + [ + [{"function_call": {"id": "call-a", "name": "weather", "args": {"city": "Paris"}}}], + [{"function_call": {"id": "call-b", "name": "weather", "args": {"city": "London"}}}], + [{"function_call": {"id": "call-a", "args": {"units": "C"}}}], + ], + [ + {"function_call": {"id": "call-a", "name": "weather", "args": {"city": "Paris", "units": "C"}}}, + {"function_call": {"id": "call-b", "name": "weather", "args": {"city": "London"}}}, + ], + ), + ( + [ + [{"text": "hello"}], + [{"file_data": {"file_uri": "gs://example/image", "mime_type": "image/png"}}], + [{"text": " world"}], + ], + [ + {"text": "hello"}, + {"file_data": {"file_uri": "gs://example/image", "mime_type": "image/png"}}, + {"text": " world"}, + ], + ), + ( + [ + [{"text": "thinking", "thought": True}], + [{"text": "hello"}], + [{"text": " world"}], + ], + [{"text": "thinking", "thought": True}, {"text": "hello world"}], + ), + ( + [ + [{"text": "hello"}], + [{"function_call": {"id": "call-a", "name": "weather", "args": {"city": "Paris"}}}], + [{"text": " world"}], + [{"function_call": {"id": "call-a", "args": {"units": "C"}}}], + [{"text": "!"}], + ], + [ + {"text": "hello"}, + {"function_call": {"id": "call-a", "name": "weather", "args": {"city": "Paris", "units": "C"}}}, + {"text": " world"}, + {"text": "!"}, + ], + ), + ], + ids=[ + "adjacent-text-chunks", + "adjacent-text-parts", + "text-call-text", + "parallel-calls", + "identical-indexless-calls", + "explicit-call-id-deltas", + "text-media-text", + "thought-boundary", + "call-delta-breaks-text-run", + ], +) +def test_stream_accumulation(asynchronous, groups, expected): + chunks = [_Chunk(parts) for parts in groups] + snapshots = deepcopy([chunk.values for chunk in chunks]) + span = Mock(spec=trace_api.Span) + span.is_recording.return_value = True + with_span = Mock(wraps=_WithSpan(span)) + + if asynchronous: + async def consume(): + async def source(): + for chunk in chunks: + yield chunk + + stream = _Stream(source(), with_span) + return [chunk async for chunk in stream] + + received = asyncio.run(consume()) + else: + stream = _Stream(iter(chunks), with_span) + received = list(stream) + + assert all(actual is original for actual, original in zip(received, chunks)) + assert len(received) == len(chunks) + assert [chunk.values for chunk in chunks] == snapshots + with_span.finish_tracing.assert_called_once() + span.end.assert_called_once_with() + kwargs = with_span.finish_tracing.call_args.kwargs + assert kwargs["status"].status_code == trace_api.StatusCode.OK + output = json.loads(kwargs["attributes"][SpanAttributes.OUTPUT_VALUE]) + assert output["candidates"][0]["content"]["parts"] == expected diff --git a/python/frameworks/google-genai/tests/test_stream_calls.py b/python/frameworks/google-genai/tests/test_stream_calls.py new file mode 100644 index 000000000..c9275d9ec --- /dev/null +++ b/python/frameworks/google-genai/tests/test_stream_calls.py @@ -0,0 +1,186 @@ +"""Preserve parallel calls in Google GenAI streams. + +The fake response implements only the SDK's serialization boundary; no client or +network calls are needed. Parts deliberately have no index, as in google-genai. +""" + +from copy import deepcopy +import json + +import pytest + +from fi_instrumentation.fi_types import ( + MessageAttributes, + SpanAttributes, + ToolCallAttributes, +) +from traceai_google_genai._stream import _ResponseAccumulator, _ResponseExtractor + + +class _Chunk: + def __init__(self, payload): + self.payload = payload + + def model_dump(self, *, exclude_unset, warnings): + assert exclude_unset is True + assert warnings is False + # Returning the original also catches accidental mutation by tracing. + return self.payload + + +def _chunk(*parts, index=0): + return _Chunk({ + "candidates": [{ + "index": index, + "content": {"role": "model", "parts": list(parts)}, + }], + }) + + +def _call(name, args, call_id=None): + function_call = {"name": name, "args": args} + if call_id is not None: + function_call["id"] = call_id + return {"function_call": function_call} + + +def _assert_recorded_calls(accumulator, expected, candidate_index=0): + result = accumulator._result() + parts = result["candidates"][candidate_index]["content"]["parts"] + assert [part["function_call"] for part in parts if "function_call" in part] == expected + + extractor = _ResponseExtractor(accumulator) + raw_attributes = dict(extractor.get_attributes()) + assert json.loads(raw_attributes[SpanAttributes.OUTPUT_VALUE]) == result + + attributes = dict(extractor.get_extra_attributes()) + prefix = ( + f"{SpanAttributes.GEN_AI_OUTPUT_MESSAGES}.{candidate_index}." + f"{MessageAttributes.MESSAGE_TOOL_CALLS}." + ) + for index, function_call in enumerate(expected): + assert attributes[ + f"{prefix}{index}.{ToolCallAttributes.TOOL_CALL_FUNCTION_NAME}" + ] == function_call["name"] + if function_call["args"]: + assert json.loads(attributes[ + f"{prefix}{index}.{ToolCallAttributes.TOOL_CALL_FUNCTION_ARGUMENTS_JSON}" + ]) == function_call["args"] + names = [ + value for key, value in attributes.items() + if key.startswith(prefix) and key.endswith( + f".{ToolCallAttributes.TOOL_CALL_FUNCTION_NAME}" + ) + ] + assert names == [function_call["name"] for function_call in expected] + + +@pytest.mark.parametrize("separate_chunks", [False, True], ids=["one-chunk", "across-chunks"]) +@pytest.mark.parametrize("second", [ + _call("clock", {"zone": "UTC"}), + _call("weather", {"city": "Rome"}), + _call("weather", {"city": "Paris"}), +], ids=["different-names", "same-name", "identical-calls"]) +def test_parallel_function_calls_without_indexes(separate_chunks, second): + first = _call("weather", {"city": "Paris"}) + chunks = [_chunk(first), _chunk(second)] if separate_chunks else [_chunk(first, second)] + before = deepcopy([chunk.payload for chunk in chunks]) + accumulator = _ResponseAccumulator() + + for chunk in chunks: + accumulator.process_chunk(chunk) + + _assert_recorded_calls(accumulator, [first["function_call"], second["function_call"]]) + assert [chunk.payload for chunk in chunks] == before + + +def test_identified_call_deltas_follow_identity_not_position(): + chunks = [ + _chunk( + _call("weather", {"city": "Paris"}, "call-paris"), + _call("weather", {"city": "Rome"}, "call-rome"), + ), + _chunk( + {"function_call": {"id": "call-rome", "args": {"units": "fahrenheit"}}}, + {"function_call": {"id": "call-paris", "args": {"units": "celsius"}}}, + ), + ] + before = deepcopy([chunk.payload for chunk in chunks]) + accumulator = _ResponseAccumulator() + for chunk in chunks: + accumulator.process_chunk(chunk) + + _assert_recorded_calls(accumulator, [ + {"id": "call-paris", "name": "weather", "args": {"city": "Paris", "units": "celsius"}}, + {"id": "call-rome", "name": "weather", "args": {"city": "Rome", "units": "fahrenheit"}}, + ]) + assert [chunk.payload for chunk in chunks] == before + + +def test_text_deltas_do_not_hide_function_calls(): + weather = _call("weather", {"city": "Paris"}) + clock = _call("clock", {"zone": "UTC"}) + accumulator = _ResponseAccumulator() + accumulator.process_chunk(_chunk({"text": "I will "})) + accumulator.process_chunk(_chunk(weather, {"text": "check."}, clock)) + + _assert_recorded_calls(accumulator, [weather["function_call"], clock["function_call"]]) + attributes = dict(_ResponseExtractor(accumulator).get_extra_attributes()) + assert attributes[ + f"{SpanAttributes.GEN_AI_OUTPUT_MESSAGES}.0.{MessageAttributes.MESSAGE_CONTENT}" + ] == "I will check." + + +@pytest.mark.parametrize("args", [{"city": "Paris"}, {}], ids=["with-args", "empty-args"]) +def test_single_function_call_control(args): + call = _call("weather", args) + accumulator = _ResponseAccumulator() + accumulator.process_chunk(_chunk(call)) + _assert_recorded_calls(accumulator, [call["function_call"]]) + assert accumulator._result()["candidates"][0]["content"]["parts"] == [call] + + +def test_call_identity_is_scoped_to_candidate_index(): + accumulator = _ResponseAccumulator() + first = _chunk(_call("weather", {"city": "Paris"}, "shared-id"), index=0) + second = _chunk(_call("weather", {"city": "Rome"}, "shared-id"), index=1) + accumulator.process_chunk(_Chunk({ + "candidates": second.payload["candidates"] + first.payload["candidates"], + })) + accumulator.process_chunk(_chunk( + {"function_call": {"id": "shared-id", "args": {"units": "celsius"}}}, + index=0, + )) + + _assert_recorded_calls(accumulator, [ + {"id": "shared-id", "name": "weather", "args": {"city": "Paris", "units": "celsius"}}, + ], candidate_index=0) + _assert_recorded_calls(accumulator, [ + {"id": "shared-id", "name": "weather", "args": {"city": "Rome"}}, + ], candidate_index=1) + + +def test_empty_stream_and_metadata_control(): + accumulator = _ResponseAccumulator() + assert accumulator._result() is None + accumulator.process_chunk(_chunk({"text": "Hello "})) + accumulator.process_chunk(_chunk({"text": "world"})) + accumulator.process_chunk(_Chunk({ + "candidates": [{"index": 0, "finish_reason": "STOP"}], + "usage_metadata": { + "prompt_token_count": 3, + "candidates_token_count": 2, + "total_token_count": 5, + }, + "model_version": "fake-genai-model", + })) + + attributes = dict(_ResponseExtractor(accumulator).get_extra_attributes()) + assert attributes[ + f"{SpanAttributes.GEN_AI_OUTPUT_MESSAGES}.0.{MessageAttributes.MESSAGE_CONTENT}" + ] == "Hello world" + assert attributes[SpanAttributes.GEN_AI_REQUEST_MODEL] == "fake-genai-model" + assert attributes[SpanAttributes.GEN_AI_USAGE_INPUT_TOKENS] == 3 + assert attributes[SpanAttributes.GEN_AI_USAGE_OUTPUT_TOKENS] == 2 + assert attributes[SpanAttributes.GEN_AI_USAGE_TOTAL_TOKENS] == 5 + assert accumulator._result()["candidates"][0]["finish_reason"] == "STOP" diff --git a/python/frameworks/google-genai/tests/test_stream_lifecycle.py b/python/frameworks/google-genai/tests/test_stream_lifecycle.py new file mode 100644 index 000000000..4f59d2997 --- /dev/null +++ b/python/frameworks/google-genai/tests/test_stream_lifecycle.py @@ -0,0 +1,258 @@ +"""Closing a GenAI stream must finish its partial span.""" + +import asyncio +import json +from types import SimpleNamespace +from unittest.mock import AsyncMock, Mock + +import pytest +from opentelemetry import trace as trace_api + +from fi_instrumentation.fi_types import SpanAttributes +from traceai_google_genai._stream import _Stream +from traceai_google_genai._with_span import _WithSpan + + +class _Chunk: + def __init__(self, text): + self.text = text + + def model_dump(self, **kwargs): + return { + "candidates": [ + {"content": {"role": "model", "parts": [{"text": self.text}]}} + ] + } + + +def _make_stream(provider): + span = Mock(spec=trace_api.Span) + span.is_recording.return_value = True + # Spy on the real span lifetime as well as the stream's finalization count. + with_span = Mock(wraps=_WithSpan(span)) + return _Stream(provider, with_span), with_span, span + + +def _assert_finished(with_span, span, text=None, status_code=None): + with_span.finish_tracing.assert_called_once() + span.end.assert_called_once_with() + kwargs = with_span.finish_tracing.call_args.kwargs + if text is None: + assert SpanAttributes.OUTPUT_VALUE not in kwargs["attributes"] + else: + output = json.loads(kwargs["attributes"][SpanAttributes.OUTPUT_VALUE]) + assert output["candidates"][0]["content"]["parts"][0]["text"] == text + if status_code is None: + assert kwargs["status"] is None + else: + assert kwargs["status"].status_code == status_code + + +def _sync_provider(closed, error=None): + try: + yield _Chunk("hello") + if error is not None: + raise error + yield _Chunk(" world") + finally: + closed.append(True) + + +async def _async_provider(closed, error=None): + try: + yield _Chunk("hello") + if error is not None: + raise error + yield _Chunk(" world") + finally: + closed.append(True) + + +@pytest.mark.parametrize("close_target", ["iterator", "stream"]) +def test_sync_early_close_finishes_partial_span_once(close_target): + closed = [] + provider = _sync_provider(closed) + stream, with_span, span = _make_stream(provider) + iterator = iter(stream) + try: + assert next(iterator).text == "hello" + (iterator if close_target == "iterator" else stream).close() + _assert_finished(with_span, span, "hello") + assert closed == [True] + iterator.close() + stream.close() + _assert_finished(with_span, span, "hello") + finally: + iterator.close() + provider.close() + + +@pytest.mark.parametrize("close_target", ["iterator", "stream"]) +def test_async_early_close_finishes_partial_span_once(close_target): + async def run(): + closed = [] + provider = _async_provider(closed) + stream, with_span, span = _make_stream(provider) + iterator = stream.__aiter__() + try: + assert (await iterator.__anext__()).text == "hello" + await (iterator if close_target == "iterator" else stream).aclose() + _assert_finished(with_span, span, "hello") + assert closed == [True] + await iterator.aclose() + await stream.aclose() + _assert_finished(with_span, span, "hello") + finally: + await iterator.aclose() + await provider.aclose() + + asyncio.run(run()) + + +def test_async_cancellation_finishes_partial_span_and_propagates(): + async def run(): + waiting = asyncio.Event() + closed = [] + + async def provider(): + try: + yield _Chunk("hello") + waiting.set() + await asyncio.Event().wait() + finally: + closed.append(True) + + source = provider() + stream, with_span, span = _make_stream(source) + iterator = stream.__aiter__() + assert (await iterator.__anext__()).text == "hello" + pending = asyncio.create_task(iterator.__anext__()) + try: + await asyncio.wait_for(waiting.wait(), timeout=1) + pending.cancel() + with pytest.raises(asyncio.CancelledError): + await pending + # Check before asyncio.run's automatic async-generator cleanup. + _assert_finished(with_span, span, "hello") + assert closed == [True] + await stream.aclose() + _assert_finished(with_span, span, "hello") + finally: + pending.cancel() + try: + await pending + except asyncio.CancelledError: + pass + await iterator.aclose() + await source.aclose() + + asyncio.run(run()) + + +@pytest.mark.parametrize("fails", [False, True], ids=["full", "provider-error"]) +def test_sync_completion_and_errors_keep_status_and_finish_once(fails): + closed = [] + error = ValueError("fake provider error") if fails else None + stream, with_span, span = _make_stream(_sync_provider(closed, error)) + iterator = iter(stream) + assert next(iterator).text == "hello" + if fails: + with pytest.raises(ValueError) as caught: + next(iterator) + assert caught.value is error + span.record_exception.assert_called_once_with(error) + else: + assert [chunk.text for chunk in iterator] == [" world"] + span.record_exception.assert_not_called() + assert closed == [True] + stream.close() + _assert_finished( + with_span, + span, + "hello" if fails else "hello world", + trace_api.StatusCode.ERROR if fails else trace_api.StatusCode.OK, + ) + + +@pytest.mark.parametrize("fails", [False, True], ids=["full", "provider-error"]) +def test_async_completion_and_errors_keep_status_and_finish_once(fails): + async def run(): + closed = [] + error = ValueError("fake provider error") if fails else None + stream, with_span, span = _make_stream(_async_provider(closed, error)) + iterator = stream.__aiter__() + assert (await iterator.__anext__()).text == "hello" + if fails: + with pytest.raises(ValueError) as caught: + await iterator.__anext__() + assert caught.value is error + span.record_exception.assert_called_once_with(error) + else: + assert [chunk.text async for chunk in iterator] == [" world"] + span.record_exception.assert_not_called() + assert closed == [True] + await stream.aclose() + _assert_finished( + with_span, + span, + "hello" if fails else "hello world", + trace_api.StatusCode.ERROR if fails else trace_api.StatusCode.OK, + ) + + asyncio.run(run()) + + +def test_close_before_iteration_finishes_span(): + provider = _sync_provider([]) + stream, with_span, span = _make_stream(provider) + stream.close() + stream.close() + assert provider.gi_frame is None + _assert_finished(with_span, span) + + +def test_aclose_before_iteration_finishes_span(): + async def run(): + provider = _async_provider([]) + stream, with_span, span = _make_stream(provider) + await stream.aclose() + await stream.aclose() + assert provider.ag_frame is None + _assert_finished(with_span, span) + + asyncio.run(run()) + + +@pytest.mark.parametrize("fails", [False, True], ids=["result", "cleanup-error"]) +def test_close_preserves_provider_result_or_error_and_finishes(fails): + result = object() + error = RuntimeError("fake close error") + close = Mock(return_value=result, side_effect=error if fails else None) + stream, with_span, span = _make_stream(SimpleNamespace(close=close)) + if fails: + with pytest.raises(RuntimeError) as caught: + stream.close() + assert caught.value is error + else: + assert stream.close() is result + close.assert_called_once_with() + _assert_finished(with_span, span) + + +@pytest.mark.parametrize("fails", [False, True], ids=["result", "cleanup-error"]) +def test_aclose_preserves_provider_result_or_error_and_finishes(fails): + async def run(): + result = object() + error = RuntimeError("fake aclose error") + close = AsyncMock(return_value=result, side_effect=error if fails else None) + stream, with_span, span = _make_stream(SimpleNamespace(aclose=close)) + if fails: + with pytest.raises(RuntimeError) as caught: + await stream.aclose() + assert caught.value is error + else: + assert await stream.aclose() is result + close.assert_awaited_once_with() + _assert_finished(with_span, span) + + asyncio.run(run()) diff --git a/python/frameworks/google-genai/tests/test_stream_parts.py b/python/frameworks/google-genai/tests/test_stream_parts.py index f4a27f4d9..952ab091f 100644 --- a/python/frameworks/google-genai/tests/test_stream_parts.py +++ b/python/frameworks/google-genai/tests/test_stream_parts.py @@ -204,6 +204,9 @@ def test_stream_preserves_parts(tracing, asynchronous, groups): [part.model_dump(exclude_unset=True) for part in expected_parts] ) ) + if all(set(part) == {"text"} for part in expected): + # Adjacent text deltas form a single part; non-text boundaries remain. + expected = [{"text": "".join(part["text"] for part in expected)}] actual = _messages(result)[0] assert actual["content"]["parts"] == expected assert actual["finish_reason"] == "STOP" diff --git a/python/frameworks/google-genai/traceai_google_genai/_stream.py b/python/frameworks/google-genai/traceai_google_genai/_stream.py index 3f05adfe9..17589ddf6 100644 --- a/python/frameworks/google-genai/traceai_google_genai/_stream.py +++ b/python/frameworks/google-genai/traceai_google_genai/_stream.py @@ -1,3 +1,4 @@ +import logging from collections import defaultdict from copy import deepcopy from typing import ( @@ -37,6 +38,9 @@ if TYPE_CHECKING: from google.genai.types import GenerateContentResponse +logger = logging.getLogger(__name__) +logger.addHandler(logging.NullHandler()) + class _Stream(ObjectProxy): # type: ignore __slots__ = ( @@ -53,8 +57,10 @@ def __init__( super().__init__(stream) self._response_accumulator = _ResponseAccumulator() self._with_span = with_span + self._is_finished = False def __iter__(self) -> Iterator["GenerateContentResponse"]: + status = None try: for item in self.__wrapped__: self._response_accumulator.process_chunk(item) @@ -65,15 +71,20 @@ def __iter__(self) -> Iterator["GenerateContentResponse"]: description=f"{type(exception).__name__}: {exception}", ) self._with_span.record_exception(exception) - self._finish_tracing(status=status) raise - # completed without exception - status = trace_api.Status( - status_code=trace_api.StatusCode.OK, - ) - self._finish_tracing(status=status) + else: + status = trace_api.Status(status_code=trace_api.StatusCode.OK) + finally: + self._finish_tracing(status=status) + if status is None: + # Forward early iterator closure without replacing GeneratorExit. + try: + self.close() + except Exception: + logger.exception("Failed to close interrupted stream") async def __aiter__(self) -> AsyncIterator["GenerateContentResponse"]: + status = None try: async for item in self.__wrapped__: self._response_accumulator.process_chunk(item) @@ -84,18 +95,39 @@ async def __aiter__(self) -> AsyncIterator["GenerateContentResponse"]: description=f"{type(exception).__name__}: {exception}", ) self._with_span.record_exception(exception) - self._finish_tracing(status=status) raise - # completed without exception - status = trace_api.Status( - status_code=trace_api.StatusCode.OK, - ) - self._finish_tracing(status=status) + else: + status = trace_api.Status(status_code=trace_api.StatusCode.OK) + finally: + self._finish_tracing(status=status) + if status is None: + # Cancellation/GeneratorExit must propagate, even if cleanup fails. + try: + await self.aclose() + except Exception: + logger.exception("Failed to close interrupted async stream") + + def close(self) -> Any: + try: + if close := getattr(self.__wrapped__, "close", None): + return close() + finally: + self._finish_tracing() + + async def aclose(self) -> Any: + try: + if aclose := getattr(self.__wrapped__, "aclose", None): + return await aclose() + finally: + self._finish_tracing() def _finish_tracing( self, status: Optional[trace_api.Status] = None, ) -> None: + if self._is_finished: + return + self._is_finished = True response_extractor = _ResponseExtractor(response_accumulator=self._response_accumulator) _finish_tracing( with_span=self._with_span, @@ -117,9 +149,7 @@ def __init__(self) -> None: candidates=_IndexedAccumulator( lambda: _ValuesAccumulator( content=_ValuesAccumulator( - # Parts have no index: retain each streamed part rather - # than merging unrelated text, tool calls, or media. - parts=[], + parts=_PartsAccumulator(), role=_SimpleStringReplace(), ), finish_reason=_SimpleStringReplace(), @@ -222,7 +252,7 @@ def __iter__(self) -> Iterator[Tuple[str, Any]]: elif isinstance(value, _StringAccumulator): if str_value := str(value): yield key, str_value - elif isinstance(value, _IndexedAccumulator): + elif isinstance(value, (_IndexedAccumulator, _PartsAccumulator)): yield key, list(value) elif isinstance(value, _DictReplace): if dict_value := dict(value): @@ -246,7 +276,7 @@ def __iadd__(self, values: Optional[Mapping[str, Any]]) -> "_ValuesAccumulator": elif isinstance(self_value, _SimpleStringReplace): if isinstance(value, str): self_value += value - elif isinstance(self_value, _IndexedAccumulator): + elif isinstance(self_value, (_IndexedAccumulator, _PartsAccumulator)): self_value += value elif isinstance(self_value, _DictReplace): if isinstance(value, Mapping): @@ -312,6 +342,67 @@ def __iadd__( return self +class _PartsAccumulator: + """Coalesce adjacent text; keep calls distinct unless their IDs match.""" + + __slots__ = ("_parts", "_function_calls", "_text_part") + + def __init__(self) -> None: + self._parts: List[_ValuesAccumulator] = [] + self._function_calls: Dict[str, _ValuesAccumulator] = {} + self._text_part: Optional[_ValuesAccumulator] = None + + def __iter__(self) -> Iterator[Dict[str, Any]]: + for part in self._parts: + yield dict(part) + + def __iadd__( + self, values: Optional[Union[Mapping[str, Any], List[Any]]] + ) -> "_PartsAccumulator": + if not values: + return self + if isinstance(values, Mapping): + values = [values] + for value in values: + if not isinstance(value, Mapping) or not value: + continue + part = deepcopy(dict(value)) + # Only plain text fragments can share a part. Media, calls and + # text carrying metadata retain their own boundaries and ordering. + text_only = isinstance(part.get("text"), str) and all( + key == "text" or val is None for key, val in part.items() + ) + if text_only and self._text_part is not None: + self._text_part += part + continue + self._text_part = None + function_call = part.get("function_call") + call_id = function_call.get("id") if isinstance(function_call, Mapping) else None + has_id = isinstance(call_id, str) and bool(call_id) + accumulator = self._function_calls.get(call_id) if has_id else None + if accumulator is None: + # Positions restart in each chunk, and names (or even arguments) + # can be identical for distinct calls. Do not guess their identity. + accumulator = _ValuesAccumulator( + text=_StringAccumulator(), + function_call=_DictReplace(), + ) + self._parts.append(accumulator) + if has_id: + self._function_calls[call_id] = accumulator + else: + # Merge argument fields only after establishing call identity. + previous_args = dict(accumulator).get("function_call", {}).get("args") + updates = {key: val for key, val in function_call.items() if val is not None} + if isinstance(previous_args, Mapping) and isinstance(updates.get("args"), Mapping): + updates["args"] = {**previous_args, **updates["args"]} + part["function_call"] = updates + accumulator += part + if text_only: + self._text_part = accumulator + return self + + class _SimpleStringReplace: __slots__ = ("_str_val",) diff --git a/python/frameworks/google-genai/traceai_google_genai/version.py b/python/frameworks/google-genai/traceai_google_genai/version.py index 97c339acc..596a9bba5 100644 --- a/python/frameworks/google-genai/traceai_google_genai/version.py +++ b/python/frameworks/google-genai/traceai_google_genai/version.py @@ -1 +1 @@ -__version__ = "0.1.3" # x-release-please-version +__version__ = "0.1.4" # x-release-please-version diff --git a/python/frameworks/groq/CHANGELOG.md b/python/frameworks/groq/CHANGELOG.md index 0215883a3..df5fa8f0a 100644 --- a/python/frameworks/groq/CHANGELOG.md +++ b/python/frameworks/groq/CHANGELOG.md @@ -1,5 +1,14 @@ +## [0.1.11] - pending release + +### Changed +- traceai-groq now requires fi-instrumentation-otel 1.0.0 or later. Action: Upgrade fi-instrumentation-otel if your environment pins an earlier version. + +### Fixed +- Groq spans record only known inference parameters, so request headers (including credentials in extra_headers), extra_body and extra_query are no longer recorded in span attributes. +- Messages passed as an iterator reach Groq intact, and traced completion streams keep their with/close behaviour. + ## [0.1.10] - 2026-10-29 ### Fixed diff --git a/python/frameworks/groq/README.md b/python/frameworks/groq/README.md index 170ffd713..48b8370a0 100644 --- a/python/frameworks/groq/README.md +++ b/python/frameworks/groq/README.md @@ -43,6 +43,10 @@ Instrument the Groq client to enable telemetry collection. This step ensures tha ```python from groq import Groq +from groq.types.chat import ChatCompletionToolMessageParam +from traceai_groq import GroqInstrumentor + +GroqInstrumentor().instrument(tracer_provider=trace_provider) def test(): client = Groq( diff --git a/python/frameworks/groq/poetry.lock b/python/frameworks/groq/poetry.lock index a979a1043..c0c58a994 100644 --- a/python/frameworks/groq/poetry.lock +++ b/python/frameworks/groq/poetry.lock @@ -1041,4 +1041,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.1" python-versions = ">=3.10,<3.14" -content-hash = "f99a45c192d10c5245d80126c0775dc0cce2e0e373b304d082c6de7e953e5f29" +content-hash = "d6790374f7369f9771f5c94727b9c2a888ea8767adac2420566016789d0ae3f5" diff --git a/python/frameworks/groq/pyproject.toml b/python/frameworks/groq/pyproject.toml index d8e34bc1a..3b0d4feed 100644 --- a/python/frameworks/groq/pyproject.toml +++ b/python/frameworks/groq/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "traceAI-groq" -version = "0.1.10" +version = "0.1.11" description = "OpenTelemetry instrumentation for Groq" authors = ["Future AGI "] readme = "README.md" @@ -29,7 +29,7 @@ classifiers = [ [tool.poetry.dependencies] python = ">=3.10,<3.14" groq = ">=0.9.0" -fi-instrumentation-otel = ">=0.1.11" +fi-instrumentation-otel = ">=1.0.0" [tool.poetry.urls] diff --git a/python/frameworks/groq/tests/test_request_credentials.py b/python/frameworks/groq/tests/test_request_credentials.py new file mode 100644 index 000000000..d84afa6f1 --- /dev/null +++ b/python/frameworks/groq/tests/test_request_credentials.py @@ -0,0 +1,304 @@ +"""T4-PY-CLOUD-001: request credentials must never become span attributes. + +The wrapped SDK methods are fakes with explicit create() signatures. The real +Groq wrappers, extractors, finalizer and OpenTelemetry in-memory exporter run; +no HTTP client or provider service is used. Requires the package dependencies. +""" + +import asyncio +import json +import unittest +from types import SimpleNamespace + +from fi_instrumentation.fi_types import SpanAttributes +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +from opentelemetry.trace import StatusCode +from traceai_groq._request_attributes_extractor import _RequestAttributesExtractor +from traceai_groq._wrappers import _AsyncCompletionsWrapper, _CompletionsWrapper + + +FAKE_CONSTRUCTOR = "FAKE_CONSTRUCTOR_CREDENTIAL_NOT_REAL" +FAKE_AUTHORIZATION = "Bearer FAKE_PROVIDER_CREDENTIAL_NOT_REAL" +FAKE_HEADER = "FAKE_HEADER_CREDENTIAL_NOT_REAL" +FAKE_QUERY = "FAKE_QUERY_CREDENTIAL_NOT_REAL" +FAKE_BODY = "FAKE_BODY_CREDENTIAL_NOT_REAL" +FAKE_OPAQUE = "FAKE_OPAQUE_CREDENTIAL_NOT_REAL" +FAKE_VALUES = ( + FAKE_CONSTRUCTOR, + FAKE_AUTHORIZATION, + FAKE_HEADER, + FAKE_QUERY, + FAKE_BODY, + FAKE_OPAQUE, +) + + +class _OpaqueConfiguration: + def __init__(self): + self.string_calls = 0 + + def __str__(self): + self.string_calls += 1 + return FAKE_OPAQUE + + __repr__ = __str__ + + +class _FakeCompletions: + def __init__(self, mode): + self.client = SimpleNamespace(api_key=FAKE_CONSTRUCTOR) + self.mode = mode + self.calls = [] + self.error = RuntimeError("FAKE_COMPLETION_FAILURE") + self.response = { + "model": "fake-model", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "hello"}}], + } + self.chunks = [ + { + "model": "fake-model", + "choices": [{"index": 0, "delta": {"role": "assistant", "content": "hello"}}], + }, + {"choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}]}, + ] + + def create( + self, *, model, messages, temperature, top_p, max_completion_tokens, + stop, stream=False, extra_headers=None, extra_query=None, extra_body=None, + timeout=None, + ): + self.calls.append({ + "model": model, "messages": messages, "temperature": temperature, + "top_p": top_p, "max_completion_tokens": max_completion_tokens, + "stop": stop, "stream": stream, "extra_headers": extra_headers, + "extra_query": extra_query, "extra_body": extra_body, "timeout": timeout, + }) + if self.mode == "error": + raise self.error + if stream: + return self._stream() + return self.response + + def _stream(self): + yield self.chunks[0] + if self.mode == "stream_error": + raise self.error + yield self.chunks[1] + + +class _FakeAsyncCompletions(_FakeCompletions): + async def create( + self, *, model, messages, temperature, top_p, max_completion_tokens, + stop, stream=False, extra_headers=None, extra_query=None, extra_body=None, + timeout=None, + ): + response = super().create( + model=model, messages=messages, temperature=temperature, top_p=top_p, + max_completion_tokens=max_completion_tokens, stop=stop, stream=stream, + extra_headers=extra_headers, extra_query=extra_query, extra_body=extra_body, + timeout=timeout, + ) + if stream: + async def chunks(): + for chunk in response: + yield chunk + return chunks() + return response + + +def _request(stream=False): + # No api_key argument on create(): it belongs to client construction only. + return { + "model": "fake-model", + "messages": [{"role": "user", "content": "hello"}], + "temperature": 0.25, + "top_p": 0.9, + "max_completion_tokens": 64, + "stop": ["END"], + "stream": stream, + "extra_headers": {"Authorization": FAKE_AUTHORIZATION, "X-Api-Key": FAKE_HEADER}, + "extra_query": {"api_key": FAKE_QUERY}, + "extra_body": {"config": {"authentication": {"token": FAKE_BODY}}}, + "timeout": 3.0, + } + + +def _expected_parameters(stream=False): + return { + "model": "fake-model", "temperature": 0.25, "top_p": 0.9, + "max_completion_tokens": 64, "stop": ["END"], "stream": stream, + } + + +class RequestCredentialTests(unittest.TestCase): + def setUp(self): + self.exporter = InMemorySpanExporter() + self.provider = TracerProvider() + self.provider.add_span_processor(SimpleSpanProcessor(self.exporter)) + self.tracer = self.provider.get_tracer("groq-request-credential-regression") + self.addCleanup(self.provider.shutdown) + + def assert_no_credentials(self, value): + encoded = json.dumps(value, default=str) + for credential in FAKE_VALUES: + self.assertNotIn(credential, encoded) + + def assert_exported_request(self, request, error=False): + spans = self.exporter.get_finished_spans() + self.assertEqual(len(spans), 1) + span = spans[0] + self.assertIsNotNone(span.end_time) + attributes = dict(span.attributes) + self.assert_no_credentials(attributes) + parameters = json.loads(attributes[SpanAttributes.GEN_AI_REQUEST_PARAMETERS]) + self.assertEqual(parameters, _expected_parameters(request["stream"])) + self.assertEqual(json.loads(attributes[SpanAttributes.INPUT_VALUE]), request["messages"]) + self.assertEqual(attributes[SpanAttributes.GEN_AI_REQUEST_MODEL], "fake-model") + self.assertEqual(span.status.status_code, StatusCode.ERROR if error else StatusCode.OK) + for event in span.events: + self.assert_no_credentials(dict(event.attributes)) + if not error: + self.assertEqual(attributes[SpanAttributes.OUTPUT_VALUE], "hello") + + def assert_forwarded_unchanged(self, sdk, request): + self.assertEqual(len(sdk.calls), 1) + self.assertEqual(sdk.calls[0], request) + for key in ("messages", "extra_headers", "extra_query", "extra_body", "stop"): + self.assertIs(sdk.calls[0][key], request[key]) + self.assertEqual(sdk.client.api_key, FAKE_CONSTRUCTOR) + self.assertEqual(request["extra_headers"]["Authorization"], FAKE_AUTHORIZATION) + self.assertEqual(request["extra_headers"]["X-Api-Key"], FAKE_HEADER) + self.assertEqual(request["extra_body"]["config"]["authentication"]["token"], FAKE_BODY) + + def test_sync_finished_spans_never_export_transport_credentials(self): + for mode in ("normal", "error", "stream", "stream_error"): + with self.subTest(mode=mode): + self.exporter.clear() + request = _request(stream=mode.startswith("stream")) + sdk = _FakeCompletions(mode) + wrapper = _CompletionsWrapper(tracer=self.tracer) + if "error" in mode: + with self.assertRaises(RuntimeError) as raised: + result = wrapper(sdk.create, sdk, (), request) + if request["stream"]: + list(result) + self.assertIs(raised.exception, sdk.error) + else: + result = wrapper(sdk.create, sdk, (), request) + if request["stream"]: + self.assertEqual(list(result), sdk.chunks) + else: + self.assertIs(result, sdk.response) + self.assert_forwarded_unchanged(sdk, request) + self.assert_exported_request(request, error="error" in mode) + + def test_async_finished_spans_never_export_transport_credentials(self): + async def exercise(mode): + request = _request(stream=mode.startswith("stream")) + sdk = _FakeAsyncCompletions(mode) + wrapper = _AsyncCompletionsWrapper(tracer=self.tracer) + if "error" in mode: + with self.assertRaises(RuntimeError) as raised: + result = await wrapper(sdk.create, sdk, (), request) + if request["stream"]: + [chunk async for chunk in result] + self.assertIs(raised.exception, sdk.error) + else: + result = await wrapper(sdk.create, sdk, (), request) + if request["stream"]: + self.assertEqual([chunk async for chunk in result], sdk.chunks) + else: + self.assertIs(result, sdk.response) + self.assert_forwarded_unchanged(sdk, request) + self.assert_exported_request(request, error="error" in mode) + + for mode in ("normal", "error", "stream", "stream_error"): + with self.subTest(mode=mode): + self.exporter.clear() + asyncio.run(exercise(mode)) + + def test_every_input_yield_is_safe_without_relying_on_overwrite(self): + extractor = _RequestAttributesExtractor() + for message_mode in ("normal", "empty", "none", "absent"): + with self.subTest(messages=message_mode): + request = _request() + if message_mode == "empty": + request["messages"] = [] + elif message_mode == "none": + request["messages"] = None + elif message_mode == "absent": + del request["messages"] + attributes = list(extractor.get_attributes_from_request(request)) + # Check every emitted pair, not only a dict that drops earlier values. + self.assert_no_credentials(attributes) + final_input = json.loads(dict(attributes)[SpanAttributes.INPUT_VALUE]) + expected = request.get("messages") + if expected is None: + expected = _expected_parameters() + self.assertEqual(final_input, expected) + + def test_only_builtin_inference_values_are_serialized(self): + extractor = _RequestAttributesExtractor() + opaque = _OpaqueConfiguration() + + class CustomString(str): + pass + + request = _request() + del request["messages"] + request.update({ + "model": opaque, "temperature": {"api_key": FAKE_BODY}, + "stop": ["END", {"token": FAKE_BODY}], + "reasoning_effort": CustomString(FAKE_OPAQUE), + "response_format": {"config": {"token": FAKE_BODY}}, + "client": opaque, "provider_options": {"client": opaque}, + "extra_headers": {"Authorization": opaque}, + }) + attributes = list(extractor.get_attributes_from_request(request)) + attributes.extend(extractor.get_extra_attributes_from_request(request)) + self.assert_no_credentials(attributes) + self.assertEqual(opaque.string_calls, 0) + final = dict(attributes) + expected = {"top_p": 0.9, "max_completion_tokens": 64, "stream": False} + self.assertEqual(json.loads(final[SpanAttributes.GEN_AI_REQUEST_PARAMETERS]), expected) + self.assertEqual(json.loads(final[SpanAttributes.INPUT_VALUE]), expected) + self.assertNotIn(SpanAttributes.GEN_AI_REQUEST_MODEL, final) + + def test_opaque_transport_configuration_is_not_stringified_by_argument_binding(self): + for asynchronous in (False, True): + with self.subTest(asynchronous=asynchronous): + self.exporter.clear() + opaque = _OpaqueConfiguration() + request = _request() + request["extra_body"]["client"] = opaque + if asynchronous: + sdk = _FakeAsyncCompletions("normal") + wrapper = _AsyncCompletionsWrapper(tracer=self.tracer) + result = asyncio.run(wrapper(sdk.create, sdk, (), request)) + else: + sdk = _FakeCompletions("normal") + wrapper = _CompletionsWrapper(tracer=self.tracer) + result = wrapper(sdk.create, sdk, (), request) + self.assertIs(result, sdk.response) + self.assertEqual(opaque.string_calls, 0) + self.assert_forwarded_unchanged(sdk, request) + self.assert_exported_request(request) + + def test_tool_definitions_remain_separate_from_invocation_parameters(self): + request = _request() + tool = {"type": "function", "function": {"name": "fake_tool", "parameters": {"type": "object"}}} + request["tools"] = [tool] + attributes = dict(_RequestAttributesExtractor().get_extra_attributes_from_request(request)) + self.assert_no_credentials(attributes) + self.assertEqual( + json.loads(attributes[SpanAttributes.GEN_AI_REQUEST_PARAMETERS]), + _expected_parameters(), + ) + schemas = [value for key, value in attributes.items() if key.startswith(SpanAttributes.GEN_AI_TOOL_DEFINITIONS)] + self.assertEqual([json.loads(value) for value in schemas], [tool]) + + +if __name__ == "__main__": + unittest.main() diff --git a/python/frameworks/groq/tests/test_transport_contract.py b/python/frameworks/groq/tests/test_transport_contract.py new file mode 100644 index 000000000..5c554180f --- /dev/null +++ b/python/frameworks/groq/tests/test_transport_contract.py @@ -0,0 +1,251 @@ +"""Public SDK contracts, using the real client and only an in-memory HTTP transport.""" + +import asyncio +import json +import socket + +import groq +import httpx +import pytest +from fi_instrumentation import TraceConfig +from fi_instrumentation.fi_types import SpanAttributes +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +from opentelemetry.trace import StatusCode +from traceai_groq import GroqInstrumentor + + +MESSAGES = [{"role": "user", "content": "private-request-text"}] +OUTPUT = "private-response-text" +SYNTHETIC_KEY = "synthetic-not-a-credential" +COMPLETION = { + "id": "synthetic", + "object": "chat.completion", + "created": 1, + "model": "synthetic-model", + "choices": [{"index": 0, "finish_reason": "stop", "message": { + "role": "assistant", "content": OUTPUT, + }}], +} +CHUNK = { + "id": "synthetic", + "object": "chat.completion.chunk", + "created": 1, + "model": "synthetic-model", + "choices": [{"index": 0, "finish_reason": None, "delta": { + "role": "assistant", "content": OUTPUT, + }}], +} +SSE = ("data: " + json.dumps(CHUNK) + "\n\n").encode() + + +@pytest.fixture(autouse=True) +def no_network(monkeypatch): + def denied(*args, **kwargs): + raise AssertionError("Only MockTransport is permitted") + + monkeypatch.setattr(socket.socket, "connect", denied) + monkeypatch.setattr(socket, "getaddrinfo", denied) + + +@pytest.fixture(params=["uninstrumented", "default", "hidden"]) +def telemetry(request): + exporter = InMemorySpanExporter() + provider = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + mode = request.param + instrumentor = GroqInstrumentor() + if mode != "uninstrumented": + instrumentor.instrument( + tracer_provider=provider, + config=TraceConfig(hide_inputs=mode == "hidden", hide_outputs=mode == "hidden"), + ) + try: + yield mode, exporter + finally: + if mode != "uninstrumented": + instrumentor.uninstrument() + provider.shutdown() + + +def assert_spans(telemetry, status=None): + mode, exporter = telemetry + spans = exporter.get_finished_spans() + if mode == "uninstrumented": + assert not spans + return + assert len(spans) == 1 + assert spans[0].end_time is not None + if status is not None: + assert spans[0].status.status_code == status + encoded = json.dumps(dict(spans[0].attributes), default=str) + assert SYNTHETIC_KEY not in encoded + if mode == "hidden": + assert MESSAGES[0]["content"] not in encoded + assert OUTPUT not in encoded + + +@pytest.mark.parametrize("asynchronous", [False, True], ids=["sync", "async"]) +@pytest.mark.parametrize("kind", ["list", "iterator", "generator"]) +def test_messages_reach_transport(telemetry, asynchronous, kind): + wire = [] + + def handle(request): + wire.append(json.loads(request.content)) + return httpx.Response(200, json=COMPLETION) + + messages = list(MESSAGES) + if kind == "iterator": + messages = iter(messages) + elif kind == "generator": + messages = (message for message in messages) + kwargs = {"model": "synthetic-model", "messages": messages} + if asynchronous: + async def run(): + async with groq.AsyncGroq( + api_key=SYNTHETIC_KEY, max_retries=0, + http_client=httpx.AsyncClient(transport=httpx.MockTransport(handle)), + ) as client: + return await client.chat.completions.create(**kwargs) + response = asyncio.run(run()) + else: + with groq.Groq( + api_key=SYNTHETIC_KEY, max_retries=0, + http_client=httpx.Client(transport=httpx.MockTransport(handle)), + ) as client: + response = client.chat.completions.create(**kwargs) + assert wire[0]["messages"] == MESSAGES + assert response.choices[0].message.content == OUTPUT + assert_spans(telemetry, StatusCode.OK) + mode, exporter = telemetry + if kind == "list" and mode == "default": + assert json.loads(exporter.get_finished_spans()[0].attributes[SpanAttributes.INPUT_VALUE]) == MESSAGES + + +class SyncBody(httpx.SyncByteStream): + def __init__(self): + self.pulls = 0 + self.closes = 0 + + def __iter__(self): + self.pulls += 1 + yield SSE + self.pulls += 1 + yield b"data: [DONE]\n\n" + + def close(self): + self.closes += 1 + + +class AsyncBody(httpx.AsyncByteStream): + def __init__(self): + self.pulls = 0 + self.closes = 0 + + async def __aiter__(self): + self.pulls += 1 + yield SSE + self.pulls += 1 + yield b"data: [DONE]\n\n" + + async def aclose(self): + self.closes += 1 + + +class ConsumerError(Exception): + pass + + +ACTIONS = ["close_empty", "close_one", "context_empty", "context_one", "context_error", "exhaust"] + + +@pytest.mark.parametrize("action", ACTIONS) +def test_sync_stream_lifecycle(telemetry, action): + body = SyncBody() + response = httpx.Response(200, headers={"content-type": "text/event-stream"}, stream=body) + with groq.Groq( + api_key=SYNTHETIC_KEY, max_retries=0, + http_client=httpx.Client(transport=httpx.MockTransport(lambda request: response)), + ) as client: + stream = client.chat.completions.create(model="synthetic-model", messages=MESSAGES, stream=True) + try: + assert body.pulls == 0 + assert not telemetry[1].get_finished_spans() + if action.startswith("context"): + error = ConsumerError("synthetic consumer failure") + try: + with stream as opened: + assert opened is stream + if action != "context_empty": + assert next(iter(opened)).choices[0].delta.content == OUTPUT + if action == "context_error": + raise error + except ConsumerError as caught: + assert caught is error + elif action == "exhaust": + assert [item.choices[0].delta.content for item in stream] == [OUTPUT] + else: + if action == "close_one": + assert next(iter(stream)).choices[0].delta.content == OUTPUT + stream.close() + assert response.is_closed + assert body.pulls == (2 if action == "exhaust" else 0 if action.endswith("empty") else 1) + assert body.closes == 1 + assert_spans(telemetry) + stream.close() + assert_spans(telemetry) + assert body.closes == 1 + finally: + stream.close() + response.close() + + +@pytest.mark.parametrize("action", ACTIONS) +def test_async_stream_lifecycle(telemetry, action): + async def run(): + body = AsyncBody() + response = httpx.Response(200, headers={"content-type": "text/event-stream"}, stream=body) + async with groq.AsyncGroq( + api_key=SYNTHETIC_KEY, max_retries=0, + http_client=httpx.AsyncClient(transport=httpx.MockTransport(lambda request: response)), + ) as client: + stream = await client.chat.completions.create(model="synthetic-model", messages=MESSAGES, stream=True) + try: + assert body.pulls == 0 + assert not telemetry[1].get_finished_spans() + if action.startswith("context"): + error = ConsumerError("synthetic consumer failure") + try: + async with stream as opened: + assert opened is stream + if action != "context_empty": + assert (await opened.__aiter__().__anext__()).choices[0].delta.content == OUTPUT + if action == "context_error": + raise error + except ConsumerError as caught: + assert caught is error + elif action == "exhaust": + assert [item.choices[0].delta.content async for item in stream] == [OUTPUT] + else: + if action == "close_one": + assert (await stream.__aiter__().__anext__()).choices[0].delta.content == OUTPUT + await stream.close() + assert response.is_closed + assert body.pulls == (2 if action == "exhaust" else 0 if action.endswith("empty") else 1) + assert body.closes == 1 + assert_spans(telemetry) + # The old wrapper exposed aclose(); retain that compatibility too. + if telemetry[0] != "uninstrumented": + await stream.aclose() + else: + await stream.close() + assert_spans(telemetry) + assert body.closes == 1 + finally: + if hasattr(stream, "close"): + await stream.close() + else: + await stream.aclose() + await response.aclose() + asyncio.run(run()) diff --git a/python/frameworks/groq/traceai_groq/_request_attributes_extractor.py b/python/frameworks/groq/traceai_groq/_request_attributes_extractor.py index e97b99e2e..29e6311d3 100644 --- a/python/frameworks/groq/traceai_groq/_request_attributes_extractor.py +++ b/python/frameworks/groq/traceai_groq/_request_attributes_extractor.py @@ -1,6 +1,6 @@ import logging from enum import Enum -from typing import Any, Iterable, Iterator, Mapping, Tuple, TypeVar +from typing import Any, Dict, Iterable, Iterator, Mapping, Tuple, TypeVar from fi_instrumentation import safe_json_dumps from fi_instrumentation.fi_types import ( @@ -24,6 +24,48 @@ logger.addHandler(logging.NullHandler()) +# Request arguments also include credentials, headers and transport options. +# Match the completion allowlist policy: only known, typed inference controls +# belong in request-parameter telemetry, never arbitrary provider configuration. +_INFERENCE_PARAMETER_TYPES = { + "model": (str,), + "temperature": (int, float), + "top_p": (int, float), + "n": (int,), + "max_tokens": (int,), + "max_completion_tokens": (int,), + "frequency_penalty": (int, float), + "presence_penalty": (int, float), + "seed": (int,), + "logprobs": (bool,), + "top_logprobs": (int,), + "stream": (bool,), + "parallel_tool_calls": (bool,), + "reasoning_effort": (str,), + "stop": (str,), +} + + +def _get_inference_parameters(request_parameters: Mapping[str, Any]) -> Dict[str, Any]: + """Project telemetry without inspecting or mutating transport/auth objects. + + Exact builtin types exclude clients, nested configuration and subclasses + with custom serialization. Stop sequences are the only allowed container; + headers, extra_body, extra_query and future SDK options are omitted by default. + """ + parameters: Dict[str, Any] = {} + for name, allowed_types in _INFERENCE_PARAMETER_TYPES.items(): + if name in request_parameters: + value = request_parameters[name] + if value is None or type(value) in allowed_types: + parameters[name] = value + + stop = request_parameters.get("stop") + if type(stop) in (list, tuple) and all(type(item) is str for item in stop): + parameters["stop"] = list(stop) + return parameters + + class _RequestAttributesExtractor: __slots__ = () @@ -32,13 +74,13 @@ def get_attributes_from_request( request_parameters: Mapping[str, Any], ) -> Iterator[Tuple[str, AttributeValue]]: yield SpanAttributes.GEN_AI_SPAN_KIND, FiSpanKindValues.LLM.value - if model := request_parameters.get("model"): + inference_parameters = _get_inference_parameters(request_parameters) + if model := inference_parameters.get("model"): yield SpanAttributes.GEN_AI_REQUEST_MODEL, model try: - yield SpanAttributes.INPUT_VALUE, safe_json_dumps(request_parameters) - + # Never yield a raw request, even temporarily before message input. messages = request_parameters.get("messages") - input_data = messages if messages is not None else request_parameters + input_data = messages if messages is not None else inference_parameters if messages and isinstance(messages, Iterable): yield from _extract_eval_input(messages) @@ -59,11 +101,9 @@ def get_extra_attributes_from_request( ) -> Iterator[Tuple[str, AttributeValue]]: if not isinstance(request_parameters, Mapping): return - invocation_params = dict(request_parameters) - invocation_params.pop("messages", None) # Remove LLM input messages - invocation_params.pop("functions", None) + invocation_params = _get_inference_parameters(request_parameters) - if isinstance((tools := invocation_params.pop("tools", None)), Iterable): + if isinstance((tools := request_parameters.get("tools")), Iterable): for i, tool in enumerate(tools): yield f"{SpanAttributes.GEN_AI_TOOL_DEFINITIONS}.{i}.{ToolAttributes.TOOL_JSON_SCHEMA}", safe_json_dumps( tool diff --git a/python/frameworks/groq/traceai_groq/_wrappers.py b/python/frameworks/groq/traceai_groq/_wrappers.py index d55786186..a239eee46 100644 --- a/python/frameworks/groq/traceai_groq/_wrappers.py +++ b/python/frameworks/groq/traceai_groq/_wrappers.py @@ -1,4 +1,3 @@ -import json import logging from abc import ABC from contextlib import contextmanager @@ -7,7 +6,7 @@ from typing import Any, Callable, Dict, Iterable, Iterator, List, Mapping, Tuple import opentelemetry.context as context_api -from fi_instrumentation import get_attributes_from_context, safe_json_dumps +from fi_instrumentation import get_attributes_from_context from fi_instrumentation.fi_types import ( EmbeddingAttributes, FiSpanKindValues, @@ -15,6 +14,7 @@ SpanAttributes, ) from groq import NOT_GIVEN +from wrapt import ObjectProxy from opentelemetry import trace as trace_api from opentelemetry.trace import INVALID_SPAN from opentelemetry.util.types import AttributeValue @@ -93,20 +93,174 @@ def _parse_args( ) -> Dict[str, Any]: bound_signature = signature.bind(*args, **kwargs) bound_signature.apply_defaults() - bound_arguments = bound_signature.arguments # Defaults empty to NOT_GIVEN - request_data: Dict[str, Any] = {} - for key, value in bound_arguments.items(): + # Binding must not serialize opaque clients or transport/auth configuration. + # The extractor projects safe telemetry; the SDK still receives the original + # args and kwargs unchanged. + return { + key: value + for key, value in bound_signature.arguments.items() + if value is not None and value is not NOT_GIVEN + # This is a telemetry-only view. Never drain caller-owned messages. + and not (key == "messages" and isinstance(value, Iterator)) + } + + +class _Stream(ObjectProxy): + """Keep SDK lifecycle APIs while owning one invocation's tracing state.""" + + def __init__(self, response, span, extractor, request_parameters): + super().__init__(response) + self._self_span = span + self._self_extractor = extractor + self._self_request_parameters = request_parameters + self._self_chunks = [] + self._self_iterator = None + self._self_finished = False + + def __del__(self): + # Finalize abandoned tracing without reading or driving the transport. + # The SDK still owns its explicit (possibly asynchronous) close API. try: - if value is not None and value is not NOT_GIVEN: - try: - # ensure the value is JSON-serializable - safe_json_dumps(value) - request_data[key] = value - except json.JSONDecodeError: - request_data[key] = str(value) + self._finish() except Exception: - request_data[key] = str(value) - return request_data + pass + + def _finish(self, error=None, complete=False): + if self._self_finished: + return + self._self_finished = True + span = self._self_span + try: + if error is not None: + span.record_exception(error) + span.finish_tracing(status=trace_api.Status( + status_code=trace_api.StatusCode.ERROR, + description=f"{type(error).__name__}: {error}", + )) + elif complete: + _finish_tracing( + status=trace_api.Status(status_code=trace_api.StatusCode.OK), + with_span=span, + attributes=self._self_extractor.get_attributes( + response=self._self_chunks, is_streaming=True, + ), + extra_attributes=self._self_extractor.get_extra_attributes( + response=self._self_chunks, + request_parameters=self._self_request_parameters, + is_streaming=True, + ), + ) + else: + # Early close is not a completed response. Do not invent output + # or an OK status for a stream the application did not exhaust. + span.finish_tracing() + except Exception: + logger.exception("Failed to finalize streaming response") + span.finish_tracing() + finally: + self._self_chunks.clear() + + +class _SyncStream(_Stream): + def __iter__(self): + return self + + def __next__(self): + try: + if self._self_iterator is None: + self._self_iterator = iter(self.__wrapped__) + item = next(self._self_iterator) + except StopIteration: + self._finish(complete=True) + raise + except BaseException as error: + self._finish(error=error) + raise + if not self._self_finished: + self._self_chunks.append(item) + return item + + def close(self): + try: + if close := getattr(self.__wrapped__, "close", None): + return close() + except BaseException as error: + self._finish(error=error) + raise + finally: + self._finish() + + def __enter__(self): + try: + self.__wrapped__.__enter__() + except BaseException as error: + self._finish(error=error) + raise + return self + + def __exit__(self, exc_type, exc, traceback): + try: + return self.__wrapped__.__exit__(exc_type, exc, traceback) + except BaseException as error: + self._finish(error=error) + raise + finally: + # An exception in the consumer's context body is not an SDK error. + self._finish() + + +class _AsyncStream(_Stream): + def __aiter__(self): + return self + + async def __anext__(self): + try: + if self._self_iterator is None: + self._self_iterator = self.__wrapped__.__aiter__() + item = await self._self_iterator.__anext__() + except StopAsyncIteration: + self._finish(complete=True) + raise + except BaseException as error: + self._finish(error=error) + raise + if not self._self_finished: + self._self_chunks.append(item) + return item + + async def close(self): + try: + close = getattr(self.__wrapped__, "close", None) + if close is None: + close = getattr(self.__wrapped__, "aclose", None) + if close is not None: + return await close() + except BaseException as error: + self._finish(error=error) + raise + finally: + self._finish() + + async def aclose(self): + return await self.close() + + async def __aenter__(self): + try: + await self.__wrapped__.__aenter__() + except BaseException as error: + self._finish(error=error) + raise + return self + + async def __aexit__(self, exc_type, exc, traceback): + try: + return await self.__wrapped__.__aexit__(exc_type, exc, traceback) + except BaseException as error: + self._finish(error=error) + raise + finally: + # An exception in the consumer's context body is not an SDK error. + self._finish() class _CompletionsWrapper(_WithTracer): @@ -130,12 +284,6 @@ def __call__( if context_api.get_value(context_api._SUPPRESS_INSTRUMENTATION_KEY): return wrapped(*args, **kwargs) - # Prepare invocation parameters by merging args and kwargs - invocation_parameters = {} - for arg in args: - if arg and isinstance(arg, dict): - invocation_parameters.update(arg) - invocation_parameters.update(kwargs) request_parameters = _parse_args(signature(wrapped), *args, **kwargs) span_name = "Completions" @@ -162,48 +310,9 @@ def __call__( raise if streaming and isinstance(response, Iterable): - # If streaming, wrap the iterator without consuming it - def streaming_response_wrapper(): - chunks = [] # State belongs to this invocation, not the installed hook. - try: - for item in response: - chunks.append(item) - yield item - - except Exception as exception: - span.record_exception(exception) - status = trace_api.Status( - status_code=trace_api.StatusCode.ERROR, - description=f"{type(exception).__name__}: {exception}", - ) - span.finish_tracing(status=status) - raise - else: - try: - _finish_tracing( - status=trace_api.Status( - status_code=trace_api.StatusCode.OK - ), - with_span=span, - attributes=self._response_extractor.get_attributes( - response=chunks, is_streaming=streaming - ), - extra_attributes=self._response_extractor.get_extra_attributes( - response=chunks, - request_parameters=request_parameters, - is_streaming=streaming, - ), - ) - except Exception: - logger.exception( - f"Failed to finalize streaming response of type {type(response)}" - ) - span.finish_tracing() - finally: - if span._span.is_recording(): - span.finish_tracing() - - return streaming_response_wrapper() + return _SyncStream( + response, span, self._response_extractor, request_parameters, + ) else: # Non-streaming response try: @@ -248,12 +357,6 @@ async def __call__( if context_api.get_value(context_api._SUPPRESS_INSTRUMENTATION_KEY): return await wrapped(*args, **kwargs) - # Prepare invocation parameters by merging args and kwargs - invocation_parameters = {} - for arg in args: - if arg and isinstance(arg, dict): - invocation_parameters.update(arg) - invocation_parameters.update(kwargs) request_parameters = _parse_args(signature(wrapped), *args, **kwargs) span_name = "AsyncCompletions" @@ -280,48 +383,9 @@ async def __call__( raise if streaming and hasattr(response, "__aiter__"): - # If streaming, wrap the async iterator without consuming it - async def streaming_response_wrapper(): - chunks = [] # Concurrent streams must never share response data. - try: - async for item in response: - chunks.append(item) - yield item - - except Exception as exception: - span.record_exception(exception) - status = trace_api.Status( - status_code=trace_api.StatusCode.ERROR, - description=f"{type(exception).__name__}: {exception}", - ) - span.finish_tracing(status=status) - raise - else: - try: - _finish_tracing( - status=trace_api.Status( - status_code=trace_api.StatusCode.OK - ), - with_span=span, - attributes=self._response_extractor.get_attributes( - response=chunks, is_streaming=streaming - ), - extra_attributes=self._response_extractor.get_extra_attributes( - response=chunks, - request_parameters=request_parameters, - is_streaming=streaming, - ), - ) - except Exception: - logger.exception( - f"Failed to finalize streaming response of type {type(response)}" - ) - span.finish_tracing() - finally: - if span._span.is_recording(): - span.finish_tracing() - - return streaming_response_wrapper() + return _AsyncStream( + response, span, self._response_extractor, request_parameters, + ) else: # Non-streaming response try: diff --git a/python/frameworks/groq/traceai_groq/version.py b/python/frameworks/groq/traceai_groq/version.py index b2b8e3d08..8c39d1009 100644 --- a/python/frameworks/groq/traceai_groq/version.py +++ b/python/frameworks/groq/traceai_groq/version.py @@ -1 +1 @@ -__version__ = "0.1.10" # x-release-please-version +__version__ = "0.1.11" # x-release-please-version diff --git a/python/frameworks/huggingface/CHANGELOG.md b/python/frameworks/huggingface/CHANGELOG.md new file mode 100644 index 000000000..04dd270b5 --- /dev/null +++ b/python/frameworks/huggingface/CHANGELOG.md @@ -0,0 +1,5 @@ +## [0.1.1] - pending release + +### Changed +- traceai-huggingface now requires fi-instrumentation-otel 1.0.0 or later. Action: Upgrade fi-instrumentation-otel if your environment pins an earlier version. + diff --git a/python/frameworks/huggingface/pyproject.toml b/python/frameworks/huggingface/pyproject.toml index d03dac494..fd907e14a 100644 --- a/python/frameworks/huggingface/pyproject.toml +++ b/python/frameworks/huggingface/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "traceAI-huggingface" -version = "0.1.0" +version = "0.1.1" description = "OpenTelemetry instrumentation for HuggingFace Inference API - text generation, chat, and embeddings" authors = ["Future AGI "] readme = "README.md" @@ -28,7 +28,7 @@ classifiers = [ [tool.poetry.dependencies] python = ">3.9,<3.14" -fi-instrumentation-otel = ">=0.1.11" +fi-instrumentation-otel = ">=1.0.0" huggingface-hub = ">=0.20.0" [tool.poetry.urls] diff --git a/python/frameworks/huggingface/traceai_huggingface/version.py b/python/frameworks/huggingface/traceai_huggingface/version.py index 75716eb9a..34d76ee9e 100644 --- a/python/frameworks/huggingface/traceai_huggingface/version.py +++ b/python/frameworks/huggingface/traceai_huggingface/version.py @@ -1 +1 @@ -__version__ = "0.1.0" # x-release-please-version +__version__ = "0.1.1" # x-release-please-version diff --git a/python/frameworks/instructor/CHANGELOG.md b/python/frameworks/instructor/CHANGELOG.md index 5530f6b3d..332843a67 100644 --- a/python/frameworks/instructor/CHANGELOG.md +++ b/python/frameworks/instructor/CHANGELOG.md @@ -1,5 +1,10 @@ +## [0.2.2] - pending release + +### Changed +- traceai-instructor now requires fi-instrumentation-otel 1.0.0 or later. Action: Upgrade fi-instrumentation-otel if your environment pins an earlier version. + ## [0.2.1] - 2026-10-29 ### Fixed diff --git a/python/frameworks/instructor/poetry.lock b/python/frameworks/instructor/poetry.lock index 6c272361e..401ffb3ce 100644 --- a/python/frameworks/instructor/poetry.lock +++ b/python/frameworks/instructor/poetry.lock @@ -2082,4 +2082,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.1" python-versions = ">3.9, <3.14" -content-hash = "998ad1ae2bc71595db3e3d18d35745277d6e18502d0d4898263b3f7d7df06196" +content-hash = "4651ecaf98d834f9eb96d381bf693f5882d46047861bbe2d63ef8ce5406d42c3" diff --git a/python/frameworks/instructor/pyproject.toml b/python/frameworks/instructor/pyproject.toml index a26260039..d9b35f21b 100644 --- a/python/frameworks/instructor/pyproject.toml +++ b/python/frameworks/instructor/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "traceAI-instructor" -version = "0.2.1" +version = "0.2.2" description = "OpenTelemetry instrumentation for Instructor" authors = ["Future AGI "] readme = "README.md" @@ -29,7 +29,7 @@ classifiers = [ [tool.poetry.dependencies] python = ">3.9, <3.14" -fi-instrumentation-otel = ">=0.1.11" +fi-instrumentation-otel = ">=1.0.0" instructor = ">=1.0.0" diff --git a/python/frameworks/instructor/traceai_instructor/version.py b/python/frameworks/instructor/traceai_instructor/version.py index 130276d81..210c13fdd 100644 --- a/python/frameworks/instructor/traceai_instructor/version.py +++ b/python/frameworks/instructor/traceai_instructor/version.py @@ -1 +1 @@ -__version__ = "0.2.1" # x-release-please-version +__version__ = "0.2.2" # x-release-please-version diff --git a/python/frameworks/langchain/CHANGELOG.md b/python/frameworks/langchain/CHANGELOG.md index 862169e51..8cd98e357 100644 --- a/python/frameworks/langchain/CHANGELOG.md +++ b/python/frameworks/langchain/CHANGELOG.md @@ -1,3 +1,8 @@ +## [0.2.3] - pending release + +### Fixed +- Chat spans keep the conversation history and normalized tool calls, with their IDs when supplied. For image URL parts in chat input, a URL given as a plain string no longer stops input export, inline base64 image data is redacted, and input hiding leaves their image URLs out. + ## [0.2.2] - 2026-11-10 ### Fixed - Restore the optional Protect hook when `LangChainInstrumentor.uninstrument()` runs, so subsequent calls and re-instrumentation do not retain a stale wrapper (#393). diff --git a/python/frameworks/langchain/pyproject.toml b/python/frameworks/langchain/pyproject.toml index 1289a1583..57af3f78b 100644 --- a/python/frameworks/langchain/pyproject.toml +++ b/python/frameworks/langchain/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "traceAI-langchain" -version = "0.2.2" +version = "0.2.3" description = "OpenTelemetry instrumentation for LangChain" authors = ["Future AGI "] readme = "README.md" diff --git a/python/frameworks/langchain/tests/test_canonical_input.py b/python/frameworks/langchain/tests/test_canonical_input.py new file mode 100644 index 000000000..fd66c7e9d --- /dev/null +++ b/python/frameworks/langchain/tests/test_canonical_input.py @@ -0,0 +1,177 @@ +"""canonical input must retain the full chat conversation.""" + +from __future__ import annotations + +import json +from copy import deepcopy + +import pytest +from fi_instrumentation import REDACTED_VALUE, TraceConfig +from fi_instrumentation.fi_types import MessageAttributes, SpanAttributes +from langchain_core.language_models.fake_chat_models import FakeMessagesListChatModel +from langchain_core.messages import AIMessage, HumanMessage, SystemMessage +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter + + +@pytest.fixture +def exporter(request): + from traceai_langchain import LangChainInstrumentor + + exporter = InMemorySpanExporter() + provider = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + instrumentor = LangChainInstrumentor() + instrumentor.instrument( + tracer_provider=provider, config=TraceConfig(**getattr(request, "param", {})) + ) + try: + yield exporter + finally: + try: + instrumentor.uninstrument() + finally: + provider.shutdown() + + +@pytest.mark.parametrize( + "messages", + [ + pytest.param( + [HumanMessage(content="CURRENT_USER_QUESTION_MARKER")], + id="single-user", + ), + pytest.param( + [ + SystemMessage(content="SYSTEM_INSTRUCTION_MARKER"), + HumanMessage(content="CURRENT_USER_QUESTION_MARKER"), + ], + id="system-first", + ), + pytest.param( + [ + HumanMessage(content="PREVIOUS_USER_QUESTION_MARKER"), + AIMessage(content="PREVIOUS_ASSISTANT_ANSWER_MARKER"), + HumanMessage(content="CURRENT_USER_QUESTION_MARKER"), + ], + id="history-first", + ), + pytest.param( + [ + SystemMessage(content="SYSTEM_INSTRUCTION_MARKER"), + HumanMessage(content="PREVIOUS_USER_QUESTION_MARKER"), + AIMessage(content="PREVIOUS_ASSISTANT_ANSWER_MARKER"), + HumanMessage(content="CURRENT_USER_QUESTION_MARKER"), + ], + id="system-and-history", + ), + ], +) +def test_canonical_input_preserves_conversation(exporter, messages): + original_messages = deepcopy(messages) + model = FakeMessagesListChatModel( + responses=[AIMessage(content="offline answer")], cache=False + ) + + response = model.invoke(messages) + + assert response.content == "offline answer" + assert messages == original_messages + (span,) = [ + span + for span in exporter.get_finished_spans() + if (span.attributes or {}).get(SpanAttributes.GEN_AI_SPAN_KIND) == "LLM" + ] + attrs = span.attributes or {} + canonical_input = json.loads(attrs[SpanAttributes.INPUT_VALUE]) + # At the audited revision this is only the first message's content, even + # though structured message attributes still contain the current question. + assert isinstance(canonical_input, dict), "canonical input lost the conversation" + (conversation,) = canonical_input["messages"] + assert [ + (message["id"][-1], message["kwargs"]["content"]) + for message in conversation + ] == [(type(message).__name__, message.content) for message in original_messages] + + last_message_content = ( + f"{SpanAttributes.GEN_AI_INPUT_MESSAGES}.{len(messages) - 1}." + f"{MessageAttributes.MESSAGE_CONTENT}" + ) + assert attrs[last_message_content] == "CURRENT_USER_QUESTION_MARKER" + + +_IMAGE_PAYLOAD = "c3ludGhldGljLWltYWdlLXBheWxvYWQ=" +_IMAGE_URL = "https://example.invalid/picture.png" +_IMAGE_DATA_URL = f"data:image/png;base64,{_IMAGE_PAYLOAD}" + + +@pytest.mark.parametrize( + "image_url, expected_url", + [ + pytest.param(_IMAGE_URL, _IMAGE_URL, id="string-url"), + pytest.param(_IMAGE_DATA_URL, REDACTED_VALUE, id="string-data-url"), + pytest.param({"url": _IMAGE_URL}, _IMAGE_URL, id="dict-url"), + pytest.param({"url": _IMAGE_DATA_URL}, REDACTED_VALUE, id="dict-data-url"), + ], +) +@pytest.mark.parametrize( + "exporter, hide_inputs", + [ + pytest.param({"hide_inputs": False}, False, id="visible"), + pytest.param({"hide_inputs": True}, True, id="hidden"), + ], + indirect=["exporter"], +) +def test_canonical_input_preserves_image_messages( + exporter, hide_inputs, image_url, expected_url, caplog +): + text = "Describe this synthetic picture" + messages = [ + HumanMessage( + content=[ + {"type": "text", "text": text}, + {"type": "image_url", "image_url": image_url}, + ] + ) + ] + original_messages = deepcopy(messages) + model = FakeMessagesListChatModel( + responses=[AIMessage(content="offline answer")], cache=False + ) + + response = model.invoke(messages) + + assert response.content == "offline answer" + assert messages == original_messages + (span,) = [ + span + for span in exporter.get_finished_spans() + if (span.attributes or {}).get(SpanAttributes.GEN_AI_SPAN_KIND) == "LLM" + ] + assert span.status.status_code.name == "OK" + attrs = span.attributes or {} + assert SpanAttributes.INPUT_VALUE in attrs, "canonical input is missing" + exported = json.dumps([dict(s.attributes or {}) for s in exporter.get_finished_spans()]) + assert _IMAGE_PAYLOAD not in exported + assert _IMAGE_DATA_URL not in exported + assert not [ + record + for record in caplog.records + if record.name.startswith("traceai_langchain") and record.levelno >= 40 + ] + if hide_inputs: + assert attrs[SpanAttributes.INPUT_VALUE] == REDACTED_VALUE + assert text not in exported + assert _IMAGE_URL not in exported + else: + canonical_input = json.loads(attrs[SpanAttributes.INPUT_VALUE]) + (conversation,) = canonical_input["messages"] + (message,) = conversation + assert message["id"][-1] == "HumanMessage" + assert message["kwargs"]["content"] == [ + {"type": "text", "text": text}, + {"type": "text", "text": f"[Image: {expected_url}]"}, + ] + assert json.loads(attrs[SpanAttributes.INPUT_IMAGES]) == [expected_url] + assert expected_url in exported diff --git a/python/frameworks/langchain/tests/test_tool_calls.py b/python/frameworks/langchain/tests/test_tool_calls.py new file mode 100644 index 000000000..09169f41d --- /dev/null +++ b/python/frameworks/langchain/tests/test_tool_calls.py @@ -0,0 +1,153 @@ +"""normalized tool calls must reach semantic attributes.""" + +from copy import deepcopy +import json + +import pytest +from fi_instrumentation import TraceConfig +from langchain_core.language_models.fake_chat_models import FakeMessagesListChatModel +from langchain_core.messages import AIMessage +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter + +from traceai_langchain import LangChainInstrumentor +from traceai_langchain._tracer import ( + _flatten, + _input_messages, + _output_messages, + _parse_message_data, +) + + +def _serialized_message(kwargs, message_class="AIMessage"): + return { + "lc": 1, + "type": "constructor", + "id": ["langchain", "schema", "messages", message_class], + "kwargs": {"content": "", **kwargs}, + } + + +def _normalized_calls(): + return [ + {"name": "lookup", "args": {"city": "Paris"}, "id": "call_test", "type": "tool_call"}, + {"name": "ping", "args": {}, "id": None, "type": "tool_call"}, + ] + + +def _provider_calls(): + return [{ + "id": "call_provider", + "type": "function", + "function": {"name": "provider_lookup", "arguments": '{"city": "Rome"}'}, + }] + + +@pytest.mark.parametrize("direction", ["input", "output"]) +@pytest.mark.parametrize("message_class", ["AIMessage", "AIMessageChunk"]) +def test_normalized_tool_calls_are_semantic_messages(direction, message_class): + message = _serialized_message({"tool_calls": _normalized_calls()}, message_class) + before = deepcopy(message) + if direction == "input": + attributes = dict(_flatten(_input_messages({"messages": [[message]]}))) + else: + attributes = dict(_flatten(_output_messages({"generations": [[{"message": message}]]}))) + + prefix = f"gen_ai.{direction}.messages.0" + assert attributes == { + f"{prefix}.message.role": "assistant", + f"{prefix}.message.tool_calls.0.tool_call.id": "call_test", + f"{prefix}.message.tool_calls.0.tool_call.function.name": "lookup", + f"{prefix}.message.tool_calls.0.tool_call.function.arguments": json.dumps({"city": "Paris"}), + f"{prefix}.message.tool_calls.1.tool_call.function.name": "ping", + f"{prefix}.message.tool_calls.1.tool_call.function.arguments": "{}", + } + assert message == before + + +def test_normalized_tool_calls_take_precedence_without_duplicates(): + message = _serialized_message({ + "tool_calls": _normalized_calls()[:1], + "additional_kwargs": {"tool_calls": [ + {"id": "call_test", "type": "function", "function": { + "name": "lookup", "arguments": '{"city": "Paris"}', + }}, + *_provider_calls(), + ]}, + }) + before = deepcopy(message) + attributes = dict(_parse_message_data(message)) + assert attributes["message.tool_calls"] == [{ + "tool_call.id": "call_test", + "tool_call.function.name": "lookup", + "tool_call.function.arguments": json.dumps({"city": "Paris"}), + }] + assert message == before + + +@pytest.mark.parametrize("normalized", [{}, {"tool_calls": []}], ids=["absent", "empty"]) +def test_provider_tool_calls_remain_fallback(normalized): + message = _serialized_message({ + **normalized, + "additional_kwargs": {"tool_calls": _provider_calls()}, + }) + before = deepcopy(message) + attributes = dict(_parse_message_data(message)) + assert attributes["message.tool_calls"] == [{ + "tool_call.id": "call_provider", + "tool_call.function.name": "provider_lookup", + "tool_call.function.arguments": '{"city": "Rome"}', + }] + assert message == before + + +def test_legacy_function_call_without_tool_calls_is_unchanged(): + message = _serialized_message({ + "content": "answer", + "additional_kwargs": {"function_call": { + "name": "lookup", "arguments": '{"city": "Paris"}', + }}, + }) + assert dict(_parse_message_data(message)) == { + "message.role": "assistant", + "message.content": "answer", + "message.function_call_name": "lookup", + "message.function_call_arguments_json": '{"city": "Paris"}', + } + + +def test_normalized_tool_calls_reach_exported_span(): + reply = AIMessage(content="", tool_calls=_normalized_calls()[:1]) + before = deepcopy(reply.tool_calls) + exporter = InMemorySpanExporter() + provider = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + instrumentor = LangChainInstrumentor() + instrumentor.instrument( + tracer_provider=provider, + config=TraceConfig(hide_outputs=False, hide_output_messages=False, hide_output_text=False), + ) + try: + result = FakeMessagesListChatModel(responses=[reply]).invoke("look up Paris") + assert result.tool_calls == before + assert result.additional_kwargs == {} + spans = [ + span for span in exporter.get_finished_spans() + if span.attributes.get("gen_ai.span.kind") == "LLM" + ] + assert len(spans) == 1 + tool_attributes = { + name: value for name, value in spans[0].attributes.items() + if name.startswith("gen_ai.output.messages.0.message.tool_calls.") + } + prefix = "gen_ai.output.messages.0.message.tool_calls.0" + assert tool_attributes == { + f"{prefix}.tool_call.id": "call_test", + f"{prefix}.tool_call.function.name": "lookup", + f"{prefix}.tool_call.function.arguments": json.dumps({"city": "Paris"}), + } + assert reply.tool_calls == before + finally: + instrumentor.uninstrument() + provider.shutdown() diff --git a/python/frameworks/langchain/traceai_langchain/_tracer.py b/python/frameworks/langchain/traceai_langchain/_tracer.py index 61a7d6f02..e7d4a7781 100644 --- a/python/frameworks/langchain/traceai_langchain/_tracer.py +++ b/python/frameworks/langchain/traceai_langchain/_tracer.py @@ -29,7 +29,12 @@ from uuid import UUID import wrapt # type: ignore -from fi_instrumentation import TraceConfig, get_attributes_from_context, safe_json_dumps +from fi_instrumentation import ( + REDACTED_VALUE, + TraceConfig, + get_attributes_from_context, + safe_json_dumps, +) from fi_instrumentation.fi_types import ( DocumentAttributes, EmbeddingAttributes, @@ -528,6 +533,12 @@ def _update_span( filtered_data, images, eval_input, query = _filter_images( run.inputs.get("messages", []) ) + # Older supported cores do not mask the aggregate image attribute. + config = getattr(span, "_self_config", None) + if isinstance(config, TraceConfig) and ( + config.hide_inputs or config.hide_input_images + ): + images = [] if not filtered_data: if isinstance(run.inputs, dict): @@ -560,8 +571,8 @@ def _update_span( _retrieval_documents(run), _metadata(run), _session_id_from_thread_id(run, captured_context), + # Keep the full run input canonical, not the first message's query. _as_raw_input_output(run, outputs), - _as_query(query), ) ) ) @@ -641,9 +652,9 @@ def _as_raw_input_output( ) -> Iterator[Tuple[str, Any]]: if raw_input := run.inputs: if isinstance(raw_input, dict): - yield INPUT_VALUE, safe_json_dumps(raw_input) + yield INPUT_VALUE, safe_json_dumps(_replace_image_blocks(raw_input)) elif hasattr(raw_input, "__dict__"): - yield INPUT_VALUE, safe_json_dumps(raw_input.__dict__) + yield INPUT_VALUE, safe_json_dumps(_replace_image_blocks(raw_input.__dict__)) if raw_output := outputs: if isinstance(raw_output, dict): yield OUTPUT_VALUE, safe_json_dumps(raw_output) @@ -798,10 +809,23 @@ def _parse_message_data( assert hasattr(obj, "get"), f"expected Mapping, found {type(obj)}" for k, v in _get_attributes_from_message_content(obj): yield f"{MESSAGE_CONTENTS}.{i}.{k}", v - if additional_kwargs := kwargs.get("additional_kwargs"): - assert hasattr( - additional_kwargs, "get" - ), f"expected Mapping, found {type(additional_kwargs)}" + additional_kwargs = kwargs.get("additional_kwargs") or {} + assert hasattr( + additional_kwargs, "get" + ), f"expected Mapping, found {type(additional_kwargs)}" + # LangChain's normalized calls are authoritative; providers may also + # include a redundant legacy representation in additional_kwargs. + if tool_calls := kwargs.get("tool_calls") or additional_kwargs.get("tool_calls"): + assert isinstance( + tool_calls, Iterable + ), f"expected Iterable, found {type(tool_calls)}" + message_tool_calls = [] + for tool_call in tool_calls: + if message_tool_call := dict(_get_tool_call(tool_call)): + message_tool_calls.append(message_tool_call) + if message_tool_calls: + yield MESSAGE_TOOL_CALLS, message_tool_calls + if additional_kwargs: if function_call := additional_kwargs.get("function_call"): assert hasattr( function_call, "get" @@ -814,17 +838,6 @@ def _parse_message_data( arguments, str ), f"expected str, found {type(arguments)}" yield MESSAGE_FUNCTION_CALL_ARGUMENTS_JSON, arguments - if tool_calls := additional_kwargs.get("tool_calls"): - assert isinstance( - tool_calls, Iterable - ), f"expected Iterable, found {type(tool_calls)}" - message_tool_calls = [] - for tool_call in tool_calls: - if message_tool_call := dict(_get_tool_call(tool_call)): - message_tool_calls.append(message_tool_call) - if message_tool_calls: - yield MESSAGE_TOOL_CALLS, message_tool_calls - # Add handling for tool_outputs if tool_outputs := additional_kwargs.get("tool_outputs"): assert isinstance( @@ -845,7 +858,16 @@ def _get_tool_call(tool_call: Optional[Mapping[str, Any]]) -> Iterator[Tuple[str if not tool_call: return assert hasattr(tool_call, "get"), f"expected Mapping, found {type(tool_call)}" - if function := tool_call.get("function"): + if "args" in tool_call: + arguments = tool_call["args"] + assert isinstance(arguments, Mapping), f"expected Mapping, found {type(arguments)}" + function = { + "name": tool_call.get("name"), + "arguments": safe_json_dumps(arguments), + } + else: + function = tool_call.get("function") + if function: assert hasattr(function, "get"), f"expected Mapping, found {type(function)}" if name := function.get("name"): assert isinstance(name, str), f"expected str, found {type(name)}" @@ -853,6 +875,9 @@ def _get_tool_call(tool_call: Optional[Mapping[str, Any]]) -> Iterator[Tuple[str if arguments := function.get("arguments"): assert isinstance(arguments, str), f"expected str, found {type(arguments)}" yield TOOL_CALL_FUNCTION_ARGUMENTS_JSON, arguments + if id_ := tool_call.get("id"): + assert isinstance(id_, str), f"expected str, found {type(id_)}" + yield TOOL_CALL_ID, id_ @stop_on_exception @@ -1222,14 +1247,32 @@ def _get_attributes_from_message_content( yield f"{MESSAGE_CONTENT_AUDIO}", audio.get("data", "") +def _get_image_url(image: Any) -> Optional[str]: + url = image.get("url") if isinstance(image, Mapping) else image + if not isinstance(url, str) or not url: + return None + return REDACTED_VALUE if url.lower().startswith("data:") else url + + def _get_attributes_from_image( - image: Mapping[str, Any], + image: Any, ) -> Iterator[Tuple[str, AttributeValue]]: - image = dict(image) - if url := image.pop("url"): + if url := _get_image_url(image): yield f"{IMAGE_URL}", url +def _replace_image_blocks(value: Any) -> Any: + """Keep the conversation intact while replacing images with safe placeholders.""" + if isinstance(value, dict): + if value.get("type") == "image_url": + if url := _get_image_url(value.get("image_url")): + return {"type": "text", "text": f"[Image: {url}]"} + return {key: _replace_image_blocks(item) for key, item in value.items()} + if isinstance(value, list): + return [_replace_image_blocks(item) for item in value] + return value + + def _filter_images(messages: Any) -> Tuple[Any, List[str], str, str]: filtered_data = [] images = [] @@ -1264,7 +1307,7 @@ def _filter_images(messages: Any) -> Tuple[Any, List[str], str, str]: if text: eval_input.append(str(text)) elif obj.get("type") == "image_url": - image_url = obj.get("image_url", {}).get("url") + image_url = _get_image_url(obj.get("image_url")) if image_url: images.append(image_url) filtered_data.append( @@ -1333,6 +1376,7 @@ def _filter_images(messages: Any) -> Tuple[Any, List[str], str, str]: SESSION_ID = SpanAttributes.SESSION_ID TOOL_CALL_FUNCTION_ARGUMENTS_JSON = ToolCallAttributes.TOOL_CALL_FUNCTION_ARGUMENTS_JSON TOOL_CALL_FUNCTION_NAME = ToolCallAttributes.TOOL_CALL_FUNCTION_NAME +TOOL_CALL_ID = ToolCallAttributes.TOOL_CALL_ID GEN_AI_TOOL_DESCRIPTION = SpanAttributes.GEN_AI_TOOL_DESCRIPTION GEN_AI_TOOL_NAME = SpanAttributes.GEN_AI_TOOL_NAME TOOL_PARAMETERS = SpanAttributes.TOOL_PARAMETERS diff --git a/python/frameworks/langchain/traceai_langchain/version.py b/python/frameworks/langchain/traceai_langchain/version.py index 210c13fdd..9fbd69814 100644 --- a/python/frameworks/langchain/traceai_langchain/version.py +++ b/python/frameworks/langchain/traceai_langchain/version.py @@ -1 +1 @@ -__version__ = "0.2.2" # x-release-please-version +__version__ = "0.2.3" # x-release-please-version diff --git a/python/frameworks/litellm/CHANGELOG.md b/python/frameworks/litellm/CHANGELOG.md index 3bad22708..5d69d86a9 100644 --- a/python/frameworks/litellm/CHANGELOG.md +++ b/python/frameworks/litellm/CHANGELOG.md @@ -1,5 +1,16 @@ +## [0.1.12] - pending release + +### Changed +- traceai-litellm now requires fi-instrumentation-otel 1.0.0 or later. Action: Upgrade fi-instrumentation-otel if your environment pins an earlier version. + +### Fixed +- LiteLLM spans record only known invocation parameters on completion, embedding and image generation calls, sync and async, so API keys and headers passed to LiteLLM are no longer recorded in span attributes. +- A failed LiteLLM stream ends its span and restores the current context, async stream setup keeps the provider's exception, and successful streams record token usage when the stream includes it. +- Completion calls with a structured-output response_format class, or another argument that tracing cannot serialize, complete normally. +- Positional calls record their model and input, image-generation calls record token usage when LiteLLM reports it, and input capture keeps a conversation's system and earlier text messages. + ## [0.1.11] - 2026-10-29 ### Fixed diff --git a/python/frameworks/litellm/README.md b/python/frameworks/litellm/README.md index 5fd7f4065..6407e9f72 100644 --- a/python/frameworks/litellm/README.md +++ b/python/frameworks/litellm/README.md @@ -8,7 +8,7 @@ This integration provides support for using OpenTelemetry with the Litellm frame 1. **Install traceAI Litellm** ```bash -pip install pip install traceAI-litellm +pip install traceAI-litellm tenacity ``` diff --git a/python/frameworks/litellm/poetry.lock b/python/frameworks/litellm/poetry.lock index 9d22c0f2f..eb0022c4d 100644 --- a/python/frameworks/litellm/poetry.lock +++ b/python/frameworks/litellm/poetry.lock @@ -2301,4 +2301,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.1" python-versions = ">3.9, <3.14" -content-hash = "bdee8eb7b356a448b33abd2a71a4e1fdcc305f9c81288cba5c4cce416cbf1522" +content-hash = "d9bf370836d07d07be215e216281e76b2ddc6c48f178ef82022104d40e4b8f00" diff --git a/python/frameworks/litellm/pyproject.toml b/python/frameworks/litellm/pyproject.toml index a799c34d2..36d0aab1a 100644 --- a/python/frameworks/litellm/pyproject.toml +++ b/python/frameworks/litellm/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "traceAI-litellm" -version = "0.1.11" +version = "0.1.12" description = "OpenTelemetry instrumentation for LiteLLM" authors = ["Future AGI "] readme = "README.md" @@ -30,7 +30,7 @@ classifiers = [ [tool.poetry.dependencies] python = ">3.9, <3.14" litellm = ">=1.43.0" -fi-instrumentation-otel = ">=0.1.11" +fi-instrumentation-otel = ">=1.0.0" diff --git a/python/frameworks/litellm/tests/conftest.py b/python/frameworks/litellm/tests/conftest.py new file mode 100644 index 000000000..d4792be6a --- /dev/null +++ b/python/frameworks/litellm/tests/conftest.py @@ -0,0 +1,56 @@ +"""Environment fixtures for the package's offline regression tests.""" + +import os + +import pytest + + +@pytest.fixture(autouse=True) +def _collector_at_a_closed_local_port(monkeypatch): + """Point the collector URLs at a closed local port for every test. + + A test that calls register() exports spans at shutdown; with the URLs unset that goes to + api.futureagi.com. A test that needs them absent (a default-endpoint test) removes them + itself with monkeypatch.delenv. + """ + monkeypatch.setenv("FI_BASE_URL", "http://127.0.0.1:9") + monkeypatch.setenv("FI_GRPC_URL", "http://127.0.0.1:9") + + +@pytest.fixture +def clean_env(): + """Fixture to clean environment variables before each test.""" + # Store original env vars + original_env = dict(os.environ) + + # Clear relevant environment variables + env_vars_to_clear = [ + "FI_PROJECT_NAME", + "FI_PROJECT_VERSION_NAME", + "FI_API_KEY", + "FI_SECRET_KEY", + "FI_HIDE_LLM_INVOCATION_PARAMETERS", + "FI_HIDE_INPUTS", + "FI_HIDE_OUTPUTS", + "FI_HIDE_INPUT_MESSAGES", + "FI_HIDE_OUTPUT_MESSAGES", + "FI_HIDE_INPUT_IMAGES", + "FI_HIDE_INPUT_TEXT", + "FI_HIDE_OUTPUT_TEXT", + "FI_HIDE_EMBEDDING_VECTORS", + "FI_BASE64_IMAGE_MAX_LENGTH", + "FI_HIDE_ERROR_DETAILS", + "FI_PII_REDACTION", + ] + + for var in env_vars_to_clear: + os.environ.pop(var, None) + + # FI_BASE_URL and FI_GRPC_URL are left alone: the autouse fixture + # _collector_at_a_closed_local_port sets them for every test. + + yield + + # Restore original environment + os.environ.clear() + os.environ.update(original_env) diff --git a/python/frameworks/litellm/tests/test_framework_litellm_async_stream_setup.py b/python/frameworks/litellm/tests/test_framework_litellm_async_stream_setup.py new file mode 100644 index 000000000..dafdb2e4b --- /dev/null +++ b/python/frameworks/litellm/tests/test_framework_litellm_async_stream_setup.py @@ -0,0 +1,68 @@ +"""Regression coverage for LiteLLM failures before an async stream is created.""" + +import asyncio +from unittest.mock import AsyncMock + +import pytest +from opentelemetry import context as context_api +from opentelemetry import trace as trace_api +from opentelemetry.context import _SUPPRESS_INSTRUMENTATION_KEY +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter + + +def test_acompletion_stream_setup_preserves_provider_exception(monkeypatch): + # Avoid LiteLLM's remote cost-map lookup when importing the provider library. + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + import litellm + from fi_instrumentation import FITracer, TraceConfig + from traceai_litellm import LiteLLMInstrumentor + + failure = litellm.RateLimitError( + message="synthetic rate limit", + llm_provider="openai", + model="gpt-4o-mini", + ) + mock_acompletion = AsyncMock(side_effect=failure) + exporter = InMemorySpanExporter() + provider = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + instrumentor = LiteLLMInstrumentor() + monkeypatch.setattr( + instrumentor, + "_tracer", + FITracer(provider.get_tracer(__name__), config=TraceConfig()), + raising=False, + ) + monkeypatch.setattr( + instrumentor, "original_litellm_funcs", {"acompletion": mock_acompletion} + ) + kwargs = { + "model": "gpt-4o-mini", + "messages": [{"role": "user", "content": "hello"}], + "stream": True, + } + + async def invoke(): + token = context_api.attach( + context_api.set_value(_SUPPRESS_INSTRUMENTATION_KEY, False) + ) + previous_span = trace_api.get_current_span() + try: + with pytest.raises(litellm.RateLimitError) as caught: + await instrumentor._acompletion_wrapper(**kwargs) + assert caught.value is failure + assert trace_api.get_current_span() is previous_span + finally: + context_api.detach(token) + + try: + asyncio.run(invoke()) + mock_acompletion.assert_awaited_once_with(**kwargs) + spans = exporter.get_finished_spans() + assert len(spans) == 1 + assert spans[0].name == "acompletion" + assert spans[0].status.status_code is trace_api.StatusCode.ERROR + finally: + provider.shutdown() diff --git a/python/frameworks/litellm/tests/test_framework_litellm_canonical_input.py b/python/frameworks/litellm/tests/test_framework_litellm_canonical_input.py new file mode 100644 index 000000000..8a0b3da5d --- /dev/null +++ b/python/frameworks/litellm/tests/test_framework_litellm_canonical_input.py @@ -0,0 +1,90 @@ +"""Regression coverage for T3-LITELLM-007 using offline completion fakes.""" + +import asyncio +from copy import deepcopy +import json +from types import MethodType, SimpleNamespace +from unittest.mock import AsyncMock, Mock + +import pytest +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter + + +@pytest.mark.parametrize( + "function_name", + [ + "completion", + "acompletion", + "completion_with_retries", + "acompletion_with_retries", + ], +) +@pytest.mark.parametrize( + "messages", + [ + pytest.param( + [{"role": "user", "content": "CURRENT_USER_QUESTION_MARKER"}], + id="single-user", + ), + pytest.param( + [ + {"role": "system", "content": "SYSTEM_INSTRUCTION_MARKER"}, + {"role": "user", "content": "CURRENT_USER_QUESTION_MARKER"}, + ], + id="system-plus-user", + ), + pytest.param( + [ + {"role": "user", "content": "PREVIOUS_USER_QUESTION_MARKER"}, + {"role": "assistant", "content": "PREVIOUS_ASSISTANT_ANSWER_MARKER"}, + {"role": "user", "content": "CURRENT_USER_QUESTION_MARKER"}, + ], + id="multi-turn", + ), + ], +) +def test_completion_preserves_canonical_conversation(monkeypatch, function_name, messages): + # Avoid LiteLLM's remote cost-map lookup when importing the real wrappers. + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + from fi_instrumentation.fi_types import SpanAttributes + from traceai_litellm import LiteLLMInstrumentor + + exporter = InMemorySpanExporter() + provider = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + response = {"choices": [{"message": {"role": "assistant", "content": "hello"}}]} + is_async = function_name.startswith("a") + original = (AsyncMock if is_async else Mock)(return_value=response) + # Exercise the exact wrapper without installing global instrumentation. + instrumentor = SimpleNamespace( + _tracer=provider.get_tracer(__name__), + original_litellm_funcs={function_name: original}, + ) + if hasattr(LiteLLMInstrumentor, "_get_call_arguments"): + instrumentor._get_call_arguments = MethodType( + LiteLLMInstrumentor._get_call_arguments, instrumentor + ) + original_messages = deepcopy(messages) + kwargs = {"model": "gpt-4o-mini", "messages": messages} + + try: + wrapper = getattr(LiteLLMInstrumentor, f"_{function_name}_wrapper") + result = wrapper(instrumentor, **kwargs) + if is_async: + result = asyncio.run(result) + original.assert_awaited_once_with(**kwargs) + else: + original.assert_called_once_with(**kwargs) + + assert result is response + assert messages == original_messages + spans = exporter.get_finished_spans() + assert len(spans) == 1 + # Assert the final exported value, not an earlier set_attribute call. + assert spans[0].attributes[SpanAttributes.INPUT_VALUE] == json.dumps( + original_messages + ) + finally: + provider.shutdown() diff --git a/python/frameworks/litellm/tests/test_framework_litellm_image_usage.py b/python/frameworks/litellm/tests/test_framework_litellm_image_usage.py new file mode 100644 index 000000000..674d00364 --- /dev/null +++ b/python/frameworks/litellm/tests/test_framework_litellm_image_usage.py @@ -0,0 +1,211 @@ +"""T3-LITELLM-003: successful image calls must survive usage finalization.""" + +import asyncio +from unittest.mock import AsyncMock, Mock + +import pytest +from opentelemetry import context as context_api +from opentelemetry.context import _SUPPRESS_INSTRUMENTATION_KEY +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +from opentelemetry.trace import StatusCode + + +IMAGE_URL = "https://example.invalid/generated.png" + + +@pytest.fixture +def litellm_module(monkeypatch): + # LiteLLM must not fetch its model cost map during import. + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + import traceai_litellm + + return traceai_litellm + + +@pytest.fixture +def image_harness(monkeypatch, litellm_module): + exporter = InMemorySpanExporter() + provider = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + instrumentor = litellm_module.LiteLLMInstrumentor() + # Exercise the real wrappers without mutating LiteLLM's global functions or + # the singleton instrumentor's shared original-function dictionary. + monkeypatch.setattr(instrumentor, "original_litellm_funcs", {}) + monkeypatch.setattr( + instrumentor, "_tracer", provider.get_tracer(__name__), raising=False + ) + context = context_api.attach( + context_api.set_value(_SUPPRESS_INSTRUMENTATION_KEY, False) + ) + try: + yield instrumentor, exporter, litellm_module + finally: + context_api.detach(context) + provider.shutdown() + + +def _call_image(image_harness, asynchronous, response): + instrumentor, _, _ = image_harness + kwargs = {"model": "dall-e-3", "prompt": "a garden"} + if asynchronous: + original = AsyncMock(return_value=response) + instrumentor.original_litellm_funcs["aimage_generation"] = original + result = asyncio.run(instrumentor._aimage_generation_wrapper(**kwargs)) + original.assert_awaited_once_with(**kwargs) + else: + original = Mock(return_value=response) + instrumentor.original_litellm_funcs["image_generation"] = original + result = instrumentor._image_generation_wrapper(**kwargs) + original.assert_called_once_with(**kwargs) + assert result is response + + +def _image_span(image_harness): + _, exporter, module = image_harness + spans = exporter.get_finished_spans() + assert len(spans) == 1 + span = spans[0] + assert span.status.status_code == StatusCode.OK + assert not span.events + assert span.attributes[module.SpanAttributes.OUTPUT_VALUE] == IMAGE_URL + return span + + +@pytest.mark.parametrize("asynchronous", [False, True], ids=["sync", "async"]) +def test_image_response_default_usage(image_harness, asynchronous): + """Real LiteLLM 1.81.3 supplies image-style usage even when it is omitted.""" + _, _, module = image_harness + response = module.ImageResponse(data=[{"url": IMAGE_URL}]) + + _call_image(image_harness, asynchronous, response) + + _image_span(image_harness) + + +@pytest.mark.parametrize("asynchronous", [False, True], ids=["sync", "async"]) +def test_image_response_native_usage(image_harness, asynchronous): + """Keep the native LiteLLM image usage representation, not just a dict fake.""" + _, _, module = image_harness + response = module.ImageResponse( + data=[{"url": IMAGE_URL}], + usage={ + "input_tokens": 3, + "input_tokens_details": {"text_tokens": 3, "image_tokens": 0}, + "output_tokens": 5, + "total_tokens": 8, + }, + ) + + _call_image(image_harness, asynchronous, response) + + span = _image_span(image_harness) + attributes = module.SpanAttributes + assert span.attributes[attributes.GEN_AI_USAGE_INPUT_TOKENS] == 3 + assert span.attributes[attributes.GEN_AI_USAGE_OUTPUT_TOKENS] == 5 + assert span.attributes[attributes.GEN_AI_USAGE_TOTAL_TOKENS] == 8 + + +@pytest.mark.parametrize("asynchronous", [False, True], ids=["sync", "async"]) +@pytest.mark.parametrize( + "usage, expected", + [ + pytest.param( + {"input_tokens": 3, "output_tokens": 5, "total_tokens": 8}, + (3, 5, 8), + id="image", + ), + pytest.param( + {"prompt_tokens": 3, "completion_tokens": 5, "total_tokens": 8}, + (3, 5, 8), + id="legacy", + ), + pytest.param( + { + "input_tokens": 0, + "output_tokens": 0, + "total_tokens": 0, + "prompt_tokens": 99, + "completion_tokens": 99, + }, + (0, 0, 0), + id="image-zero-preferred", + ), + pytest.param( + { + "input_tokens": None, + "output_tokens": None, + "prompt_tokens": 3, + "completion_tokens": 5, + "total_tokens": 8, + }, + (3, 5, 8), + id="legacy-fallback", + ), + pytest.param({"input_tokens": 3}, (3, None, None), id="input-only"), + pytest.param({"output_tokens": 5}, (None, 5, None), id="output-only"), + pytest.param({"total_tokens": 8}, (None, None, 8), id="total-only"), + pytest.param(None, (None, None, None), id="none"), + pytest.param({}, (None, None, None), id="empty"), + pytest.param({"provider_units": 7}, (None, None, None), id="unknown-fields"), + pytest.param("provider-specific", (None, None, None), id="unknown-shape"), + ], +) +def test_image_response_usage_shapes(image_harness, asynchronous, usage, expected): + _, _, module = image_harness + response = module.ImageResponse(data=[{"url": IMAGE_URL}]) + # Preserve the exact provider usage shape, rather than letting a particular + # LiteLLM version normalize partial, missing, or legacy fields for us. + object.__setattr__(response, "usage", usage) + + _call_image(image_harness, asynchronous, response) + + span = _image_span(image_harness) + attributes = module.SpanAttributes + for name, value in zip( + ( + attributes.GEN_AI_USAGE_INPUT_TOKENS, + attributes.GEN_AI_USAGE_OUTPUT_TOKENS, + attributes.GEN_AI_USAGE_TOTAL_TOKENS, + ), + expected, + ): + if value is None: + assert name not in span.attributes + else: + assert span.attributes[name] == value + + +@pytest.mark.parametrize("asynchronous", [False, True], ids=["sync", "async"]) +def test_image_response_without_usage(image_harness, asynchronous): + _, _, module = image_harness + response = module.ImageResponse(data=[{"url": IMAGE_URL}]) + if hasattr(response, "usage"): + object.__delattr__(response, "usage") + + _call_image(image_harness, asynchronous, response) + + span = _image_span(image_harness) + attributes = module.SpanAttributes + assert attributes.GEN_AI_USAGE_INPUT_TOKENS not in span.attributes + assert attributes.GEN_AI_USAGE_OUTPUT_TOKENS not in span.attributes + assert attributes.GEN_AI_USAGE_TOTAL_TOKENS not in span.attributes + + +@pytest.mark.parametrize("response_kind", ["chat", "embedding"]) +def test_non_image_usage_is_preserved(litellm_module, response_kind): + module = litellm_module + usage = {"prompt_tokens": 3, "completion_tokens": 5, "total_tokens": 8} + if response_kind == "chat": + response = module.ModelResponse(choices=[], usage=usage) + else: + response = module.EmbeddingResponse(data=[], usage=usage) + span = Mock() + + module._finalize_span(span, response) + + attributes = module.SpanAttributes + span.set_attribute.assert_any_call(attributes.GEN_AI_USAGE_INPUT_TOKENS, 3) + span.set_attribute.assert_any_call(attributes.GEN_AI_USAGE_OUTPUT_TOKENS, 5) + span.set_attribute.assert_any_call(attributes.GEN_AI_USAGE_TOTAL_TOKENS, 8) diff --git a/python/frameworks/litellm/tests/test_framework_litellm_positional.py b/python/frameworks/litellm/tests/test_framework_litellm_positional.py new file mode 100644 index 000000000..403fa7f21 --- /dev/null +++ b/python/frameworks/litellm/tests/test_framework_litellm_positional.py @@ -0,0 +1,233 @@ +"""T3-LITELLM-008: positional requests must retain their tracing attributes.""" + +import asyncio +import json +from functools import wraps + +import pytest +from opentelemetry import context as context_api +from opentelemetry.context import _SUPPRESS_INSTRUMENTATION_KEY +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter + +from fi_instrumentation.fi_types import ( + EmbeddingAttributes, + MessageAttributes, + SpanAttributes, +) + + +MARKER = "POSITIONAL_QUESTION_MARKER" +API_NAMES = ( + "completion", + "acompletion", + "completion_with_retries", + "acompletion_with_retries", + "embedding", + "aembedding", + "image_generation", + "aimage_generation", +) + + +@pytest.fixture +def positional_instrumentor(monkeypatch): + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + from traceai_litellm import LiteLLMInstrumentor + + exporter = InMemorySpanExporter() + provider = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + instrumentor = LiteLLMInstrumentor() + # Exercise the actual wrappers without changing global LiteLLM functions or + # the instrumentor's shared registry. No provider or collector is contacted. + monkeypatch.setattr(instrumentor, "original_litellm_funcs", {}) + monkeypatch.setattr( + instrumentor, "_tracer", provider.get_tracer(__name__), raising=False + ) + context_token = context_api.attach( + context_api.set_value(_SUPPRESS_INSTRUMENTATION_KEY, False) + ) + try: + yield instrumentor, exporter + finally: + context_api.detach(context_token) + provider.shutdown() + + +def _install_fake(instrumentor, api_name, response): + def completion(model, messages, stream=False, callback=None, **kwargs): + return response + + def embedding(model, input, **kwargs): + return response + + def image_generation(prompt, model=None, **kwargs): + return response + + if "completion" in api_name: + base_name, signature_source = "completion", completion + elif "embedding" in api_name: + base_name, signature_source = "embedding", embedding + else: + base_name, signature_source = "image_generation", image_generation + instrumentor.original_litellm_funcs[base_name] = signature_source + calls = [] + + def sync_original(*args, **kwargs): + calls.append((args, kwargs)) + return signature_source(*args, **kwargs) + + async def async_original(*args, **kwargs): + calls.append((args, kwargs)) + return signature_source(*args, **kwargs) + + original = async_original if api_name.startswith("a") else sync_original + # Forwarding APIs can expose only (*args, **kwargs). Other decorators retain + # the underlying API signature via __wrapped__. + if api_name not in ( + "completion_with_retries", + "acompletion_with_retries", + "aembedding", + "aimage_generation", + ): + original = wraps(signature_source)(original) + instrumentor.original_litellm_funcs[api_name] = original + return calls + + +def _invoke(instrumentor, api_name, args, kwargs): + result = getattr(instrumentor, f"_{api_name}_wrapper")(*args, **kwargs) + return asyncio.run(result) if api_name.startswith("a") else result + + +@pytest.mark.parametrize("api_name", API_NAMES) +@pytest.mark.parametrize("call_style", ("positional", "mixed", "keyword")) +def test_request_attributes_and_passthrough( + positional_instrumentor, api_name, call_style +): + instrumentor, exporter = positional_instrumentor + response = {"result": "hello"} + calls = _install_fake(instrumentor, api_name, response) + if "completion" in api_name: + request = {"model": "gpt-4o-mini", "messages": [{"role": "user", "content": MARKER}]} + elif "embedding" in api_name: + request = {"model": "embedding-test-model", "input": [MARKER]} + else: + # Image APIs take prompt before model, unlike completion/embedding. + request = {"prompt": MARKER, "model": "image-test-model"} + items = list(request.items()) + positional_count = {"positional": 2, "mixed": 1, "keyword": 0}[call_style] + args = tuple(value for _, value in items[:positional_count]) + kwargs = dict(items[positional_count:]) + kwargs["request_marker"] = "fake-provider-option" + + assert _invoke(instrumentor, api_name, args, kwargs) is response + assert calls == [(args, kwargs)] + assert all(actual is expected for actual, expected in zip(calls[0][0], args)) + assert all(calls[0][1][name] is value for name, value in kwargs.items()) + spans = exporter.get_finished_spans() + assert len(spans) == 1 + attributes = spans[0].attributes + model_attribute = ( + SpanAttributes.EMBEDDING_MODEL_NAME + if "embedding" in api_name + else SpanAttributes.GEN_AI_REQUEST_MODEL + ) + assert attributes[model_attribute] == request["model"] + assert MARKER in attributes[SpanAttributes.INPUT_VALUE] + if "completion" in api_name: + message_prefix = f"{SpanAttributes.GEN_AI_INPUT_MESSAGES}.0" + assert attributes[f"{message_prefix}.{MessageAttributes.MESSAGE_ROLE}"] == "user" + assert attributes[f"{message_prefix}.{MessageAttributes.MESSAGE_CONTENT}"] == MARKER + # Unknown provider options are forwarded, never exported as telemetry. + assert json.loads(attributes[SpanAttributes.GEN_AI_REQUEST_PARAMETERS]) == {} + elif "embedding" in api_name: + text_attribute = ( + f"{SpanAttributes.EMBEDDING_EMBEDDINGS}.0." + f"{EmbeddingAttributes.EMBEDDING_TEXT}" + ) + assert attributes[text_attribute] == str(request["input"]) + + +@pytest.mark.parametrize("api_name", ("completion", "acompletion")) +@pytest.mark.parametrize("stream_style", ("positional", "keyword")) +def test_streaming_positional_request(positional_instrumentor, api_name, stream_style): + instrumentor, exporter = positional_instrumentor + chunk = {"choices": [{"delta": {"content": "hello"}}]} + + async def async_chunks(): + yield chunk + + response = async_chunks() if api_name.startswith("a") else iter([chunk]) + calls = _install_fake(instrumentor, api_name, response) + args = ("gpt-4o-mini", [{"role": "user", "content": MARKER}]) + kwargs = {} + if stream_style == "positional": + args += (True,) + else: + kwargs["stream"] = True + + async def call_and_consume(): + result = await instrumentor._acompletion_wrapper(*args, **kwargs) + assert not exporter.get_finished_spans() + return [item async for item in result] + + if api_name.startswith("a"): + chunks = asyncio.run(call_and_consume()) + else: + result = instrumentor._completion_wrapper(*args, **kwargs) + assert not exporter.get_finished_spans() + chunks = list(result) + assert calls == [(args, kwargs)] + assert len(chunks) == 1 + assert chunks[0] is chunk + spans = exporter.get_finished_spans() + assert len(spans) == 1 + assert spans[0].attributes[SpanAttributes.GEN_AI_REQUEST_MODEL] == "gpt-4o-mini" + assert json.loads(spans[0].attributes[SpanAttributes.INPUT_VALUE]) == args[1] + + +def test_non_json_positional_option_preserves_call(positional_instrumentor): + instrumentor, exporter = positional_instrumentor + response = {"result": "hello"} + calls = _install_fake(instrumentor, "completion", response) + + def callback(): + pass + + args = ("gpt-4o-mini", [{"role": "user", "content": MARKER}], False, callback) + assert _invoke(instrumentor, "completion", args, {}) is response + assert calls == [(args, {})] + attributes = exporter.get_finished_spans()[0].attributes + assert attributes[SpanAttributes.GEN_AI_REQUEST_MODEL] == "gpt-4o-mini" + assert json.loads(attributes[SpanAttributes.INPUT_VALUE]) == args[1] + assert json.loads(attributes[SpanAttributes.GEN_AI_REQUEST_PARAMETERS]) == { + "stream": False, + } + + +def test_unavailable_signature_preserves_call(positional_instrumentor): + instrumentor, exporter = positional_instrumentor + response = {"result": "hello"} + calls = _install_fake(instrumentor, "completion", response) + # inspect.signature rejects this, but the original callable remains valid. + instrumentor.original_litellm_funcs["completion"].__signature__ = object() + args = ("gpt-4o-mini", [{"role": "user", "content": MARKER}]) + kwargs = {"request_marker": "fake-provider-option"} + + assert _invoke(instrumentor, "completion", args, kwargs) is response + assert calls == [(args, kwargs)] + assert len(exporter.get_finished_spans()) == 1 + + +def test_binding_error_is_left_to_original_call(positional_instrumentor): + instrumentor, _ = positional_instrumentor + calls = _install_fake(instrumentor, "completion", {"result": "hello"}) + args = ("gpt-4o-mini", [{"role": "user", "content": MARKER}]) + kwargs = {"model": "duplicate-model"} + + with pytest.raises(TypeError, match="multiple values for argument 'model'"): + _invoke(instrumentor, "completion", args, kwargs) + assert calls == [(args, kwargs)] diff --git a/python/frameworks/litellm/tests/test_framework_litellm_stream_lifecycle.py b/python/frameworks/litellm/tests/test_framework_litellm_stream_lifecycle.py new file mode 100644 index 000000000..060437bbb --- /dev/null +++ b/python/frameworks/litellm/tests/test_framework_litellm_stream_lifecycle.py @@ -0,0 +1,238 @@ +"""T3-LITELLM-004: offline stream lifecycle regressions with real FI/OTel spans.""" + +import asyncio +import os +from contextlib import contextmanager +from types import MethodType, SimpleNamespace + +import pytest +from opentelemetry import trace +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter + +# Prevent LiteLLM's import-time model cost lookup from using the network. +os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" + +from fi_instrumentation import FITracer, TraceConfig +from fi_instrumentation.fi_types import SpanAttributes +from traceai_litellm import LiteLLMInstrumentor + + +CHUNK = {"choices": [{"delta": {"content": "partial"}}]} +REQUEST = {"model": "gpt-4o-mini", "messages": [], "stream": True} + + +class _TrackedContext: + def __init__(self, context): + self.context = context + self.exits = [] + self.span = None + + def __enter__(self): + self.span = self.context.__enter__() + return self.span + + def __exit__(self, *exc_info): + self.exits.append(exc_info) + return self.context.__exit__(*exc_info) + + +class _TrackedTracer: + def __init__(self, tracer): + self.tracer = tracer + self.contexts = [] + + def start_as_current_span(self, *args, **kwargs): + context = _TrackedContext(self.tracer.start_as_current_span(*args, **kwargs)) + # Retain construction contexts even when the provider raises: cleanup + # must be explicit, not dependent on traceback/generator finalization. + self.contexts.append(context) + return context + + +@contextmanager +def _tracing(): + exporter = InMemorySpanExporter() + provider = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + parent_tracer = provider.get_tracer("stream-lifecycle-test") + tracked = _TrackedTracer(FITracer(parent_tracer, config=TraceConfig())) + try: + with parent_tracer.start_as_current_span("parent") as parent: + try: + yield SimpleNamespace( + tracer=tracked, parent_tracer=parent_tracer, + exporter=exporter, parent=parent, + ) + finally: + # Keep the RED tests isolated, including on assertion failure. + for context in reversed(tracked.contexts): + if context.span is not None and context.span.is_recording(): + context.__exit__(None, None, None) + finally: + provider.shutdown() + + +def _instrumentor(harness, function_name, provider_call): + # Exercise the real completion methods without global LiteLLM monkeypatches + # or BaseInstrumentor's process-wide singleton instrumentation state. + instrumentor = SimpleNamespace( + _tracer=harness.tracer, + original_litellm_funcs={function_name: provider_call}, + ) + if hasattr(LiteLLMInstrumentor, "_get_call_arguments"): + instrumentor._get_call_arguments = MethodType( + LiteLLMInstrumentor._get_call_arguments, instrumentor + ) + return instrumentor + + +def _assert_finished(harness, failure=None): + context = harness.tracer.contexts[0] + assert len(context.exits) == 1 + assert not context.span.is_recording() + assert trace.get_current_span().get_span_context() == harness.parent.get_span_context() + spans = harness.exporter.get_finished_spans() + assert len(spans) == 1 + assert spans[0].context == context.span.get_span_context() + assert spans[0].parent == harness.parent.get_span_context() + if failure is not None: + exc_type, exc_value, traceback = context.exits[0] + assert exc_type is type(failure) + assert exc_value is failure + # The original provider frame must still be in the propagated traceback. + frames = [] + current = failure.__traceback__ + while current is not None: + frames.append(current) + current = current.tb_next + assert traceback in frames + assert frames[-1].tb_frame.f_code.co_name in {"provider_call", "chunks"} + if isinstance(failure, Exception): + assert spans[0].status.status_code is trace.StatusCode.ERROR + assert len([event for event in spans[0].events if event.name == "exception"]) == 1 + return context, spans[0] + + +@pytest.mark.parametrize("stage", ["construction", "iteration"]) +@pytest.mark.parametrize("failure_type", [ValueError, asyncio.CancelledError]) +def test_sync_stream_failure_ends_and_detaches_span(stage, failure_type): + failure = failure_type("synthetic stream failure") + + def chunks(): + yield CHUNK + raise failure + + def provider_call(**kwargs): + if stage == "construction": + raise failure + return chunks() + + with _tracing() as harness: + instrumentor = _instrumentor(harness, "completion", provider_call) + stream = None + with pytest.raises(failure_type) as caught: + stream = LiteLLMInstrumentor._completion_wrapper(instrumentor, **REQUEST) + assert next(stream) is CHUNK + next(stream) + assert caught.value is failure + context, _ = _assert_finished(harness, failure) + with harness.parent_tracer.start_as_current_span("following") as following: + assert following.parent == harness.parent.get_span_context() + if stream is not None: + stream.__exit__(None, None, None) + assert len(context.exits) == 1 + assert trace.get_current_span().get_span_context() == following.get_span_context() + + +@pytest.mark.parametrize("stage", ["construction", "iteration"]) +@pytest.mark.parametrize("failure_type", [ValueError, asyncio.CancelledError]) +def test_async_stream_failure_ends_and_detaches_span(stage, failure_type): + async def scenario(): + failure = failure_type("synthetic stream failure") + + async def chunks(): + yield CHUNK + raise failure + + async def provider_call(**kwargs): + if stage == "construction": + raise failure + return chunks() + + # Creation, consumption, assertions and RED cleanup share one task. + with _tracing() as harness: + instrumentor = _instrumentor(harness, "acompletion", provider_call) + stream = None + with pytest.raises(failure_type) as caught: + stream = await LiteLLMInstrumentor._acompletion_wrapper(instrumentor, **REQUEST) + assert await stream.__anext__() is CHUNK + await stream.__anext__() + assert caught.value is failure + context, _ = _assert_finished(harness, failure) + with harness.parent_tracer.start_as_current_span("following") as following: + assert following.parent == harness.parent.get_span_context() + if stream is not None: + await stream.close() + await stream.close() + assert len(context.exits) == 1 + assert trace.get_current_span().get_span_context() == following.get_span_context() + + asyncio.run(scenario()) + + +@pytest.mark.parametrize("exhaust", [False, True], ids=["early-exit", "exhaustion"]) +def test_sync_stream_context_exit_is_idempotent(exhaust): + def provider_call(**kwargs): + return iter([CHUNK]) + + with _tracing() as harness: + instrumentor = _instrumentor(harness, "completion", provider_call) + stream = LiteLLMInstrumentor._completion_wrapper(instrumentor, **REQUEST) + with stream: + assert next(stream) is CHUNK + if exhaust: + with pytest.raises(StopIteration): + next(stream) + context, span = _assert_finished(harness) + if exhaust: + assert span.status.status_code is trace.StatusCode.OK + assert span.attributes[SpanAttributes.OUTPUT_VALUE] == "partial" + with harness.parent_tracer.start_as_current_span("following") as following: + stream.__exit__(None, None, None) + with pytest.raises(StopIteration): + next(stream) + assert len(context.exits) == 1 + assert trace.get_current_span().get_span_context() == following.get_span_context() + + +@pytest.mark.parametrize("exhaust", [False, True], ids=["early-close", "exhaustion"]) +def test_async_stream_close_is_idempotent(exhaust): + async def scenario(): + async def chunks(): + yield CHUNK + + async def provider_call(**kwargs): + return chunks() + + with _tracing() as harness: + instrumentor = _instrumentor(harness, "acompletion", provider_call) + stream = await LiteLLMInstrumentor._acompletion_wrapper(instrumentor, **REQUEST) + assert await stream.__anext__() is CHUNK + if exhaust: + with pytest.raises(StopAsyncIteration): + await stream.__anext__() + await stream.close() + context, span = _assert_finished(harness) + if exhaust: + assert span.status.status_code is trace.StatusCode.OK + assert span.attributes[SpanAttributes.OUTPUT_VALUE] == "partial" + with harness.parent_tracer.start_as_current_span("following") as following: + await stream.close() + with pytest.raises(StopAsyncIteration): + await stream.__anext__() + assert len(context.exits) == 1 + assert trace.get_current_span().get_span_context() == following.get_span_context() + + asyncio.run(scenario()) diff --git a/python/frameworks/litellm/tests/test_framework_litellm_stream_usage.py b/python/frameworks/litellm/tests/test_framework_litellm_stream_usage.py new file mode 100644 index 000000000..2665fca7e --- /dev/null +++ b/python/frameworks/litellm/tests/test_framework_litellm_stream_usage.py @@ -0,0 +1,138 @@ +"""Offline regression coverage for T3-LITELLM-006.""" + +import asyncio +from types import SimpleNamespace +from unittest.mock import MagicMock, Mock + +import pytest +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +from opentelemetry.trace import StatusCode + + +@pytest.mark.parametrize("async_stream", [False, True], ids=["sync", "async"]) +@pytest.mark.parametrize("chunk_format", ["mapping", "to_dict", "attributes"]) +@pytest.mark.parametrize( + "usage_snapshots, empty_choices, expected", + [ + pytest.param( + [{"prompt_tokens": 7, "completion_tokens": 2, "total_tokens": 9}], + False, + {"input_tokens": 7, "output_tokens": 2, "total_tokens": 9}, + id="final-usage", + ), + pytest.param( + [ + {"prompt_tokens": 7, "completion_tokens": 1, "total_tokens": 8}, + {"prompt_tokens": 7, "completion_tokens": 2, "total_tokens": 9}, + {"prompt_tokens": 7, "completion_tokens": 2, "total_tokens": 9}, + ], + False, + {"input_tokens": 7, "output_tokens": 2, "total_tokens": 9}, + id="latest-cumulative-not-summed", + ), + pytest.param( + [ + {"prompt_tokens": 7, "completion_tokens": 2, "total_tokens": 9}, + {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0}, + ], + False, + {"input_tokens": 0, "output_tokens": 0, "total_tokens": 0}, + id="explicit-zero-overwrites", + ), + pytest.param( + [{"prompt_tokens": 7, "completion_tokens": 2, "total_tokens": 9}], + True, + {"input_tokens": 7, "output_tokens": 2, "total_tokens": 9}, + id="usage-only-empty-choices", + ), + pytest.param( + [SimpleNamespace(prompt_tokens=7, completion_tokens=2, total_tokens=9)], + False, + {"input_tokens": 7, "output_tokens": 2, "total_tokens": 9}, + id="attribute-usage-object", + ), + pytest.param( + [{"prompt_tokens": 0, "completion_tokens": None}], + False, + {"input_tokens": 0}, + id="partial-usage-no-invented-counts", + ), + pytest.param( + [None, {}], + False, + {}, + id="missing-null-empty-usage", + ), + ], +) +def test_stream_usage_is_exported( + monkeypatch, async_stream, chunk_format, usage_snapshots, empty_choices, expected +): + """Export final cumulative counts without changing chunks or output text.""" + # Keep LiteLLM's import-time model-cost lookup offline as well as the stream. + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + from traceai_litellm import ( + AsyncStreamingIteratorWrapper, + StreamingIteratorWrapper, + ) + + payloads = [{"choices": [{"delta": {"content": "hello"}}]}] + payloads.extend( + { + "choices": [] if empty_choices else [{"delta": {}}], + "usage": usage, + } + for usage in usage_snapshots + ) + # A later chunk without usage must not erase the last reported counts. + payloads.append({"choices": [{"delta": {"content": "!"}}], "usage": None}) + payloads.append({"choices": [{"delta": {}}]}) + if chunk_format == "to_dict": + chunks = [ + Mock(spec=["to_dict"], to_dict=Mock(return_value=payload)) + for payload in payloads + ] + elif chunk_format == "attributes": + chunks = [SimpleNamespace(**payload) for payload in payloads] + else: + chunks = payloads + + provider = TracerProvider() + exporter = InMemorySpanExporter() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + span = provider.get_tracer(__name__).start_span("completion") + span_cm = MagicMock(spec=["__exit__"]) + span_cm.__exit__.side_effect = lambda *_: span.end() + + try: + if async_stream: + async def consume(): + async def source(): + for chunk in chunks: + yield chunk + + wrapper = AsyncStreamingIteratorWrapper(source(), span, span_cm) + return [chunk async for chunk in wrapper] + + received = asyncio.run(consume()) + else: + received = list(StreamingIteratorWrapper(iter(chunks), span, span_cm)) + + assert len(received) == len(chunks) + assert all(actual is original for actual, original in zip(received, chunks)) + span_cm.__exit__.assert_called_once_with(None, None, None) + finished = exporter.get_finished_spans() + assert len(finished) == 1 + assert finished[0].status.status_code == StatusCode.OK + assert finished[0].attributes["output.value"] == "hello!" + assert { + name: value + for name, value in finished[0].attributes.items() + if name.startswith("gen_ai.usage.") + } == {f"gen_ai.usage.{name}": value for name, value in expected.items()} + finally: + if span.is_recording(): + span.end() + provider.shutdown() diff --git a/python/frameworks/litellm/tests/test_framework_litellm_structured_output.py b/python/frameworks/litellm/tests/test_framework_litellm_structured_output.py new file mode 100644 index 000000000..afcce63df --- /dev/null +++ b/python/frameworks/litellm/tests/test_framework_litellm_structured_output.py @@ -0,0 +1,190 @@ +"""T3-LITELLM-002: request telemetry must not reject valid completion arguments.""" + +import asyncio +import json +import os +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock, Mock, patch + +import httpx +import pytest +from opentelemetry import context as context_api +from opentelemetry import trace as trace_api +from opentelemetry.context import _SUPPRESS_INSTRUMENTATION_KEY +from pydantic import BaseModel + +# Import LiteLLM without fetching its remote model-cost map. All calls are mocked. +with patch.dict(os.environ, {"LITELLM_LOCAL_MODEL_COST_MAP": "True"}): + from traceai_litellm import LiteLLMInstrumentor, SpanAttributes + + +class Answer(BaseModel): + value: int + + +class Unprintable: + def __str__(self): + raise ValueError("cannot render telemetry") + + +@pytest.fixture( + params=[ + pytest.param(("completion", False), id="sync"), + pytest.param(("acompletion", False), id="async"), + pytest.param(("completion", True), id="sync-stream"), + pytest.param(("acompletion", True), id="async-stream"), + pytest.param(("completion_with_retries", False), id="sync-retries"), + pytest.param(("acompletion_with_retries", False), id="async-retries"), + ] +) +def completion_case(request, monkeypatch): + function_name, streaming = request.param + span = MagicMock(spec=trace_api.Span) + tracer = MagicMock() + tracer.start_as_current_span.return_value.__enter__.return_value = span + instrumentor = LiteLLMInstrumentor() + monkeypatch.setattr(instrumentor, "_tracer", tracer, raising=False) + + response = {"choices": [{"delta": {"content": '{"value":1}'}}]} + + async def async_chunks(): + yield response + + is_async = function_name.startswith("a") + original = AsyncMock() if is_async else Mock() + original.return_value = ( + (async_chunks() if is_async else iter([response])) if streaming else response + ) + monkeypatch.setattr(instrumentor, "original_litellm_funcs", {function_name: original}) + token = context_api.attach( + context_api.set_value(_SUPPRESS_INSTRUMENTATION_KEY, False) + ) + try: + yield SimpleNamespace( + wrapper=getattr(instrumentor, f"_{function_name}_wrapper"), + is_async=is_async, + streaming=streaming, + original=original, + response=response, + span=span, + span_context=tracer.start_as_current_span.return_value, + kwargs={ + "model": "gpt-4o-mini", + "messages": [{"role": "user", "content": "Return value 1"}], + "stream": streaming, + "temperature": 0.2, + }, + ) + finally: + context_api.detach(token) + + +def _call_completion(case): + if case.is_async: + async def call(): + result = await case.wrapper(**case.kwargs) + return [chunk async for chunk in result] if case.streaming else result + + return asyncio.run(call()) + result = case.wrapper(**case.kwargs) + return list(result) if case.streaming else result + + +def _assert_forwarded_once(case): + case.original.assert_called_once_with(**case.kwargs) + if case.is_async: + case.original.assert_awaited_once_with(**case.kwargs) + assert case.original.call_args.kwargs["messages"] is case.kwargs["messages"] + + +def _assert_success(case): + result = _call_completion(case) + if case.streaming: + assert len(result) == 1 + assert result[0] is case.response + else: + assert result is case.response + _assert_forwarded_once(case) + case.span.record_exception.assert_not_called() + + +def _request_parameters(case): + attributes = dict(call.args for call in case.span.set_attribute.call_args_list) + return json.loads(attributes[SpanAttributes.GEN_AI_REQUEST_PARAMETERS]) + + +@pytest.mark.parametrize("parameter", ["response_format", "timeout"]) +def test_supported_non_json_parameters_preserve_completion(completion_case, parameter): + case = completion_case + value = Answer if parameter == "response_format" else httpx.Timeout(10.0) + case.kwargs[parameter] = value + + _assert_success(case) + + # Serialization must operate on telemetry copies, not the provider's arguments. + assert case.original.call_args.kwargs[parameter] is value + parameters = _request_parameters(case) + # Opaque/structured provider options are deliberately excluded by the + # credential-safe inference allowlist, but still reach LiteLLM unchanged. + assert parameter not in parameters + assert parameters["temperature"] == 0.2 + assert "model" not in parameters + assert "messages" not in parameters + + +def test_json_response_format_is_unchanged(completion_case): + case = completion_case + response_format = { + "type": "json_schema", + "json_schema": {"name": "Answer", "schema": Answer.model_json_schema()}, + } + case.kwargs["response_format"] = response_format + + _assert_success(case) + + assert case.original.call_args.kwargs["response_format"] is response_format + assert "response_format" not in _request_parameters(case) + + +@pytest.mark.parametrize("failure", ["unprintable", "circular", "message-extraction"]) +def test_request_attribute_failures_do_not_block_completion( + completion_case, monkeypatch, failure +): + case = completion_case + if failure == "unprintable": + case.kwargs["metadata"] = Unprintable() + elif failure == "circular": + circular = {} + circular["self"] = circular + case.kwargs["metadata"] = circular + else: + monkeypatch.setattr( + "traceai_litellm._process_messages", + Mock(side_effect=ValueError("cannot extract telemetry")), + ) + + _assert_success(case) + + +def test_application_error_is_preserved(completion_case): + case = completion_case + case.kwargs["response_format"] = Answer + error = RuntimeError("provider failure") + case.original.side_effect = error + + with pytest.raises(RuntimeError, match="provider failure") as raised: + _call_completion(case) + + assert raised.value is error + _assert_forwarded_once(case) + if case.streaming: + # The real span context records the exception exactly once. A mock + # context must receive the same exception rather than record it twice. + case.span_context.__exit__.assert_called_once() + exc_type, exc_value, traceback = case.span_context.__exit__.call_args.args + assert exc_type is type(error) + assert exc_value is error + assert traceback is not None + case.span.record_exception.assert_not_called() + else: + case.span.record_exception.assert_called_once_with(error) diff --git a/python/frameworks/litellm/tests/test_litellm_credential_parameters.py b/python/frameworks/litellm/tests/test_litellm_credential_parameters.py new file mode 100644 index 000000000..215eca5f8 --- /dev/null +++ b/python/frameworks/litellm/tests/test_litellm_credential_parameters.py @@ -0,0 +1,386 @@ +"""T3-LITELLM-001/001b: provider authentication is not request telemetry. + +Provider functions are mocked; spans are exported only to memory. Run with the +local LiteLLM package on PYTHONPATH and LITELLM_LOCAL_MODEL_COST_MAP=True. +""" + +import asyncio +import copy +import json +from types import MethodType, SimpleNamespace +from unittest.mock import AsyncMock, Mock + +import pytest +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter + +from fi_instrumentation import FITracer, TraceConfig +from fi_instrumentation.fi_types import SpanAttributes + + +_FAKE_CREDENTIAL = "FAKE_PROVIDER_CREDENTIAL_ONLY" + + +@pytest.fixture +def instrumentation(monkeypatch): + # LiteLLM must not download its model cost map even during import. + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + import traceai_litellm + + return traceai_litellm + + +@pytest.fixture +def memory_tracer(clean_env): + exporter = InMemorySpanExporter() + provider = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + tracer = FITracer(provider.get_tracer(__name__), config=TraceConfig()) + try: + yield tracer, exporter + finally: + provider.shutdown() + + +def _wrapper_harness(instrumentation, tracer, functions): + harness = SimpleNamespace(_tracer=tracer, original_litellm_funcs=functions) + if hasattr(instrumentation.LiteLLMInstrumentor, "_get_call_arguments"): + harness._get_call_arguments = MethodType( + instrumentation.LiteLLMInstrumentor._get_call_arguments, harness + ) + return harness + + +def _completion_kwargs(with_messages=True): + kwargs = { + "model": "gpt-4o-mini", + "temperature": 0.7, + "max_tokens": 20, + "stop": ["END"], + "api_key": _FAKE_CREDENTIAL, + "extra_headers": { + "Authorization": "Bearer " + _FAKE_CREDENTIAL, + "X-API-Key": _FAKE_CREDENTIAL, + }, + "azure_ad_token": _FAKE_CREDENTIAL, + "aws_secret_access_key": _FAKE_CREDENTIAL, + "extra_body": { + "provider_options": [ + {"api_key": _FAKE_CREDENTIAL}, + {"extra_headers": {"Authorization": _FAKE_CREDENTIAL}}, + ] + }, + "metadata": {"provider": {"credentials": _FAKE_CREDENTIAL}}, + "unknown_provider_option": _FAKE_CREDENTIAL, + } + if with_messages: + kwargs["messages"] = [{"role": "user", "content": "hello"}] + return kwargs + + +def _assert_no_credentials(values): + assert not any(_FAKE_CREDENTIAL in str(value) for value in values), ( + "Provider authentication was captured in a span attribute" + ) + + +@pytest.mark.parametrize( + "method,stream", + [ + pytest.param("completion", False, id="sync"), + pytest.param("acompletion", False, id="async"), + pytest.param("completion_with_retries", False, id="sync-retry"), + pytest.param("acompletion_with_retries", False, id="async-retry"), + pytest.param("completion", True, id="sync-stream"), + pytest.param("acompletion", True, id="async-stream"), + ], +) +@pytest.mark.parametrize("with_messages", [True, False], ids=["messages", "no-messages"]) +def test_completion_provider_credentials_not_exported( + instrumentation, memory_tracer, method, stream, with_messages +): + tracer, exporter = memory_tracer + kwargs = _completion_kwargs(with_messages) + kwargs["stream"] = stream + before = copy.deepcopy(kwargs) + response = {"choices": [{"message": {"role": "assistant", "content": "hello"}}]} + chunk = {"choices": [{"delta": {"content": "hello"}}]} + + async def async_chunks(): + yield chunk + + if method.startswith("a"): + original = AsyncMock(return_value=async_chunks() if stream else response) + else: + original = Mock(return_value=iter([chunk]) if stream else response) + harness = _wrapper_harness(instrumentation, tracer, {method: original}) + wrapper = getattr(instrumentation.LiteLLMInstrumentor, "_" + method + "_wrapper") + + if method.startswith("a"): + async def invoke(): + result = await wrapper(harness, **kwargs) + if stream: + chunks = [item async for item in result] + assert len(chunks) == 1 and chunks[0] is chunk + else: + assert result is response + + asyncio.run(invoke()) + original.assert_awaited_once_with(**before) + else: + result = wrapper(harness, **kwargs) + if stream: + chunks = list(result) + assert len(chunks) == 1 and chunks[0] is chunk + else: + assert result is response + original.assert_called_once_with(**before) + + # Filtering is a telemetry-only projection, never a change to provider input. + assert kwargs == before + received = original.call_args.kwargs + assert received["extra_headers"] is kwargs["extra_headers"] + assert received["extra_body"] is kwargs["extra_body"] + spans = exporter.get_finished_spans() + assert len(spans) == 1 + attributes = spans[0].attributes + _assert_no_credentials(attributes.values()) + assert json.loads(attributes[SpanAttributes.GEN_AI_REQUEST_PARAMETERS]) == { + "temperature": 0.7, + "max_tokens": 20, + "stop": ["END"], + "stream": stream, + } + assert attributes[SpanAttributes.GEN_AI_REQUEST_MODEL] == "gpt-4o-mini" + if with_messages: + assert json.loads(attributes[SpanAttributes.INPUT_VALUE]) == kwargs["messages"] + + +@pytest.mark.parametrize("with_messages", [True, False], ids=["messages", "no-messages"]) +def test_every_completion_attribute_write_excludes_credentials(instrumentation, with_messages): + # A final input.value overwrite must not conceal an earlier unsafe write. + span = Mock() + kwargs = _completion_kwargs(with_messages) + before = copy.deepcopy(kwargs) + instrumentation._instrument_func_type_completion(span, kwargs) + values = [call.args[1] for call in span.set_attribute.call_args_list] + _assert_no_credentials(values) + assert kwargs == before + + +def test_nested_options_and_clients_are_not_serialized( + instrumentation, memory_tracer +): + class CredentialClient: + def __init__(self): + self.render_count = 0 + + def __str__(self): + self.render_count += 1 + return _FAKE_CREDENTIAL + + __repr__ = __str__ + + client = CredentialClient() + kwargs = _completion_kwargs() + kwargs.update({ + "client": client, + "http_client": client, + "extra_body": {"options": [{"client": client}]}, + # Even allowlisted names must not admit arbitrary nested data or objects. + "temperature": {"api_key": _FAKE_CREDENTIAL}, + "max_tokens": client, + "stop": [{"Authorization": _FAKE_CREDENTIAL}], + "response_format": { + "type": "json_object", + "options": {"api_key": _FAKE_CREDENTIAL}, + }, + "seed": 7, + }) + tracer, exporter = memory_tracer + response = {"choices": []} + original = Mock(return_value=response) + harness = _wrapper_harness(instrumentation, tracer, {"completion": original}) + result = instrumentation.LiteLLMInstrumentor._completion_wrapper(harness, **kwargs) + + assert result is response + original.assert_called_once_with(**kwargs) + assert original.call_args.kwargs["client"] is client + assert original.call_args.kwargs["extra_body"] is kwargs["extra_body"] + assert client.render_count == 0 + attributes = exporter.get_finished_spans()[0].attributes + _assert_no_credentials(attributes.values()) + assert json.loads(attributes[SpanAttributes.GEN_AI_REQUEST_PARAMETERS]) == {"seed": 7} + + +def test_safe_inference_parameters_remain_visible(instrumentation): + span = Mock() + parameters = { + "temperature": 0.0, + "top_p": 0.9, + "top_k": 5, + "n": 2, + "max_tokens": 20, + "max_completion_tokens": 30, + "frequency_penalty": 0.0, + "presence_penalty": 0.2, + "seed": 0, + "logprobs": False, + "top_logprobs": 3, + "stream": False, + "parallel_tool_calls": True, + "reasoning_effort": "low", + "stop": ["END"], + } + instrumentation._instrument_func_type_completion( + span, {"model": "gpt-4o-mini", **parameters} + ) + attributes = dict(call.args for call in span.set_attribute.call_args_list) + assert json.loads(attributes[SpanAttributes.GEN_AI_REQUEST_PARAMETERS]) == parameters + + +def _input_provider_options(): + return { + "api_key": _FAKE_CREDENTIAL, + "headers": {"Authorization": "Bearer " + _FAKE_CREDENTIAL}, + "extra_headers": {"X-API-Key": _FAKE_CREDENTIAL}, + "metadata": {"provider": {"credentials": _FAKE_CREDENTIAL}}, + } + + +@pytest.mark.parametrize("method", ["image_generation", "aimage_generation"]) +@pytest.mark.parametrize("positional", [True, False], ids=["positional", "keyword"]) +@pytest.mark.parametrize("prompt", ["draw a blue circle", ""], ids=["text", "empty"]) +def test_image_provider_credentials_not_exported( + instrumentation, memory_tracer, method, positional, prompt +): + tracer, exporter = memory_tracer + kwargs = {"model": "dall-e-2", **_input_provider_options()} + args = (prompt,) if positional else () + if not positional: + kwargs["prompt"] = prompt + before = copy.deepcopy(kwargs) + response = {"data": [{"url": "https://example.invalid/image.png"}]} + + # Prompt is positional-or-keyword in LiteLLM; only the provider body is fake. + def generate(prompt, model=None, **options): + return response + + original = ( + AsyncMock(side_effect=generate) + if method.startswith("a") + else Mock(side_effect=generate) + ) + harness = _wrapper_harness(instrumentation, tracer, {method: original}) + wrapper = getattr(instrumentation.LiteLLMInstrumentor, "_" + method + "_wrapper") + if method.startswith("a"): + result = asyncio.run(wrapper(harness, *args, **kwargs)) + original.assert_awaited_once_with(*args, **before) + else: + result = wrapper(harness, *args, **kwargs) + original.assert_called_once_with(*args, **before) + + assert result is response + assert kwargs == before + for name in ("headers", "extra_headers", "metadata"): + assert original.call_args.kwargs[name] is kwargs[name] + spans = exporter.get_finished_spans() + assert len(spans) == 1 + attributes = spans[0].attributes + _assert_no_credentials(attributes.values()) + assert attributes[SpanAttributes.GEN_AI_REQUEST_MODEL] == "dall-e-2" + if prompt: + assert attributes[SpanAttributes.INPUT_VALUE] == prompt + else: + assert SpanAttributes.INPUT_VALUE not in attributes + + +@pytest.mark.parametrize("method", ["embedding", "aembedding"]) +def test_embedding_provider_credentials_not_exported( + instrumentation, memory_tracer, method +): + # Control: these helpers already overwrite the raw kwargs in ended spans. + tracer, exporter = memory_tracer + kwargs = { + "model": "text-embedding-3-small", + "input": ["hello"], + **_input_provider_options(), + } + before = copy.deepcopy(kwargs) + response = {"data": [{"embedding": [0.1, 0.2]}]} + original = ( + AsyncMock(return_value=response) + if method.startswith("a") + else Mock(return_value=response) + ) + harness = _wrapper_harness(instrumentation, tracer, {method: original}) + wrapper = getattr(instrumentation.LiteLLMInstrumentor, "_" + method + "_wrapper") + if method.startswith("a"): + result = asyncio.run(wrapper(harness, **kwargs)) + original.assert_awaited_once_with(**before) + else: + result = wrapper(harness, **kwargs) + original.assert_called_once_with(**before) + + assert result is response + assert kwargs == before + for name in ("input", "headers", "extra_headers", "metadata"): + assert original.call_args.kwargs[name] is kwargs[name] + spans = exporter.get_finished_spans() + assert len(spans) == 1 + attributes = spans[0].attributes + _assert_no_credentials(attributes.values()) + assert attributes[SpanAttributes.INPUT_VALUE] == str(kwargs["input"]) + + +@pytest.mark.parametrize( + "helper,content", + [ + ("_instrument_func_type_image_generation", {"prompt": "draw a blue circle"}), + ("_instrument_func_type_image_generation", {}), + ("_instrument_func_type_embedding", {"input": ["hello"]}), + ], + ids=["image-keyword", "image-missing-prompt", "embedding"], +) +def test_every_input_attribute_write_excludes_credentials( + instrumentation, helper, content +): + # Neither a later overwrite nor a truthy prompt may hide a raw-kwargs write. + span = Mock() + kwargs = {"model": "example-model", **content, **_input_provider_options()} + before = copy.deepcopy(kwargs) + getattr(instrumentation, helper)(span, kwargs) + _assert_no_credentials(call.args[1] for call in span.set_attribute.call_args_list) + assert kwargs == before + + +@pytest.mark.parametrize( + "helper,content", + [ + ("_instrument_func_type_image_generation", {"prompt": "draw a blue circle"}), + ("_instrument_func_type_embedding", {"input": ["hello"]}), + ], +) +def test_input_helpers_do_not_render_provider_clients(instrumentation, helper, content): + class CredentialClient: + def __str__(self): + raise AssertionError("Provider clients must not be rendered for telemetry") + + __repr__ = __str__ + + client = CredentialClient() + options = {"provider": {"client": client, "api_key": _FAKE_CREDENTIAL}} + kwargs = { + "model": "example-model", + **content, + "client": client, + "extra_body": options, + "metadata": options, + } + span = Mock() + getattr(instrumentation, helper)(span, kwargs) + _assert_no_credentials(call.args[1] for call in span.set_attribute.call_args_list) + assert kwargs["client"] is client + assert kwargs["extra_body"] is options + assert kwargs["metadata"] is options diff --git a/python/frameworks/litellm/traceai_litellm/__init__.py b/python/frameworks/litellm/traceai_litellm/__init__.py index 69b981964..483cedf88 100644 --- a/python/frameworks/litellm/traceai_litellm/__init__.py +++ b/python/frameworks/litellm/traceai_litellm/__init__.py @@ -1,6 +1,7 @@ import json from enum import Enum from functools import wraps +from inspect import Parameter, signature from typing import ( Any, AsyncIterator, @@ -45,6 +46,7 @@ from opentelemetry.context import _SUPPRESS_INSTRUMENTATION_KEY from opentelemetry.instrumentation.instrumentor import BaseInstrumentor # type: ignore from opentelemetry.util.types import AttributeValue +from pydantic import BaseModel from traceai_litellm.package import _instruments from traceai_litellm.version import __version__ from wrapt import wrap_function_wrapper @@ -56,6 +58,23 @@ def _set_span_attribute(span: trace_api.Span, name: str, value: AttributeValue) span.set_attribute(name, value) +def _set_streaming_usage_attributes(span: trace_api.Span, usage: Any) -> None: + if usage is None: + return + # Stream usage is cumulative: overwrite reported counts rather than sum them. + for field, attribute in ( + ("prompt_tokens", SpanAttributes.GEN_AI_USAGE_INPUT_TOKENS), + ("completion_tokens", SpanAttributes.GEN_AI_USAGE_OUTPUT_TOKENS), + ("total_tokens", SpanAttributes.GEN_AI_USAGE_TOTAL_TOKENS), + ): + value = ( + usage.get(field) + if isinstance(usage, Mapping) + else getattr(usage, field, None) + ) + _set_span_attribute(span, attribute, value) + + T = TypeVar("T", bound=type) @@ -108,6 +127,49 @@ def _get_attributes_from_image( yield f"{ImageAttributes.IMAGE_URL}", url +# Completion kwargs also contain provider credentials, headers and clients. +# Capture only known inference controls, never arbitrary provider-specific data. +_COMPLETION_PARAMETER_TYPES = { + "temperature": (int, float), + "top_p": (int, float), + "top_k": (int,), + "n": (int,), + "max_tokens": (int,), + "max_completion_tokens": (int,), + "frequency_penalty": (int, float), + "presence_penalty": (int, float), + "seed": (int,), + "logprobs": (bool, int), + "top_logprobs": (int,), + "stream": (bool,), + "parallel_tool_calls": (bool,), + "reasoning_effort": (str,), + "stop": (str,), +} + + +def _get_completion_invocation_parameters(kwargs: Dict[str, Any]) -> Dict[str, Any]: + """Project telemetry without inspecting or mutating provider configuration. + + Omit opaque mappings/sequences entirely (including extra_body, metadata, + headers and structured provider options), rather than guessing credential + names at each nesting level. Exact builtin types also prevent serializing + client objects or subclasses with custom string representations. + """ + parameters = {} + for name, allowed_types in _COMPLETION_PARAMETER_TYPES.items(): + if name in kwargs: + value = kwargs[name] + if value is None or type(value) in allowed_types: + parameters[name] = value + + # Stop sequences are the only supported container, and cannot nest objects. + stop = kwargs.get("stop") + if type(stop) in (list, tuple) and all(type(item) is str for item in stop): + parameters["stop"] = list(stop) + return parameters + + def _instrument_func_type_completion( span: trace_api.Span, kwargs: Dict[str, Any] ) -> None: @@ -118,7 +180,20 @@ def _instrument_func_type_completion( litellm.completion_with_retries() litellm.acompletion_with_retries() (async version of completion_with_retries) """ - _set_span_attribute(span, SpanAttributes.INPUT_VALUE, safe_json_dumps(kwargs)) + try: + _set_completion_attributes(span, kwargs) + except Exception: + # Request telemetry must never prevent the wrapped provider call. Even + # safe_json_dumps can fail on circular values or a failing __str__. + logger.warning("Failed to extract LiteLLM completion request attributes") + + +def _set_completion_attributes(span: trace_api.Span, kwargs: Dict[str, Any]) -> None: + invocation_params = _get_completion_invocation_parameters(kwargs) + # Never write raw kwargs, even temporarily before the message-derived input. + _set_span_attribute( + span, SpanAttributes.INPUT_VALUE, safe_json_dumps(invocation_params) + ) _set_span_attribute(span, SpanAttributes.GEN_AI_SPAN_KIND, FiSpanKindValues.LLM.value) _set_span_attribute( span, SpanAttributes.GEN_AI_REQUEST_MODEL, kwargs.get("model", "unknown_model") @@ -127,17 +202,14 @@ def _instrument_func_type_completion( if messages := kwargs.get("messages"): process_messages = _process_messages(messages) if filtered_messages := process_messages.get("filtered_messages"): + # Keep the full conversation as canonical input, not a derived text/query. _set_span_attribute( - span, SpanAttributes.INPUT_VALUE, json.dumps(filtered_messages) + span, SpanAttributes.INPUT_VALUE, safe_json_dumps(filtered_messages) ) if input_images := process_messages.get("input_images"): _set_span_attribute( - span, SpanAttributes.INPUT_IMAGES, json.dumps(input_images) + span, SpanAttributes.INPUT_IMAGES, safe_json_dumps(input_images) ) - if eval_input := process_messages.get("eval_input"): - _set_span_attribute(span, SpanAttributes.INPUT_VALUE, eval_input) - if query := process_messages.get("query"): - _set_span_attribute(span, SpanAttributes.INPUT_VALUE, query) for index, input_message in list(enumerate(messages)): for key, value in _get_attributes_from_message_param(input_message): @@ -145,11 +217,12 @@ def _instrument_func_type_completion( span, f"{SpanAttributes.GEN_AI_INPUT_MESSAGES}.{index}.{key}", value ) - invocation_params = { - k: v for k, v in kwargs.items() if k not in ["model", "messages"] - } + response_format = invocation_params.get("response_format") + if isinstance(response_format, type) and issubclass(response_format, BaseModel): + # Normalize only the telemetry copy; LiteLLM needs the original class. + invocation_params["response_format"] = response_format.model_json_schema() _set_span_attribute( - span, SpanAttributes.GEN_AI_REQUEST_PARAMETERS, json.dumps(invocation_params) + span, SpanAttributes.GEN_AI_REQUEST_PARAMETERS, safe_json_dumps(invocation_params) ) @@ -161,7 +234,7 @@ def _instrument_func_type_embedding( litellm.embedding() litellm.aembedding() (async version of embedding) """ - _set_span_attribute(span, SpanAttributes.INPUT_VALUE, safe_json_dumps(kwargs)) + # Only the embedding input is telemetry; provider kwargs may contain secrets. _set_span_attribute( span, SpanAttributes.GEN_AI_SPAN_KIND, @@ -179,18 +252,20 @@ def _instrument_func_type_embedding( def _instrument_func_type_image_generation( - span: trace_api.Span, kwargs: Dict[str, Any] + span: trace_api.Span, kwargs: Dict[str, Any], args: Tuple[Any, ...] = () ) -> None: """ Currently instruments the functions: litellm.image_generation() litellm.aimage_generation() (async version of image_generation) """ - _set_span_attribute(span, SpanAttributes.INPUT_VALUE, safe_json_dumps(kwargs)) _set_span_attribute(span, SpanAttributes.GEN_AI_SPAN_KIND, FiSpanKindValues.LLM.value) if model := kwargs.get("model"): _set_span_attribute(span, SpanAttributes.GEN_AI_REQUEST_MODEL, model) - if prompt := kwargs.get("prompt"): + # LiteLLM's prompt is positional-or-keyword, including through its async API. + # Project only this input; never serialize the remaining args/provider kwargs. + prompt = args[0] if args else kwargs.get("prompt") + if prompt: _set_span_attribute(span, SpanAttributes.INPUT_VALUE, str(prompt)) @@ -232,18 +307,32 @@ def _finalize_span(span: trace_api.Span, result: Any) -> None: ): _set_span_attribute(span, f"{SpanAttributes.GEN_AI_OUTPUT_MESSAGES}.{0}.{MessageAttributes.MESSAGE_CONTENT}.{idx}.{MessageContentAttributes.MESSAGE_CONTENT_IMAGE}", url) _set_span_attribute(span, SpanAttributes.OUTPUT_VALUE, url) - if hasattr(result, "usage"): - _set_span_attribute( - span, SpanAttributes.GEN_AI_USAGE_INPUT_TOKENS, result.usage["prompt_tokens"] - ) - _set_span_attribute( - span, + usage = getattr(result, "usage", None) + if usage is None: + return + # Image usage has different field names; retain legacy chat-style fallbacks. + # Missing or unrecognized usage is missing telemetry, not an application error. + is_image = isinstance(result, ImageResponse) + for attribute, keys in ( + ( + SpanAttributes.GEN_AI_USAGE_INPUT_TOKENS, + ("input_tokens", "prompt_tokens") if is_image else ("prompt_tokens",), + ), + ( SpanAttributes.GEN_AI_USAGE_OUTPUT_TOKENS, - result.usage["completion_tokens"], - ) - _set_span_attribute( - span, SpanAttributes.GEN_AI_USAGE_TOTAL_TOKENS, result.usage["total_tokens"] - ) + ("output_tokens", "completion_tokens") if is_image else ("completion_tokens",), + ), + (SpanAttributes.GEN_AI_USAGE_TOTAL_TOKENS, ("total_tokens",)), + ): + for key in keys: + value = ( + usage.get(key) + if isinstance(usage, Mapping) + else getattr(usage, key, None) + ) + if value is not None: + _set_span_attribute(span, attribute, value) + break class LiteLLMInstrumentor(BaseInstrumentor): # type: ignore @@ -307,17 +396,55 @@ def _uninstrument(self, **kwargs: Any) -> None: self.original_litellm_funcs.clear() self._tracer = None + def _get_call_arguments( + self, func_name: str, args: Tuple[Any, ...], kwargs: Dict[str, Any] + ) -> Dict[str, Any]: + """Bind tracing inputs without changing the call or adding defaults.""" + if not args: + return kwargs + positional_kinds = (Parameter.POSITIONAL_ONLY, Parameter.POSITIONAL_OR_KEYWORD) + try: + call_signature = signature(self.original_litellm_funcs[func_name]) + if not any( + parameter.kind in positional_kinds + for parameter in call_signature.parameters.values() + ): + # Variadic async/retry entry points forward the same positional + # arguments to the corresponding synchronous API. + base_name = { + "acompletion": "completion", + "completion_with_retries": "completion", + "acompletion_with_retries": "completion", + "aembedding": "embedding", + "aimage_generation": "image_generation", + }.get(func_name) + if base_name is not None and base_name in self.original_litellm_funcs: + call_signature = signature(self.original_litellm_funcs[base_name]) + bound = call_signature.bind_partial(*args, **kwargs) + except (TypeError, ValueError): + # Unavailable signatures or invalid arguments must not prevent the + # original callable from returning or raising its own exception. + return kwargs + arguments = dict(kwargs) + arguments.update( + (name, value) + for name, value in bound.arguments.items() + if call_signature.parameters[name].kind in positional_kinds + ) + return arguments + @wraps(litellm.completion) def _completion_wrapper(self, *args: Any, **kwargs: Any) -> Any: if context_api.get_value(_SUPPRESS_INSTRUMENTATION_KEY): return self.original_litellm_funcs["completion"](*args, **kwargs) - streaming = kwargs.get("stream", False) + arguments = self._get_call_arguments("completion", args, kwargs) + streaming = arguments.get("stream", False) if not streaming: with self._tracer.start_as_current_span( name="completion", attributes=dict(get_attributes_from_context()) ) as span: try: - _instrument_func_type_completion(span, kwargs) + _instrument_func_type_completion(span, arguments) result = self.original_litellm_funcs["completion"](*args, **kwargs) _finalize_span(span, result) span.set_status(trace_api.StatusCode.OK) @@ -331,26 +458,27 @@ def _completion_wrapper(self, *args: Any, **kwargs: Any) -> Any: ) span = span_cm.__enter__() try: - _instrument_func_type_completion(span, kwargs) + _instrument_func_type_completion(span, arguments) original_iterator = self.original_litellm_funcs["completion"]( *args, **kwargs ) return StreamingIteratorWrapper(original_iterator, span, span_cm) - except Exception as e: - span.record_exception(e) + except BaseException as e: + span_cm.__exit__(type(e), e, e.__traceback__) raise @wraps(litellm.acompletion) async def _acompletion_wrapper(self, *args: Any, **kwargs: Any) -> Any: if context_api.get_value(_SUPPRESS_INSTRUMENTATION_KEY): return await self.original_litellm_funcs["acompletion"](*args, **kwargs) - streaming = kwargs.get("stream", False) + arguments = self._get_call_arguments("acompletion", args, kwargs) + streaming = arguments.get("stream", False) if not streaming: with self._tracer.start_as_current_span( name="acompletion", attributes=dict(get_attributes_from_context()) ) as span: try: - _instrument_func_type_completion(span, kwargs) + _instrument_func_type_completion(span, arguments) result = await self.original_litellm_funcs["acompletion"]( *args, **kwargs ) @@ -366,20 +494,16 @@ async def _acompletion_wrapper(self, *args: Any, **kwargs: Any) -> Any: ) span = span_cm.__enter__() try: - _instrument_func_type_completion(span, kwargs) + _instrument_func_type_completion(span, arguments) original_async_iterator = await self.original_litellm_funcs[ "acompletion" ](*args, **kwargs) return AsyncStreamingIteratorWrapper( original_async_iterator, span, span_cm ) - except Exception as e: - span.set_status( - trace_api.StatusCode.ERROR, - description=f"{e.__class__.__name__}: {e}", - ) - span.record_exception(e) - span_cm.__exit__(e, type(e), e.__traceback__) + except BaseException as e: + # Context exit takes the exception type before the instance. + span_cm.__exit__(type(e), e, e.__traceback__) raise @wraps(litellm.completion_with_retries) @@ -390,12 +514,13 @@ def _completion_with_retries_wrapper( return self.original_litellm_funcs["completion_with_retries"]( *args, **kwargs ) # type:ignore + arguments = self._get_call_arguments("completion_with_retries", args, kwargs) with self._tracer.start_as_current_span( name="completion_with_retries", attributes=dict(get_attributes_from_context()), ) as span: try: - _instrument_func_type_completion(span, kwargs) + _instrument_func_type_completion(span, arguments) result = self.original_litellm_funcs["completion_with_retries"]( *args, **kwargs ) @@ -414,12 +539,13 @@ async def _acompletion_with_retries_wrapper( return await self.original_litellm_funcs["acompletion_with_retries"]( *args, **kwargs ) # type:ignore + arguments = self._get_call_arguments("acompletion_with_retries", args, kwargs) with self._tracer.start_as_current_span( name="acompletion_with_retries", attributes=dict(get_attributes_from_context()), ) as span: try: - _instrument_func_type_completion(span, kwargs) + _instrument_func_type_completion(span, arguments) result = await self.original_litellm_funcs[ "acompletion_with_retries" ](*args, **kwargs) @@ -436,11 +562,12 @@ def _embedding_wrapper(self, *args: Any, **kwargs: Any) -> EmbeddingResponse: return self.original_litellm_funcs["embedding"]( *args, **kwargs ) # type:ignore + arguments = self._get_call_arguments("embedding", args, kwargs) with self._tracer.start_as_current_span( name="embedding", attributes=dict(get_attributes_from_context()) ) as span: try: - _instrument_func_type_embedding(span, kwargs) + _instrument_func_type_embedding(span, arguments) result = self.original_litellm_funcs["embedding"](*args, **kwargs) _finalize_span(span, result) span.set_status(trace_api.StatusCode.OK) @@ -455,11 +582,12 @@ async def _aembedding_wrapper(self, *args: Any, **kwargs: Any) -> EmbeddingRespo return await self.original_litellm_funcs["aembedding"]( *args, **kwargs ) # type:ignore + arguments = self._get_call_arguments("aembedding", args, kwargs) with self._tracer.start_as_current_span( name="aembedding", attributes=dict(get_attributes_from_context()) ) as span: try: - _instrument_func_type_embedding(span, kwargs) + _instrument_func_type_embedding(span, arguments) result = await self.original_litellm_funcs["aembedding"]( *args, **kwargs ) @@ -476,11 +604,12 @@ def _image_generation_wrapper(self, *args: Any, **kwargs: Any) -> ImageResponse: return self.original_litellm_funcs["image_generation"]( *args, **kwargs ) # type:ignore + arguments = self._get_call_arguments("image_generation", args, kwargs) with self._tracer.start_as_current_span( name="image_generation", attributes=dict(get_attributes_from_context()) ) as span: try: - _instrument_func_type_image_generation(span, kwargs) + _instrument_func_type_image_generation(span, arguments, args) result = self.original_litellm_funcs["image_generation"]( *args, **kwargs ) @@ -499,11 +628,12 @@ async def _aimage_generation_wrapper( return await self.original_litellm_funcs["aimage_generation"]( *args, **kwargs ) # type:ignore + arguments = self._get_call_arguments("aimage_generation", args, kwargs) with self._tracer.start_as_current_span( name="aimage_generation", attributes=dict(get_attributes_from_context()) ) as span: try: - _instrument_func_type_image_generation(span, kwargs) + _instrument_func_type_image_generation(span, arguments, args) result = await self.original_litellm_funcs["aimage_generation"]( *args, **kwargs ) @@ -523,6 +653,7 @@ def __init__(self, iterator: Iterator[Any], span: trace_api.Span, span_cm): self.iterator = iterator self.span = span self.span_cm = span_cm + self._finished = False self.processed_chunks = [] self.content_string = "" @@ -530,6 +661,8 @@ def __iter__(self): return self def __next__(self): + if self._finished: + return next(self.iterator) try: chunk = next(self.iterator) # Process chunk @@ -541,9 +674,10 @@ def __next__(self): chunk_dict = chunk self.processed_chunks.append(chunk_dict) + _set_streaming_usage_attributes(self.span, chunk_dict.get("usage")) if ( - content := chunk_dict.get("choices", [{}])[0] + content := (chunk_dict.get("choices") or [{}])[0] .get("delta", {}) .get("content") ): @@ -551,29 +685,36 @@ def __next__(self): return chunk except StopIteration: - # After the iterator is exhausted, set the span attributes - _set_span_attribute( - self.span, - SpanAttributes.OUTPUT_VALUE, - safe_json_dumps(self.processed_chunks), - ) - _set_span_attribute( - self.span, SpanAttributes.OUTPUT_VALUE, self.content_string - ) - self.span.set_status(trace_api.StatusCode.OK) - # End the span - self.span_cm.__exit__(None, None, None) + try: + # After the iterator is exhausted, set the span attributes + _set_span_attribute( + self.span, + SpanAttributes.OUTPUT_VALUE, + safe_json_dumps(self.processed_chunks), + ) + _set_span_attribute( + self.span, SpanAttributes.OUTPUT_VALUE, self.content_string + ) + self.span.set_status(trace_api.StatusCode.OK) + finally: + self._finish_span() raise - except Exception as e: - self.span.record_exception(e) + except BaseException as e: + self._finish_span(type(e), e, e.__traceback__) raise + def _finish_span(self, exc_type=None, exc_value=None, traceback=None): + # Exhaustion, failed iteration and context exit can all reach cleanup. + if not self._finished: + self._finished = True + self.span_cm.__exit__(exc_type, exc_value, traceback) + def __enter__(self): return self def __exit__(self, exc_type, exc_value, traceback): # If the iterator exits early, we need to close the span - self.span_cm.__exit__(exc_type, exc_value, traceback) + self._finish_span(exc_type, exc_value, traceback) class AsyncStreamingIteratorWrapper: @@ -583,6 +724,7 @@ def __init__( self.async_iterator = async_iterator self.span = span self.span_cm = span_cm + self._finished = False self.processed_chunks = [] self.content_string = "" @@ -590,6 +732,8 @@ def __aiter__(self): return self async def __anext__(self): + if self._finished: + return await self.async_iterator.__anext__() try: chunk = await self.async_iterator.__anext__() # Process chunk @@ -601,9 +745,10 @@ async def __anext__(self): chunk_dict = chunk self.processed_chunks.append(chunk_dict) + _set_streaming_usage_attributes(self.span, chunk_dict.get("usage")) if ( - content := chunk_dict.get("choices", [{}])[0] + content := (chunk_dict.get("choices") or [{}])[0] .get("delta", {}) .get("content") ): @@ -611,26 +756,33 @@ async def __anext__(self): return chunk except StopAsyncIteration: - # After the iterator is exhausted, set the span attributes - _set_span_attribute( - self.span, - SpanAttributes.OUTPUT_VALUE, - safe_json_dumps(self.processed_chunks), - ) - _set_span_attribute( - self.span, SpanAttributes.OUTPUT_VALUE, self.content_string - ) - self.span.set_status(trace_api.StatusCode.OK) - # End the span - self.span_cm.__exit__(None, None, None) + try: + # After the iterator is exhausted, set the span attributes + _set_span_attribute( + self.span, + SpanAttributes.OUTPUT_VALUE, + safe_json_dumps(self.processed_chunks), + ) + _set_span_attribute( + self.span, SpanAttributes.OUTPUT_VALUE, self.content_string + ) + self.span.set_status(trace_api.StatusCode.OK) + finally: + self._finish_span() raise - except Exception as e: - self.span.record_exception(e) + except BaseException as e: + self._finish_span(type(e), e, e.__traceback__) raise + def _finish_span(self, exc_type=None, exc_value=None, traceback=None): + # A caller may close the stream after exhaustion, failure or cancellation. + if not self._finished: + self._finished = True + self.span_cm.__exit__(exc_type, exc_value, traceback) + async def close(self): # Explicitly close the span if needed - self.span_cm.__exit__(None, None, None) + self._finish_span() def _set_raw_output(span: trace_api.Span, result: Any) -> None: diff --git a/python/frameworks/litellm/traceai_litellm/version.py b/python/frameworks/litellm/traceai_litellm/version.py index 8c39d1009..d22c42508 100644 --- a/python/frameworks/litellm/traceai_litellm/version.py +++ b/python/frameworks/litellm/traceai_litellm/version.py @@ -1 +1 @@ -__version__ = "0.1.11" # x-release-please-version +__version__ = "0.1.12" # x-release-please-version diff --git a/python/frameworks/minimax/CHANGELOG.md b/python/frameworks/minimax/CHANGELOG.md new file mode 100644 index 000000000..0be8042f8 --- /dev/null +++ b/python/frameworks/minimax/CHANGELOG.md @@ -0,0 +1,8 @@ +## [0.1.1] - pending release + +### Changed +- traceai-minimax now requires fi-instrumentation-otel 1.0.0 or later. Action: Upgrade fi-instrumentation-otel if your environment pins an earlier version. + +### Fixed +- MiniMax requests that include a system message are traced. Previously their spans were silently dropped. +- Streamed MiniMax calls no longer raise a TypeError in your loop when chunks carry fields set to None, and the streamed span keeps its token usage when later chunks report none. diff --git a/python/frameworks/minimax/README.md b/python/frameworks/minimax/README.md index 6369047b8..9d0cb4f96 100644 --- a/python/frameworks/minimax/README.md +++ b/python/frameworks/minimax/README.md @@ -216,13 +216,12 @@ MiniMaxInstrumentor().instrument( | Attribute | Description | |-----------|-------------| -| `fi.span.kind` | "LLM" | -| `llm.system` | "minimax" | -| `llm.provider` | "minimax" | -| `llm.model` | Model name (MiniMax-M2.7, MiniMax-M2.7-highspeed, MiniMax-M2.5, MiniMax-M2.5-highspeed) | -| `llm.token_count.prompt` | Input token count | -| `llm.token_count.completion` | Output token count | -| `llm.token_count.total` | Total token count | +| `gen_ai.span.kind` | "LLM" | +| `gen_ai.provider.name` | "minimax" | +| `gen_ai.request.model` | Model name (MiniMax-M2.7, MiniMax-M2.7-highspeed, MiniMax-M2.5, MiniMax-M2.5-highspeed) | +| `gen_ai.usage.input_tokens` | Input token count | +| `gen_ai.usage.output_tokens` | Output token count | +| `gen_ai.usage.total_tokens` | Total token count | ### MiniMax-Specific Attributes diff --git a/python/frameworks/minimax/pyproject.toml b/python/frameworks/minimax/pyproject.toml index aee0ff6a1..dcb40d7d7 100644 --- a/python/frameworks/minimax/pyproject.toml +++ b/python/frameworks/minimax/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "traceAI-minimax" -version = "0.1.0" +version = "0.1.1" description = "OpenTelemetry instrumentation for MiniMax - chat completions via OpenAI-compatible API" authors = ["Future AGI "] readme = "README.md" @@ -27,7 +27,7 @@ classifiers = [ [tool.poetry.dependencies] python = ">=3.10,<3.14" -fi-instrumentation-otel = ">=0.1.11" +fi-instrumentation-otel = ">=1.0.0" openai = ">=1.0.0" [tool.poetry.urls] diff --git a/python/frameworks/minimax/tests/conftest.py b/python/frameworks/minimax/tests/conftest.py index 2fa183c89..49c2837a6 100644 --- a/python/frameworks/minimax/tests/conftest.py +++ b/python/frameworks/minimax/tests/conftest.py @@ -19,7 +19,6 @@ def create_mock_module(name): # Create a proper SpanAttributes class that returns string values class MockSpanAttributes: GEN_AI_PROVIDER_NAME = "llm.provider" - GEN_AI_PROVIDER_NAME_PROMPT = "llm.system_prompt" GEN_AI_REQUEST_MODEL = "llm.model" LLM_RESPONSE_MODEL = "llm.response_model" GEN_AI_REQUEST_PARAMETERS = "llm.invocation_parameters" diff --git a/python/frameworks/minimax/tests/test_real_core.py b/python/frameworks/minimax/tests/test_real_core.py new file mode 100644 index 000000000..e66466540 --- /dev/null +++ b/python/frameworks/minimax/tests/test_real_core.py @@ -0,0 +1,203 @@ +"""Exercise the installed core in fresh interpreters, outside conftest's mocks.""" + +import json +import os +from pathlib import Path +import subprocess +import sys + +import pytest + + +_PACKAGE_DIR = Path(__file__).resolve().parents[1] +_PROBE = r''' +import ast +import importlib.metadata +import json +from pathlib import Path +import sys +import traceback +from unittest.mock import patch + + +def deny_network(event, args): + if event in {"socket.connect", "socket.getaddrinfo", "socket.sendto"}: + raise AssertionError("Real-core regression checks must stay offline") + + +sys.addaudithook(deny_network) + +from fi_instrumentation import TraceConfig +from fi_instrumentation import fi_types +from fi_instrumentation.fi_types import MessageAttributes, SpanAttributes +from opentelemetry import trace +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +from opentelemetry.sdk.trace.sampling import ALWAYS_ON +import traceai_minimax +from traceai_minimax import MiniMaxInstrumentor +from traceai_minimax._request_attributes_extractor import ( + _ChatCompletionRequestAttributesExtractor, +) + +SYSTEM_CONTENT = "Synthetic system instruction: answer in short sentences." +USER_CONTENT = "Synthetic user question." +REQUEST = { + "model": "synthetic-model", + "messages": [ + {"role": "system", "content": SYSTEM_CONTENT}, + {"role": "user", "content": USER_CONTENT}, + ], +} +ROLE_KEY = f"{SpanAttributes.GEN_AI_INPUT_MESSAGES}.0.{MessageAttributes.MESSAGE_ROLE}" +CONTENT_KEY = f"{SpanAttributes.GEN_AI_INPUT_MESSAGES}.0.{MessageAttributes.MESSAGE_CONTENT}" + + +def check_constants(): + class_names = { + "SpanAttributes", "MessageAttributes", "ToolCallAttributes", + "EmbeddingAttributes", "FiSpanKindValues", "FiMimeTypeValues", + } + package_dir = Path(traceai_minimax.__file__).parent + files = sorted(package_dir.rglob("*.py")) + missing = [] + checked = 0 + for source in files: + tree = ast.parse(source.read_text(encoding="utf-8"), filename=source.name) + aliases = {} + for node in ast.walk(tree): + if isinstance(node, ast.ImportFrom) and node.module == "fi_instrumentation.fi_types": + for name in node.names: + if name.name in class_names: + aliases[name.asname or name.name] = name.name + for node in ast.walk(tree): + if not isinstance(node, ast.Attribute) or not isinstance(node.value, ast.Name): + continue + class_name = aliases.get(node.value.id) + if class_name is None: + continue + checked += 1 + real_class = getattr(fi_types, class_name, None) + if real_class is None or not hasattr(real_class, node.attr): + missing.append(f"{source.name}:{node.lineno}: {class_name}.{node.attr}") + assert checked > 0, "The semantic-constant scan must not be empty" + assert not missing, "Missing real core constants: " + "; ".join(missing) + return {"files_scanned": len(files), "references_checked": checked, "missing": missing} + + +def assert_system_message(attributes): + assert SpanAttributes.GEN_AI_INPUT_MESSAGES == "gen_ai.input.messages" + assert attributes[ROLE_KEY] == "system", attributes + assert attributes[CONTENT_KEY] == SYSTEM_CONTENT, attributes + + +def check_extractor(): + extractor = _ChatCompletionRequestAttributesExtractor() + attributes = dict(extractor.get_attributes_from_request(REQUEST)) + attributes.update(extractor.get_extra_attributes_from_request(REQUEST)) + assert_system_message(attributes) + return {"attributes": attributes} + + +def check_chat(hide_inputs): + from openai import OpenAI + from openai.resources.chat.completions import Completions + + exporter = InMemorySpanExporter() + provider = TracerProvider(sampler=ALWAYS_ON) + provider.add_span_processor(SimpleSpanProcessor(exporter)) + instrumentor = MiniMaxInstrumentor() + calls = [] + response = { + "id": "synthetic-completion", + "model": REQUEST["model"], + "choices": [{"message": {"role": "assistant", "content": "Synthetic reply."}}], + "usage": {"prompt_tokens": 4, "completion_tokens": 2, "total_tokens": 6}, + } + + def offline_create(self, *args, **kwargs): + assert kwargs == REQUEST + calls.append(trace.get_current_span().is_recording()) + return response + + # Replace the exact method the instrumentor wraps, before installing the + # wrapper. The real client and tracing stack run without making HTTP calls. + try: + with patch.object(Completions, "create", offline_create): + instrumentor.instrument(tracer_provider=provider, config=TraceConfig(hide_inputs=hide_inputs)) + try: + with OpenAI( + api_key="synthetic-offline-key", + base_url="https://api.minimax.io/v1", + max_retries=0, + ) as client: + assert client.chat.completions.create(**REQUEST) is response + finally: + instrumentor.uninstrument() + provider.force_flush() + spans = exporter.get_finished_spans() + assert len(calls) == 1, f"Expected one offline client call, got {calls}" + assert len(spans) == 1, ( + f"Expected one exported span, got {len(spans)}; recording during call: {calls}" + ) + assert calls == [True], "The chat call must run inside a recording span" + attributes = dict(spans[0].attributes) + if hide_inputs: + serialized = json.dumps(attributes, sort_keys=True) + assert SYSTEM_CONTENT not in serialized, attributes + assert USER_CONTENT not in serialized, attributes + else: + assert_system_message(attributes) + return {"span_count": len(spans), "recording_during_call": calls, "attributes": attributes} + finally: + provider.shutdown() + + +case = sys.argv[1] +output = {"case": case, "core_version": importlib.metadata.version("fi-instrumentation-otel")} +try: + if case == "constants": + output["result"] = check_constants() + elif case == "extractor": + output["result"] = check_extractor() + else: + output["result"] = check_chat(hide_inputs=case == "hide_inputs") + output["ok"] = True +except Exception as error: + output.update(ok=False, error_type=type(error).__name__, error=str(error)) + traceback.print_exc(file=sys.stderr) +print(json.dumps(output, sort_keys=True)) +sys.exit(0 if output["ok"] else 1) +''' + + +@pytest.mark.parametrize("case", ["constants", "extractor", "chat", "hide_inputs"]) +def test_real_core(case): + # Do not inherit credentials, content-hiding flags, or tracing configuration. + env = { + key: os.environ[key] + for key in ("PATH", "HOME", "TMPDIR", "TMP", "TEMP", "SystemRoot") + if key in os.environ + } + env.update(FI_BASE_URL="http://127.0.0.1:9", FI_GRPC_URL="http://127.0.0.1:9") + completed = subprocess.run( + [sys.executable, "-I", "-B", "-c", _PROBE, case], + cwd=_PACKAGE_DIR, + env=env, + text=True, + capture_output=True, + timeout=30, + check=False, + ) + assert completed.stdout.strip(), completed.stderr + result = json.loads(completed.stdout) + assert result["case"] == case + assert completed.returncode == 0, f"{result}\n{completed.stderr}" + assert result["ok"] is True, result + if case == "constants": + assert result["result"]["references_checked"] > 0 + assert result["result"]["missing"] == [] + elif case in {"chat", "hide_inputs"}: + assert result["result"]["span_count"] == 1 + assert result["result"]["recording_during_call"] == [True] diff --git a/python/frameworks/minimax/tests/test_stream_none_fields.py b/python/frameworks/minimax/tests/test_stream_none_fields.py new file mode 100644 index 000000000..b8541e491 --- /dev/null +++ b/python/frameworks/minimax/tests/test_stream_none_fields.py @@ -0,0 +1,37 @@ +"""Streamed chunks dumped with model_dump() carry None for absent fields.""" + +from traceai_minimax._response_attributes_extractor import _StreamingChatCompletionResponseExtractor + + +def _chunk(delta, finish_reason=None, usage=None): + return { + "id": "chatcmpl-synthetic", + "model": "MiniMax-M1", + "usage": usage, + "choices": [{"index": 0, "delta": delta, "finish_reason": finish_reason, "logprobs": None}], + } + + +def test_none_fields_in_stream_chunks_do_not_raise_and_keep_content_and_usage(): + extractor = _StreamingChatCompletionResponseExtractor() + chunks = [ + _chunk({"role": "assistant", "content": "Hel", "tool_calls": None, "function_call": None}), + _chunk({"role": None, "content": "lo", "tool_calls": None}), + _chunk({"content": None, "tool_calls": [{"index": 0, "id": "call_1", "type": "function", + "function": {"name": "lookup", "arguments": None}}]}), + _chunk({"content": None, "tool_calls": [{"index": None, "id": None, "type": None, + "function": {"name": None, "arguments": "{\"q\": 1}"}}]}), + _chunk({"content": None, "tool_calls": None}, finish_reason="stop", + usage={"prompt_tokens": 3, "completion_tokens": 2, "total_tokens": 5}), + _chunk(None, usage=None), + {"id": "chatcmpl-synthetic", "model": "MiniMax-M1", "usage": None, "choices": None}, + ] + for chunk in chunks: + extractor.process_chunk(chunk) + + attributes = dict(extractor.get_attributes()) + assert 5 in attributes.values() # total tokens survive later usage=None chunks + assert extractor._content_parts == ["Hel", "lo"] + assert extractor._tool_calls[0]["id"] == "call_1" + assert extractor._tool_calls[0]["function"] == {"name": "lookup", "arguments": "{\"q\": 1}"} + assert extractor._finish_reason == "stop" diff --git a/python/frameworks/minimax/traceai_minimax/_request_attributes_extractor.py b/python/frameworks/minimax/traceai_minimax/_request_attributes_extractor.py index f97c0e38c..bac082c7f 100644 --- a/python/frameworks/minimax/traceai_minimax/_request_attributes_extractor.py +++ b/python/frameworks/minimax/traceai_minimax/_request_attributes_extractor.py @@ -67,12 +67,6 @@ def get_extra_attributes_from_request( yield f"{SpanAttributes.GEN_AI_INPUT_MESSAGES}.{i}.tool_calls.{j}.function.name", tc["function"].get("name", "") yield f"{SpanAttributes.GEN_AI_INPUT_MESSAGES}.{i}.tool_calls.{j}.function.arguments", tc["function"].get("arguments", "") - # System prompt extraction - for msg in messages: - if msg.get("role") == "system": - yield SpanAttributes.GEN_AI_PROVIDER_NAME_PROMPT, str(msg.get("content", "")) - break - # User input for display for msg in reversed(messages): if msg.get("role") == "user": diff --git a/python/frameworks/minimax/traceai_minimax/_response_attributes_extractor.py b/python/frameworks/minimax/traceai_minimax/_response_attributes_extractor.py index 58c9319ac..0f3c6d903 100644 --- a/python/frameworks/minimax/traceai_minimax/_response_attributes_extractor.py +++ b/python/frameworks/minimax/traceai_minimax/_response_attributes_extractor.py @@ -113,38 +113,40 @@ def process_chunk(self, chunk: Dict[str, Any]) -> None: if "model" in chunk and not self._model: self._model = chunk["model"] - # Handle usage in final chunk - if "usage" in chunk: + # Handle usage in final chunk. SDK chunks dumped with model_dump() carry + # usage=None on every other chunk, which must not erase the real usage. + if chunk.get("usage"): self._usage = chunk["usage"] - choices = chunk.get("choices", []) - for choice in choices: - delta = choice.get("delta", {}) + # model_dump() also sets absent fields to None (delta.tool_calls, + # function.arguments, ...), so treat None like a missing field. + for choice in chunk.get("choices") or []: + delta = choice.get("delta") or {} # Content - if "content" in delta and delta["content"]: + if delta.get("content"): self._content_parts.append(delta["content"]) # Tool calls - if "tool_calls" in delta: - for tc in delta["tool_calls"]: - idx = tc.get("index", 0) - if idx not in self._tool_calls: - self._tool_calls[idx] = { - "id": tc.get("id", ""), - "type": tc.get("type", "function"), - "function": {"name": "", "arguments": ""} - } - if "id" in tc and tc["id"]: - self._tool_calls[idx]["id"] = tc["id"] - if "function" in tc: - if "name" in tc["function"] and tc["function"]["name"]: - self._tool_calls[idx]["function"]["name"] = tc["function"]["name"] - if "arguments" in tc["function"]: - self._tool_calls[idx]["function"]["arguments"] += tc["function"]["arguments"] + for tc in delta.get("tool_calls") or []: + idx = tc.get("index") + idx = 0 if idx is None else idx + if idx not in self._tool_calls: + self._tool_calls[idx] = { + "id": tc.get("id") or "", + "type": tc.get("type") or "function", + "function": {"name": "", "arguments": ""} + } + if tc.get("id"): + self._tool_calls[idx]["id"] = tc["id"] + function = tc.get("function") or {} + if function.get("name"): + self._tool_calls[idx]["function"]["name"] = function["name"] + if function.get("arguments"): + self._tool_calls[idx]["function"]["arguments"] += function["arguments"] # Finish reason - if "finish_reason" in choice and choice["finish_reason"]: + if choice.get("finish_reason"): self._finish_reason = choice["finish_reason"] def get_attributes(self) -> Iterator[Tuple[str, AttributeValue]]: diff --git a/python/frameworks/minimax/traceai_minimax/version.py b/python/frameworks/minimax/traceai_minimax/version.py index 75716eb9a..34d76ee9e 100644 --- a/python/frameworks/minimax/traceai_minimax/version.py +++ b/python/frameworks/minimax/traceai_minimax/version.py @@ -1 +1 @@ -__version__ = "0.1.0" # x-release-please-version +__version__ = "0.1.1" # x-release-please-version diff --git a/python/frameworks/mistralai/CHANGELOG.md b/python/frameworks/mistralai/CHANGELOG.md index 5a33c8411..34d04a1e7 100644 --- a/python/frameworks/mistralai/CHANGELOG.md +++ b/python/frameworks/mistralai/CHANGELOG.md @@ -1,3 +1,8 @@ +## [0.1.11] - pending release + +### Fixed +- Tool calls and tool results from the 1.x Mistral SDK can be matched in traces by their tool-call IDs. + ## [0.1.10](https://github.com/future-agi/traceAI/compare/traceai-mistralai-v0.1.9...traceai-mistralai-v0.1.10) (2026-11-05) diff --git a/python/frameworks/mistralai/pyproject.toml b/python/frameworks/mistralai/pyproject.toml index 3e8c2767a..0d8c7ebb8 100644 --- a/python/frameworks/mistralai/pyproject.toml +++ b/python/frameworks/mistralai/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "traceAI-mistralai" -version = "0.1.10" +version = "0.1.11" description = "OpenTelemetry instrumentation for Mistral AI" authors = ["Future AGI "] readme = "README.md" diff --git a/python/frameworks/mistralai/tests/test_legacy_tool_id_correlation.py b/python/frameworks/mistralai/tests/test_legacy_tool_id_correlation.py new file mode 100644 index 000000000..a7e08095f --- /dev/null +++ b/python/frameworks/mistralai/tests/test_legacy_tool_id_correlation.py @@ -0,0 +1,267 @@ +"""Legacy tool correlation IDs, with native SDK controls and no provider traffic.""" + +import asyncio +from copy import deepcopy +import importlib +import json + +import pytest +from fi_instrumentation import TraceConfig +from fi_instrumentation.fi_types import ( + MessageAttributes, + SpanAttributes, + ToolCallAttributes, +) +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter + +from traceai_mistralai import MistralAIInstrumentor +from test_sdk import NATIVE, SDK_ROOT, Mistral, httpx + + +INPUT_ID = "call00001" +OUTPUT_ID = "call00002" +FUNCTION = {"name": "weather", "arguments": '{"city":"test"}'} +MESSAGES = [ + { + "role": "assistant", + "tool_calls": [{"id": INPUT_ID, "type": "function", "function": FUNCTION}], + }, + {"role": "tool", "tool_call_id": INPUT_ID, "content": "sunny"}, +] +RESPONSE = { + "id": "synthetic-completion", + "object": "chat.completion", + "created": 1, + "model": "mistral-test", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "tool_calls": [ + {"id": OUTPUT_ID, "type": "function", "function": FUNCTION} + ], + }, + "finish_reason": "tool_calls", + } + ], + "usage": {"prompt_tokens": 2, "completion_tokens": 1, "total_tokens": 3}, +} + + +def _events(): + # Match test_sdk.completion_sse: a message delta followed by a tool call. + deltas = [ + {"role": "assistant", "content": ""}, + { + "tool_calls": [ + { + "index": 0, + "id": OUTPUT_ID, + "type": "function", + "function": FUNCTION, + } + ] + }, + ] + for index, delta in enumerate(deltas): + event = { + **RESPONSE, + "object": "chat.completion.chunk", + "choices": [ + { + "index": 0, + "delta": delta, + "finish_reason": "tool_calls" if index else None, + } + ], + "usage": RESPONSE["usage"] if index else None, + } + yield ("data: " + json.dumps(event) + "\n\n").encode() + yield b"data: [DONE]\n\n" + + +class _ToolSSE(httpx.SyncByteStream, httpx.AsyncByteStream): + def __iter__(self): + yield from _events() + + async def __aiter__(self): + for event in self: + yield event + + +def _complete(messages, streaming, asynchronous): + observed = [] + + def respond(request): + observed.append(bytes(request.content)) + if streaming: + return httpx.Response( + 200, + headers={"content-type": "text/event-stream"}, + stream=_ToolSSE(), + ) + return httpx.Response(200, json=deepcopy(RESPONSE)) + + kwargs = {"model": "mistral-test", "messages": messages} + + async def run(): + async with httpx.AsyncClient( + transport=httpx.MockTransport(respond) + ) as transport: + client = Mistral(api_key="synthetic-test-key", async_client=transport) + if not streaming: + return await client.chat.complete_async(**kwargs) + stream = await client.chat.stream_async(**kwargs) + try: + return [chunk async for chunk in stream] + finally: + if hasattr(stream, "close"): + await stream.close() + + if asynchronous: + result = asyncio.run(run()) + else: + with httpx.Client(transport=httpx.MockTransport(respond)) as transport: + client = Mistral(api_key="synthetic-test-key", client=transport) + if streaming: + # mistralai 1.1.0 returns a plain generator; later 1.x returns a + # context-managed stream. Consume both the same way. + stream = client.chat.stream(**kwargs) + try: + result = list(stream) + finally: + if hasattr(stream, "close"): + stream.close() + else: + result = client.chat.complete(**kwargs) + return observed, result + + +def _tool_fields(attributes, root, message_index): + """Read the same semantic fields from native JSON or legacy FI leaves.""" + if NATIVE: + messages = json.loads(attributes.get(root, "[]")) + if not messages: + return {} + part = next( + part + for part in messages[message_index]["parts"] + if part["type"] == "tool_call" + ) + return { + "id": part.get("id"), + "name": part.get("name"), + "arguments": part.get("arguments"), + } + prefix = f"{root}.{message_index}.{MessageAttributes.MESSAGE_TOOL_CALLS}.0." + return { + field: attributes[prefix + leaf] + for field, leaf in ( + ("id", ToolCallAttributes.TOOL_CALL_ID), + ("name", ToolCallAttributes.TOOL_CALL_FUNCTION_NAME), + ("arguments", ToolCallAttributes.TOOL_CALL_FUNCTION_ARGUMENTS_JSON), + ) + if prefix + leaf in attributes + } + + +def _result_id(attributes): + if NATIVE: + messages = json.loads(attributes.get(SpanAttributes.GEN_AI_INPUT_MESSAGES, "[]")) + return messages[1]["parts"][0].get("id") if messages else None + return attributes.get( + f"{SpanAttributes.GEN_AI_INPUT_MESSAGES}.1." + f"{MessageAttributes.MESSAGE_TOOL_CALL_ID}" + ) + + +@pytest.mark.parametrize("typed", [False, True], ids=["dict", "typed"]) +@pytest.mark.parametrize("asynchronous", [False, True], ids=["sync", "async"]) +@pytest.mark.parametrize("streaming", [False, True], ids=["complete", "stream"]) +@pytest.mark.parametrize( + "policy", + [ + "visible", + "hide_inputs", + "hide_outputs", + "hide_input_messages", + "hide_output_messages", + "hide_input_text", + "hide_output_text", + "hide_both", + ], +) +def test_tool_id_correlation(typed, asynchronous, streaming, policy): + messages = deepcopy(MESSAGES) + if typed: + models = importlib.import_module(SDK_ROOT + ".models") + messages = [ + models.AssistantMessage(**messages[0]), + models.ToolMessage(**messages[1]), + ] + original = deepcopy(messages) + baseline_wire, baseline_result = _complete(messages, streaming, asynchronous) + assert messages == original + config = TraceConfig( + **( + {"hide_inputs": True, "hide_outputs": True} + if policy == "hide_both" + else {} if policy == "visible" else {policy: True} + ) + ) + provider = TracerProvider() + exporter = InMemorySpanExporter() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + instrumentor = MistralAIInstrumentor() + instrumentor.instrument(tracer_provider=provider, config=config) + try: + wire, result = _complete(messages, streaming, asynchronous) + provider.force_flush() + spans = exporter.get_finished_spans() + finally: + instrumentor.uninstrument() + provider.shutdown() + + # Compare complete serialized requests and SDK results to an uninstrumented + # call, not just the IDs. Privacy must affect telemetry only. + assert messages == original + assert wire == baseline_wire + assert len(wire) == 1 + body = json.loads(wire[0]) + assert body["messages"][0]["tool_calls"][0]["id"] == INPUT_ID + assert body["messages"][1]["tool_call_id"] == INPUT_ID + returned = result if streaming else [result] + baseline = baseline_result if streaming else [baseline_result] + assert len(returned) == len(baseline) + for actual, expected in zip(returned, baseline): + assert type(actual) is type(expected) + assert actual.model_dump() == expected.model_dump() + output_call = ( + result[1].data.choices[0].delta.tool_calls[0] + if streaming else result.choices[0].message.tool_calls[0] + ) + assert output_call.id == OUTPUT_ID + + assert len(spans) == 1 + attributes = spans[0].attributes + hide_input = config.hide_inputs or config.hide_input_messages + hide_output = config.hide_outputs or config.hide_output_messages + # One assertion shows all three missing IDs in the RED run. Selective text + # hiding retains IDs just as it retains function names and arguments. + assert { + "input_call": _tool_fields(attributes, SpanAttributes.GEN_AI_INPUT_MESSAGES, 0), + "input_result_id": _result_id(attributes), + "output_call": _tool_fields(attributes, SpanAttributes.GEN_AI_OUTPUT_MESSAGES, 0), + } == { + "input_call": {} if hide_input else {"id": INPUT_ID, **FUNCTION}, + "input_result_id": None if hide_input else INPUT_ID, + "output_call": {} if hide_output else {"id": OUTPUT_ID, **FUNCTION}, + } + encoded = json.dumps(dict(attributes)) + if hide_input: + assert INPUT_ID not in encoded + if hide_output: + assert OUTPUT_ID not in encoded diff --git a/python/frameworks/mistralai/traceai_mistralai/_request_attributes_extractor.py b/python/frameworks/mistralai/traceai_mistralai/_request_attributes_extractor.py index b0691b602..c21c891ec 100644 --- a/python/frameworks/mistralai/traceai_mistralai/_request_attributes_extractor.py +++ b/python/frameworks/mistralai/traceai_mistralai/_request_attributes_extractor.py @@ -91,6 +91,8 @@ def _get_attributes_from_message_param( if name := message.get("name"): yield MessageAttributes.MESSAGE_NAME, name + if tool_call_id := message.get("tool_call_id"): + yield MessageAttributes.MESSAGE_TOOL_CALL_ID, tool_call_id if (function_call := message.get("function_call")) and hasattr( function_call, "get" ): @@ -104,6 +106,12 @@ def _get_attributes_from_message_param( if (tool_calls := message.get("tool_calls")) and isinstance(tool_calls, Iterable): for index, tool_call in enumerate(tool_calls): tool_call = _message_dict(tool_call) + if tool_call_id := tool_call.get("id"): + yield ( + f"{MessageAttributes.MESSAGE_TOOL_CALLS}.{index}." + f"{ToolCallAttributes.TOOL_CALL_ID}", + tool_call_id, + ) function = _message_dict(tool_call.get("function")) if tool_call else None if function: if name := function.get("name"): diff --git a/python/frameworks/mistralai/traceai_mistralai/_response_attributes_extractor.py b/python/frameworks/mistralai/traceai_mistralai/_response_attributes_extractor.py index 736337f35..16b04b0eb 100644 --- a/python/frameworks/mistralai/traceai_mistralai/_response_attributes_extractor.py +++ b/python/frameworks/mistralai/traceai_mistralai/_response_attributes_extractor.py @@ -81,6 +81,12 @@ def _get_attributes_from_chat_completion_message( tool_calls, Iterable ): for index, tool_call in enumerate(tool_calls): + if tool_call_id := _get_attribute_or_value(tool_call, "id"): + yield ( + f"{MessageAttributes.MESSAGE_TOOL_CALLS}.{index}." + f"{ToolCallAttributes.TOOL_CALL_ID}", + tool_call_id, + ) if function := _get_attribute_or_value(tool_call, "function"): if name := _get_attribute_or_value(function, "name"): yield ( diff --git a/python/frameworks/mistralai/traceai_mistralai/version.py b/python/frameworks/mistralai/traceai_mistralai/version.py index b2b8e3d08..8c39d1009 100644 --- a/python/frameworks/mistralai/traceai_mistralai/version.py +++ b/python/frameworks/mistralai/traceai_mistralai/version.py @@ -1 +1 @@ -__version__ = "0.1.10" # x-release-please-version +__version__ = "0.1.11" # x-release-please-version diff --git a/python/frameworks/ollama/CHANGELOG.md b/python/frameworks/ollama/CHANGELOG.md new file mode 100644 index 000000000..89efa67b2 --- /dev/null +++ b/python/frameworks/ollama/CHANGELOG.md @@ -0,0 +1,5 @@ +## [0.1.1] - pending release + +### Changed +- traceai-ollama now requires fi-instrumentation-otel 1.0.0 or later. Action: Upgrade fi-instrumentation-otel if your environment pins an earlier version. + diff --git a/python/frameworks/ollama/pyproject.toml b/python/frameworks/ollama/pyproject.toml index c68c8c046..bb1280b37 100644 --- a/python/frameworks/ollama/pyproject.toml +++ b/python/frameworks/ollama/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "traceAI-ollama" -version = "0.1.0" +version = "0.1.1" description = "OpenTelemetry instrumentation for Ollama - local LLM runner" authors = ["Future AGI "] readme = "README.md" @@ -28,7 +28,7 @@ classifiers = [ [tool.poetry.dependencies] python = ">3.9,<3.14" -fi-instrumentation-otel = ">=0.1.11" +fi-instrumentation-otel = ">=1.0.0" ollama = ">=0.3.0" [tool.poetry.urls] diff --git a/python/frameworks/ollama/traceai_ollama/version.py b/python/frameworks/ollama/traceai_ollama/version.py index 75716eb9a..34d76ee9e 100644 --- a/python/frameworks/ollama/traceai_ollama/version.py +++ b/python/frameworks/ollama/traceai_ollama/version.py @@ -1 +1 @@ -__version__ = "0.1.0" # x-release-please-version +__version__ = "0.1.1" # x-release-please-version diff --git a/python/frameworks/openai-agents/CHANGELOG.md b/python/frameworks/openai-agents/CHANGELOG.md index ca4732a12..6ee28135c 100644 --- a/python/frameworks/openai-agents/CHANGELOG.md +++ b/python/frameworks/openai-agents/CHANGELOG.md @@ -1,5 +1,14 @@ +## [0.1.8] - pending release + +### Changed +- traceai-openai-agents now requires fi-instrumentation-otel 1.2.2 or later. Action: Upgrade fi-instrumentation-otel if your environment pins an earlier version. + +### Fixed +- Response span output no longer repeats the request's instructions or input, and a Responses call's token usage is counted once. +- Responses with custom tool calls are traced, and their spans close even when converting their attributes fails. + ## [0.1.7] - 2026-10-29 ### Fixed diff --git a/python/frameworks/openai-agents/poetry.lock b/python/frameworks/openai-agents/poetry.lock index e201a052f..c7a069c2b 100644 --- a/python/frameworks/openai-agents/poetry.lock +++ b/python/frameworks/openai-agents/poetry.lock @@ -226,14 +226,14 @@ test = ["pytest (>=6)"] [[package]] name = "fi-instrumentation-otel" -version = "1.1.0" -description = "OpenTelemetry instrumentation" +version = "1.2.2" +description = "OpenTelemetry tracing for AI applications: tracer setup, span helpers, exporters, and semantic conventions for Future AGI." optional = false python-versions = "<3.14,>3.9" groups = ["main"] files = [ - {file = "fi_instrumentation_otel-1.1.0-py3-none-any.whl", hash = "sha256:4fa2bf663e0ce09447aca92ae798cd7b0dee35c5c92c2e033f1d89d158a9e62c"}, - {file = "fi_instrumentation_otel-1.1.0.tar.gz", hash = "sha256:7ec324c3c9242aa87a88cf23a21c11919b8736c6451c5fb50917f592a853b059"}, + {file = "fi_instrumentation_otel-1.2.2-py3-none-any.whl", hash = "sha256:25cf88f93e33aa928d4fae0f631afe8a32565639490862b715869e0633becbaf"}, + {file = "fi_instrumentation_otel-1.2.2.tar.gz", hash = "sha256:b675df47d7a03c506fbd2e4b0420fe4f74d12a6353a88b219bfd97797b276d4c"}, ] [package.dependencies] @@ -1435,4 +1435,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.1" python-versions = ">3.9, <3.14" -content-hash = "7ccd10f7b1fb5981e2e73c5733126c52526f066456643e4879c840f7a405d071" +content-hash = "f136cfede75deaf27bf3361935d90bcf0e52089ed8e611a283b126dc7b905e8c" diff --git a/python/frameworks/openai-agents/pyproject.toml b/python/frameworks/openai-agents/pyproject.toml index 914401851..02bfb5cfa 100644 --- a/python/frameworks/openai-agents/pyproject.toml +++ b/python/frameworks/openai-agents/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "traceAI-openai-agents" -version = "0.1.7" +version = "0.1.8" description = "OpenTelemetry instrumentation for OpenAI Agents" authors = ["Future AGI "] readme = "README.md" @@ -30,7 +30,7 @@ classifiers = [ [tool.poetry.dependencies] python = ">3.9, <3.14" openai-agents = ">= 0.2.6" -fi-instrumentation-otel = ">=0.1.11" +fi-instrumentation-otel = ">=1.2.2" [tool.poetry.urls] diff --git a/python/frameworks/openai-agents/tests/test_custom_tool_lifecycle.py b/python/frameworks/openai-agents/tests/test_custom_tool_lifecycle.py new file mode 100644 index 000000000..111c94fb2 --- /dev/null +++ b/python/frameworks/openai-agents/tests/test_custom_tool_lifecycle.py @@ -0,0 +1,199 @@ +"""Offline Runner contracts with the real SDK and only the HTTP transport replaced.""" + +import asyncio +import json +import logging + +import agents +import httpx +import pytest +from agents import Agent, ModelSettings, Runner, set_trace_processors, set_tracing_disabled +from agents.models.openai_responses import OpenAIResponsesModel +from fi_instrumentation import TraceConfig +from fi_instrumentation.fi_types import MessageAttributes, SpanAttributes, ToolCallAttributes +from openai import AsyncOpenAI +from openai.types.responses import ResponseCustomToolCall, ResponseFunctionToolCall, ResponseOutputMessage +from opentelemetry.sdk.trace import SpanProcessor +from traceai_openai_agents import OpenAIAgentsInstrumentor +from traceai_openai_agents._processor import _get_attributes_from_response_output + + +class _StartedSpans(SpanProcessor): + def __init__(self): + self.spans = [] + + def on_start(self, span, parent_context=None): + self.spans.append(span) + + def on_end(self, span): + pass + + +@pytest.fixture +def runner_tracing(tracer_provider): + # No vendor exporter, even in the uninstrumented controls. + set_trace_processors([]) + set_tracing_disabled(False) + started = _StartedSpans() + tracer_provider.add_span_processor(started) + yield started + set_trace_processors([]) + instrumentor = OpenAIAgentsInstrumentor() + if instrumentor.is_instrumented_by_opentelemetry: + instrumentor.uninstrument() + + +async def _run(custom): + requests, invocations = [], [] + + async def invoke(ctx, value): + invocations.append(value) + return "synthetic tool result" + + native = hasattr(agents, "CustomTool") + kwargs = {} + if custom: + if native: + kwargs["tools"] = [agents.CustomTool( + name="plain_tool", description="Accept plain text", on_invoke_tool=invoke, + )] + else: + # Older SDKs forward this public setting but do not execute custom tools. + kwargs["model_settings"] = ModelSettings(extra_body={ + "tools": [{"type": "custom", "name": "plain_tool", "description": "Accept plain text"}], + "tool_choice": {"type": "custom", "name": "plain_tool"}, + }) + + def respond(request): + body = json.loads(request.content) + requests.append(body) + assert request.url.path == "/v1/responses" + if custom and len(requests) == 1: + assert body["tools"][0]["type"] == "custom" + output = [{"type": "custom_tool_call", "id": "ctc_synthetic", "call_id": "call_synthetic", + "name": "plain_tool", "input": "synthetic argument"}] + else: + output = [{"type": "message", "id": "msg_synthetic", "role": "assistant", "status": "completed", + "content": [{"type": "output_text", "text": "synthetic answer", "annotations": []}]}] + return httpx.Response(200, json={ + "id": "resp_synthetic_" + str(len(requests)), "object": "response", "created_at": 1, + "model": "gpt-4.1", "status": "completed", "output": output, "tools": body.get("tools", []), + "parallel_tool_calls": False, "tool_choice": "auto", + "usage": {"input_tokens": 2, "output_tokens": 2, "total_tokens": 4, + "input_tokens_details": {"cached_tokens": 0}, "output_tokens_details": {"reasoning_tokens": 0}}, + }) + + async with httpx.AsyncClient(transport=httpx.MockTransport(respond)) as http: + async with AsyncOpenAI(api_key="synthetic-not-a-credential", base_url="https://offline.invalid/v1", + http_client=http, max_retries=0) as client: + agent = Agent(name="Assistant", instructions="Be concise", + model=OpenAIResponsesModel(model="gpt-4.1", openai_client=client), **kwargs) + result = await Runner.run(agent, "synthetic prompt", max_turns=3) + if custom and native: + assert invocations == ["synthetic argument"] + assert len(requests) == 2 + assert any(item.get("type") == "custom_tool_call_output" and item.get("output") == "synthetic tool result" + for item in requests[1]["input"]) + else: + assert invocations == [] + assert len(requests) == 1 + assert result.final_output == ("" if custom and not native else "synthetic answer") + return result.final_output + + +def _assert_all_finished(started, exporter): + assert started.spans + unfinished = [span.name for span in started.spans if span.is_recording()] + assert unfinished == [], f"Unfinished spans: {unfinished}" + finished = exporter.get_finished_spans() + assert {s.context.span_id for s in finished} == {s.context.span_id for s in started.spans} + assert len(finished) == len(started.spans) + assert all(s.end_time >= s.start_time for s in finished) + + +@pytest.mark.parametrize("custom", [False, True], ids=["text", "custom"]) +def test_uninstrumented_runner_control(custom, runner_tracing, in_memory_span_exporter): + asyncio.run(_run(custom)) + assert runner_tracing.spans == [] + assert in_memory_span_exporter.get_finished_spans() == () + + +def test_instrumented_text_control(runner_tracing, tracer_provider, in_memory_span_exporter, caplog): + OpenAIAgentsInstrumentor().instrument(tracer_provider=tracer_provider) + with caplog.at_level(logging.ERROR): + asyncio.run(_run(False)) + _assert_all_finished(runner_tracing, in_memory_span_exporter) + assert not [r for r in caplog.records if "trace processor" in r.getMessage()] + + +@pytest.mark.parametrize("hidden", [False, True], ids=["visible", "hidden"]) +def test_runner_custom_tool_finalizes_all_spans( + hidden, runner_tracing, tracer_provider, in_memory_span_exporter, caplog, +): + OpenAIAgentsInstrumentor().instrument( + tracer_provider=tracer_provider, + config=TraceConfig(hide_inputs=hidden, hide_outputs=hidden, pii_redaction=False), + ) + with caplog.at_level(logging.ERROR): + asyncio.run(_run(True)) + _assert_all_finished(runner_tracing, in_memory_span_exporter) + assert not [r for r in caplog.records if "trace processor" in r.getMessage()] + spans = in_memory_span_exporter.get_finished_spans() + if hidden: + exported = json.dumps([dict(s.attributes) for s in spans]) + for marker in ("synthetic prompt", "synthetic argument", "synthetic answer", "synthetic tool result"): + assert marker not in exported + else: + first_response = next(s for s in spans if s.name == "response") + prefix = f"{SpanAttributes.GEN_AI_OUTPUT_MESSAGES}.0.{MessageAttributes.MESSAGE_TOOL_CALLS}.0." + assert first_response.attributes[prefix + ToolCallAttributes.TOOL_CALL_ID] == "call_synthetic" + assert first_response.attributes[prefix + ToolCallAttributes.TOOL_CALL_FUNCTION_NAME] == "plain_tool" + assert json.loads(first_response.attributes[prefix + ToolCallAttributes.TOOL_CALL_FUNCTION_ARGUMENTS_JSON]) == { + "input": "synthetic argument", + } + assert "synthetic-not-a-credential" not in json.dumps([dict(s.attributes) for s in spans]) + + +@pytest.mark.parametrize("preceding", ["none", "message", "function", "custom"]) +def test_custom_tool_output_indices(preceding): + custom = ResponseCustomToolCall(type="custom_tool_call", call_id="call_second", name="plain_tool", input="second") + outputs = [] + msg_idx, tool_idx = 0, 0 + if preceding == "message": + outputs.append(ResponseOutputMessage(id="msg_first", type="message", role="assistant", status="completed", content=[])) + msg_idx = 1 + elif preceding == "function": + outputs.append(ResponseFunctionToolCall(type="function_call", call_id="call_first", name="function_tool", arguments="{}")) + tool_idx = 1 + elif preceding == "custom": + outputs.append(ResponseCustomToolCall(type="custom_tool_call", call_id="call_first", name="plain_tool", input="first")) + tool_idx = 1 + outputs.append(custom) + attrs = dict(_get_attributes_from_response_output(outputs)) + prefix = f"{SpanAttributes.GEN_AI_OUTPUT_MESSAGES}.{msg_idx}." + assert attrs[prefix + MessageAttributes.MESSAGE_ROLE] == "assistant" + call_prefix = prefix + f"{MessageAttributes.MESSAGE_TOOL_CALLS}.{tool_idx}." + assert attrs[call_prefix + ToolCallAttributes.TOOL_CALL_ID] == "call_second" + if tool_idx: + assert attrs[prefix + f"{MessageAttributes.MESSAGE_TOOL_CALLS}.0." + ToolCallAttributes.TOOL_CALL_ID] == "call_first" + + +def test_conversion_failure_still_finalizes_response_and_root( + runner_tracing, tracer_provider, in_memory_span_exporter, monkeypatch, caplog, +): + # Supplemental fault injection at our own converter, not an upstream SDK mock. + import traceai_openai_agents._processor as processor_module + + calls = [] + + def broken_conversion(response, **kwargs): + calls.append(kwargs.get("include_usage", True)) + raise ValueError("synthetic conversion detail must not be logged") + + monkeypatch.setattr(processor_module, "_get_attributes_from_response", broken_conversion) + OpenAIAgentsInstrumentor().instrument(tracer_provider=tracer_provider) + with caplog.at_level(logging.WARNING): + asyncio.run(_run(False)) + assert True in calls and False in calls # response and root conversion both ran + _assert_all_finished(runner_tracing, in_memory_span_exporter) + assert "synthetic conversion detail" not in caplog.text diff --git a/python/frameworks/openai-agents/tests/test_processor_usage.py b/python/frameworks/openai-agents/tests/test_processor_usage.py new file mode 100644 index 000000000..650996665 --- /dev/null +++ b/python/frameworks/openai-agents/tests/test_processor_usage.py @@ -0,0 +1,131 @@ +"""Offline usage accounting with fake SDK spans and real in-memory export.""" + +from types import SimpleNamespace + +import pytest +from agents.tracing.span_data import AgentSpanData, ResponseSpanData +from openai.types.responses import Response +from traceai_openai_agents._processor import FiTracingProcessor + + +def _sdk_span(span_id, data, parent_id=None): + return SimpleNamespace( + span_id=span_id, + trace_id="trace_fake_usage", + parent_id=parent_id, + span_data=data, + started_at="2025-01-01T00:00:00+00:00", + ended_at="2025-01-01T00:00:01+00:00", + error=None, + ) + + +@pytest.mark.parametrize("nested_in_agent", [True, False], ids=["agent", "root"]) +@pytest.mark.parametrize( + "usage_counts", + [ + pytest.param([(10, 5, 0, 0)], id="single-response"), + pytest.param([(10, 5, 3, 2), (20, 7, 4, 1)], id="multiple-responses"), + pytest.param([(0, 0, 0, 0)], id="zero-usage"), + pytest.param([None], id="missing-usage"), + ], +) +def test_response_usage_is_only_exported_on_request_spans( + tracer_provider, in_memory_span_exporter, usage_counts, nested_in_agent +): + processor = FiTracingProcessor(tracer_provider.get_tracer(__name__)) + workflow = SimpleNamespace(name="usage workflow", trace_id="trace_fake_usage") + agent = _sdk_span("span_fake_agent", AgentSpanData(name="assistant")) + expected_usage = [] + + processor.on_trace_start(workflow) + if nested_in_agent: + processor.on_span_start(agent) + try: + for index, counts in enumerate(usage_counts): + usage = None + attributes = {} + if counts is not None: + input_tokens, output_tokens, cached_tokens, reasoning_tokens = counts + usage = { + "input_tokens": input_tokens, + "output_tokens": output_tokens, + "total_tokens": input_tokens + output_tokens, + "input_tokens_details": {"cached_tokens": cached_tokens}, + "output_tokens_details": {"reasoning_tokens": reasoning_tokens}, + } + attributes = { + "gen_ai.usage.input_tokens": input_tokens, + "gen_ai.usage.output_tokens": output_tokens, + "gen_ai.usage.total_tokens": input_tokens + output_tokens, + "gen_ai.usage.input_tokens.cache_read": cached_tokens, + "gen_ai.usage.output_tokens.reasoning": reasoning_tokens, + } + expected_usage.append(attributes) + response = Response( + id=f"resp_fake_{index}", + created_at=1, + model="gpt-4o-mini", + object="response", + status="completed", + parallel_tool_calls=False, + tool_choice="none", + tools=[], + output=[{ + "id": f"msg_fake_{index}", + "type": "message", + "role": "assistant", + "status": "completed", + "content": [{ + "type": "output_text", + "text": f"answer {index}", + "annotations": [], + }], + }], + usage=usage, + ) + sdk_span = _sdk_span( + f"span_fake_response_{index}", + ResponseSpanData(input=f"question {index}", response=response), + parent_id=agent.span_id if nested_in_agent else None, + ) + processor.on_span_start(sdk_span) + processor.on_span_end(sdk_span) + finally: + if nested_in_agent: + processor.on_span_end(agent) + processor.on_trace_end(workflow) + + spans = in_memory_span_exporter.get_finished_spans() + requests = [span for span in spans if span.name == "response"] + wrappers = [span for span in spans if span.name != "response"] + assert len(requests) == len(usage_counts) + assert len(wrappers) == (2 if nested_in_agent else 1) + assert all(span.end_time is not None for span in spans) + + for index, (span, attributes) in enumerate(zip(requests, expected_usage)): + assert { + key: value for key, value in span.attributes.items() + if key.startswith("gen_ai.usage.") + } == attributes + assert span.attributes["gen_ai.span.kind"] == "LLM" + assert span.attributes["input.value"] == f"question {index}" + assert span.attributes["output.value"] == f"answer {index}" + + # Whole-trace accounting must equal the sum of the actual provider requests. + for key in {key for attributes in expected_usage for key in attributes}: + assert sum(span.attributes.get(key, 0) for span in spans) == sum( + attributes.get(key, 0) for attributes in expected_usage + ) + + for span in wrappers: + assert not any(key.startswith("gen_ai.usage.") for key in span.attributes) + assert span.attributes["gen_ai.request.model"] == "gpt-4o-mini" + assert span.attributes["output.value"] == f"answer {len(usage_counts) - 1}" + assert span.attributes[ + "gen_ai.output.messages.0.message.contents.0.message_content.text" + ] == f"answer {len(usage_counts) - 1}" + root = next(span for span in wrappers if span.name == workflow.name) + assert root.attributes["input.value"] == "question 0" + parent = next(span for span in wrappers if span.name == "assistant") if nested_in_agent else root + assert all(span.parent.span_id == parent.context.span_id for span in requests) diff --git a/python/frameworks/openai-agents/tests/test_response_privacy.py b/python/frameworks/openai-agents/tests/test_response_privacy.py new file mode 100644 index 000000000..252ea5d8c --- /dev/null +++ b/python/frameworks/openai-agents/tests/test_response_privacy.py @@ -0,0 +1,188 @@ +"""Offline regressions for T2-AGENTS-001 using fake SDK lifecycle events.""" + +import json +from types import SimpleNamespace + +import pytest +from agents.tracing.span_data import AgentSpanData, ResponseSpanData +from fi_instrumentation import FITracer, TraceConfig +from openai.types.responses import Response +from traceai_openai_agents._processor import FiTracingProcessor + + +PRIVATE_MARKERS = ( + "PRIVATE_POLICY_MARKER", + "PRIVATE_TOOL_MARKER", + "PRIVATE_METADATA_MARKER", + "PRIVATE_FUTURE_FIELD_MARKER", +) + + +def _response(output_kind): + if output_kind == "tool-only": + output = [{ + "id": "fc_fake", "type": "function_call", "call_id": "call_fake", + "name": "lookup", "arguments": '{"query":"public output"}', + "status": "completed", + }] + elif output_kind == "empty": + output = [] + else: + content = ( + {"type": "refusal", "refusal": "Cannot answer this request"} + if output_kind == "refusal" + else {"type": "output_text", "text": "Public answer", "annotations": []} + ) + output = [{ + "id": "msg_fake", "type": "message", "role": "assistant", + "status": "completed", "content": [content], + }] + return Response( + id="resp_fake", + created_at=1, + model="gpt-4o-mini", + object="response", + status="completed", + parallel_tool_calls=False, + tool_choice="auto", + tools=[{ + "type": "function", "name": "lookup", "description": PRIVATE_MARKERS[1], + "parameters": {"type": "object", "properties": {}}, "strict": False, + }], + instructions=PRIVATE_MARKERS[0], + metadata={"private_note": PRIVATE_MARKERS[2]}, + future_request_echo=PRIVATE_MARKERS[3], + temperature=0.2, + max_output_tokens=64, + output=output, + usage={ + "input_tokens": 3, "output_tokens": 5, "total_tokens": 8, + "input_tokens_details": {"cached_tokens": 0}, + "output_tokens_details": {"reasoning_tokens": 0}, + }, + ) + + +def _export_response(response, config, tracer_provider): + # Drive the processor directly: no Runner, API client, network, or global + # SDK processor registration. FITracer applies the actual privacy policy. + processor = FiTracingProcessor( + FITracer(tracer_provider.get_tracer(__name__), config=config) + ) + trace = SimpleNamespace(name="privacy workflow", trace_id="trace_fake") + agent = SimpleNamespace( + trace_id=trace.trace_id, + span_id="agent_fake", + parent_id=None, + span_data=AgentSpanData(name="Assistant"), + started_at="2025-01-01T00:00:00+00:00", + ended_at="2025-01-01T00:00:02+00:00", + error=None, + ) + response_data = ResponseSpanData(response=response) + response_data.input = "hello" + response_span = SimpleNamespace( + trace_id=trace.trace_id, + span_id="response_fake", + parent_id=agent.span_id, + span_data=response_data, + started_at="2025-01-01T00:00:00+00:00", + ended_at="2025-01-01T00:00:01+00:00", + error=None, + ) + processor.on_trace_start(trace) + processor.on_span_start(agent) + try: + processor.on_span_start(response_span) + try: + pass + finally: + processor.on_span_end(response_span) + finally: + processor.on_span_end(agent) + processor.on_trace_end(trace) + + +@pytest.mark.parametrize("output_kind", ["tool-only", "empty", "refusal", "text"]) +@pytest.mark.parametrize( + "hide_inputs,hide_outputs", + [ + pytest.param(True, False, id="inputs-hidden"), + pytest.param(True, True, id="both-hidden"), + pytest.param(False, False, id="visible"), + ], +) +def test_response_output_excludes_request_echoes_on_all_spans( + output_kind, hide_inputs, hide_outputs, tracer_provider, in_memory_span_exporter +): + response = _response(output_kind) + original = response.model_dump(mode="json") + assert original["future_request_echo"] == PRIVATE_MARKERS[3] + config = TraceConfig( + hide_inputs=hide_inputs, + hide_outputs=hide_outputs, + hide_input_messages=False, + hide_input_text=False, + hide_output_messages=False, + hide_output_text=False, + hide_llm_invocation_parameters=False, + pii_redaction=False, + ) + + _export_response(response, config, tracer_provider) + + spans = in_memory_span_exporter.get_finished_spans() + assert len(spans) == 3 + assert {span.name for span in spans} == {"response", "Assistant", "privacy workflow"} + expected_output = { + key: original[key] + for key in ( + "id", "object", "created_at", "model", "status", "output", "usage", + "incomplete_details", + ) + if original.get(key) is not None + } + for span in spans: + attributes = span.attributes + assert attributes["input.value"] == ("__REDACTED__" if hide_inputs else "hello") + output = attributes["output.value"] + assert all(marker not in output for marker in PRIVATE_MARKERS) + if hide_outputs: + assert output == "__REDACTED__" + elif output_kind == "text": + assert output == "Public answer" + else: + # Preserve tool arguments, refusals, empty outputs and usage, rather + # than suppressing legitimate output merely because inputs are hidden. + assert json.loads(output) == expected_output + if hide_inputs: + assert all(marker not in json.dumps(dict(attributes)) for marker in PRIVATE_MARKERS) + else: + # Request information still belongs to its original input direction. + assert attributes["gen_ai.input.messages.0.message.content"] == PRIVATE_MARKERS[0] + assert attributes["gen_ai.request.model"] == "gpt-4o-mini" + # Canonical usage belongs to the response request, not its wrappers. + usage = { + key: value for key, value in attributes.items() + if key.startswith("gen_ai.usage.") + } + if span.name == "response": + assert usage == { + "gen_ai.usage.input_tokens": 3, + "gen_ai.usage.output_tokens": 5, + "gen_ai.usage.total_tokens": 8, + "gen_ai.usage.input_tokens.cache_read": 0, + "gen_ai.usage.output_tokens.reasoning": 0, + } + else: + assert usage == {} + + for key, expected in ( + ("gen_ai.usage.input_tokens", 3), + ("gen_ai.usage.output_tokens", 5), + ("gen_ai.usage.total_tokens", 8), + ): + assert sum(span.attributes.get(key, 0) for span in spans) == expected + + # Telemetry filtering must not alter the response the application receives. + assert response.model_dump(mode="json") == original diff --git a/python/frameworks/openai-agents/traceai_openai_agents/_processor.py b/python/frameworks/openai-agents/traceai_openai_agents/_processor.py index 242c0e31b..48684260b 100644 --- a/python/frameworks/openai-agents/traceai_openai_agents/_processor.py +++ b/python/frameworks/openai-agents/traceai_openai_agents/_processor.py @@ -96,24 +96,30 @@ def on_trace_end(self, trace: Trace) -> None: Args: trace: The trace that started. """ + initial_input = self._trace_inputs.pop(trace.trace_id, None) + final_output = self._trace_outputs.pop(trace.trace_id, None) if root_span := self._root_spans.pop(trace.trace_id, None): - if initial_input := self._trace_inputs.pop(trace.trace_id, None): - if isinstance(initial_input, str): - root_span.set_attribute(INPUT_VALUE, initial_input) - elif isinstance(initial_input, list): - root_span.set_attribute(INPUT_MIME_TYPE, JSON) - root_span.set_attribute(INPUT_VALUE, safe_json_dumps(initial_input)) - for k, v in _get_attributes_from_input(initial_input): + try: + if initial_input: + if isinstance(initial_input, str): + root_span.set_attribute(INPUT_VALUE, initial_input) + elif isinstance(initial_input, list): + root_span.set_attribute(INPUT_MIME_TYPE, JSON) + root_span.set_attribute(INPUT_VALUE, safe_json_dumps(initial_input)) + for k, v in _get_attributes_from_input(initial_input): + root_span.set_attribute(k, v) + if final_output: + root_span.set_attribute(OUTPUT_VALUE, _get_output_from_response(final_output)) + for k, v in _get_attributes_from_response(final_output, include_usage=False): root_span.set_attribute(k, v) - if final_output := self._trace_outputs.pop(trace.trace_id, None): - root_span.set_attribute(OUTPUT_VALUE, final_output.model_dump_json()) - if text_output := _get_text_from_response(final_output): - root_span.set_attribute(OUTPUT_VALUE, text_output) - for k, v in _get_attributes_from_response(final_output): - root_span.set_attribute(k, v) - - root_span.set_status(Status(StatusCode.OK)) - root_span.end() + except Exception: + # Exception text can contain input or output data. + logger.warning("Failed to convert trace attributes") + finally: + try: + root_span.set_status(Status(StatusCode.OK)) + finally: + root_span.end() def on_span_start(self, span: Span[Any]) -> None: """Called when a span is started. @@ -154,6 +160,24 @@ def on_span_end(self, span: Span[Any]) -> None: detach(token) # type: ignore[arg-type] if not (otel_span := self._otel_spans.pop(span.span_id, None)): return + try: + self._set_span_attributes(span, otel_span) + except Exception: + # Attribute conversion must not leak data or leave a recording span. + logger.warning("Failed to convert span attributes") + finally: + end_time: Optional[int] = None + try: + if span.ended_at: + try: + end_time = _as_utc_nano(datetime.fromisoformat(span.ended_at)) + except ValueError: + pass + otel_span.set_status(status=_get_span_status(span)) + finally: + otel_span.end(end_time) + + def _set_span_attributes(self, span: Span[Any], otel_span: OtelSpan) -> None: otel_span.update_name(_get_span_name(span)) # flatten_attributes: dict[str, AttributeValue] = dict(_flatten(span.export())) # otel_span.set_attributes(flatten_attributes) @@ -165,9 +189,7 @@ def on_span_end(self, span: Span[Any]) -> None: if hasattr(data, "response") and isinstance(response := data.response, Response): self._trace_outputs[span.trace_id] = response otel_span.set_attribute(OUTPUT_MIME_TYPE, JSON) - otel_span.set_attribute(OUTPUT_VALUE, response.model_dump_json()) - if text_output := _get_text_from_response(response): - otel_span.set_attribute(OUTPUT_VALUE, text_output) + otel_span.set_attribute(OUTPUT_VALUE, _get_output_from_response(response)) for k, v in _get_attributes_from_response(response): otel_span.set_attribute(k, v) if hasattr(data, "input") and (input := data.input): @@ -196,10 +218,8 @@ def on_span_end(self, span: Span[Any]) -> None: if hasattr(data, "response") and isinstance( response := data.response, Response ): - parent_otel_span.set_attribute(OUTPUT_VALUE, response.model_dump_json()) - if text_output := _get_text_from_response(response): - parent_otel_span.set_attribute(OUTPUT_VALUE, text_output) - for k, v in _get_attributes_from_response(response): + parent_otel_span.set_attribute(OUTPUT_VALUE, _get_output_from_response(response)) + for k, v in _get_attributes_from_response(response, include_usage=False): parent_otel_span.set_attribute(k, v) elif isinstance(data, GenerationSpanData): otel_span.set_attribute(INPUT_VALUE, safe_json_dumps(data.input)) @@ -227,15 +247,6 @@ def on_span_end(self, span: Span[Any]) -> None: if parent_node := self._reverse_handoffs_dict.pop(key, None): otel_span.set_attribute(GRAPH_NODE_PARENT_ID, parent_node) - end_time: Optional[int] = None - if span.ended_at: - try: - end_time = _as_utc_nano(datetime.fromisoformat(span.ended_at)) - except ValueError: - pass - otel_span.set_status(status=_get_span_status(span)) - otel_span.end(end_time) - def force_flush(self) -> None: """Forces an immediate flush of all queued spans/traces.""" # TODO @@ -591,6 +602,21 @@ def _get_attributes_from_message_content_list( assert_never(item["type"]) +def _get_output_from_response(response: Response) -> str: + """Keep echoed request fields out of output without mutating the SDK response.""" + if text_output := _get_text_from_response(response): + return text_output + # Match the OpenAI instrumentor's response-owned allowlist. A denylist would + # let new SDK request-echo fields bypass directional input privacy settings. + return response.model_dump_json( + include={ + "id", "object", "created_at", "model", "status", "output", "usage", + "incomplete_details", + }, + exclude_none=True, + ) + + def _get_text_from_response(response: Response) -> Optional[str]: """Extracts the concatenated text from a Response object.""" texts = [] @@ -604,9 +630,13 @@ def _get_text_from_response(response: Response) -> Optional[str]: return " ".join(texts) if texts else None -def _get_attributes_from_response(obj: Response) -> Iterator[tuple[str, AttributeValue]]: +def _get_attributes_from_response( + obj: Response, *, include_usage: bool = True +) -> Iterator[tuple[str, AttributeValue]]: yield from _get_attributes_from_tools(obj.tools) - yield from _get_attributes_from_usage(obj.usage) + # Wrapper spans copy response metadata, but only LLM requests contribute usage. + if include_usage: + yield from _get_attributes_from_usage(obj.usage) yield from _get_attributes_from_response_output(obj.output) if isinstance(obj.instructions, str): yield from _get_attributes_from_response_instruction(obj.instructions) @@ -661,11 +691,13 @@ def _get_attributes_from_response_output( yield from _get_attributes_from_function_tool_call(item, prefix) tool_call_idx += 1 elif item.type == "custom_tool_call": - yield f"{prefix}{MessageAttributes.MESSAGE_ROLE}", "assistant" + prefix = f"{GEN_AI_OUTPUT_MESSAGES}.{msg_idx}." + yield f"{prefix}{MESSAGE_ROLE}", "assistant" yield from _get_attributes_from_response_custom_tool_call( item, - f"{prefix}{MessageAttributes.MESSAGE_TOOL_CALLS}.0.", + f"{prefix}{MESSAGE_TOOL_CALLS}.{tool_call_idx}.", ) + tool_call_idx += 1 elif item.type == "file_search_call": ... # TODO elif item.type == "web_search_call": diff --git a/python/frameworks/openai-agents/traceai_openai_agents/version.py b/python/frameworks/openai-agents/traceai_openai_agents/version.py index 1b09c6e18..bad05c2e5 100644 --- a/python/frameworks/openai-agents/traceai_openai_agents/version.py +++ b/python/frameworks/openai-agents/traceai_openai_agents/version.py @@ -1 +1 @@ -__version__ = "0.1.7" # x-release-please-version +__version__ = "0.1.8" # x-release-please-version diff --git a/python/frameworks/openai/CHANGELOG.md b/python/frameworks/openai/CHANGELOG.md index 094eba4c8..83cab77a4 100644 --- a/python/frameworks/openai/CHANGELOG.md +++ b/python/frameworks/openai/CHANGELOG.md @@ -1,3 +1,8 @@ +## [0.1.15] - pending release + +### Fixed +- When input hiding is off, chat spans record the conversation's roles, tool calls and content parts as input messages, and the span's input text combines the messages' text. Cached input and reasoning token counts are recorded for regular and streamed calls when the response includes them. + ## [0.1.14] - 2026-11-10 ### Added - Add Azure client deployment, API version and endpoint host/explicit port span metadata through the existing OpenAI instrumentor, without recording full URLs or credentials (#154). diff --git a/python/frameworks/openai/pyproject.toml b/python/frameworks/openai/pyproject.toml index 26e0b31fa..9961fbd9e 100644 --- a/python/frameworks/openai/pyproject.toml +++ b/python/frameworks/openai/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "traceAI-openai" -version = "0.1.14" +version = "0.1.15" description = "OpenTelemetry instrumentation for OpenAI" authors = ["Future AGI "] readme = "README.md" diff --git a/python/frameworks/openai/tests/test_chat_input_value.py b/python/frameworks/openai/tests/test_chat_input_value.py new file mode 100644 index 000000000..55fc50a98 --- /dev/null +++ b/python/frameworks/openai/tests/test_chat_input_value.py @@ -0,0 +1,272 @@ +"""Canonical chat input must retain the full conversation.""" + +import asyncio +from copy import deepcopy +import json + +import openai +import pytest +from fi_instrumentation import TraceConfig +from fi_instrumentation.fi_types import SpanAttributes +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +from opentelemetry.trace import StatusCode +from traceai_openai import OpenAIInstrumentor + +if int(openai.__version__.split(".")[0]) >= 3: + import httpx2 as httpx +else: + import httpx + + +@pytest.fixture +def tracing(): + exporter = InMemorySpanExporter() + provider = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + instrumentor = OpenAIInstrumentor() + try: + instrumentor.instrument(tracer_provider=provider, config=TraceConfig()) + yield exporter + finally: + instrumentor.uninstrument() + provider.shutdown() + + +@pytest.mark.parametrize("async_", [False, True], ids=["sync", "async"]) +@pytest.mark.parametrize("stream", [False, True], ids=["ordinary", "streaming"]) +@pytest.mark.parametrize( + "messages, expected_value, expected_mime", + [ + pytest.param( + [ + {"role": "system", "content": "SYSTEM_CONTEXT_MARKER"}, + {"role": "user", "content": "USER_QUESTION_MARKER"}, + ], + "SYSTEM_CONTEXT_MARKER \n USER_QUESTION_MARKER", + "text/plain", + id="system-user", + ), + pytest.param( + [ + {"role": "system", "content": "SYSTEM_CONTEXT_MARKER"}, + {"role": "user", "content": "USER_QUESTION_MARKER"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_synthetic", + "type": "function", + "function": {"name": "lookup", "arguments": "{}"}, + } + ], + }, + { + "role": "tool", + "tool_call_id": "call_synthetic", + "content": "TOOL_RESULT_MARKER", + }, + { + "role": "user", + "content": [ + {"type": "text", "text": "FOLLOWUP_MARKER café"}, + {"type": "text", "text": "SECOND_TEXT_PART_MARKER"}, + { + "type": "image_url", + "image_url": { + "url": "https://synthetic.invalid/example.png", + "detail": "low", + }, + }, + ], + }, + ], + " \n ".join([ + "SYSTEM_CONTEXT_MARKER", "USER_QUESTION_MARKER", "None", + "TOOL_RESULT_MARKER", "FOLLOWUP_MARKER café", + "SECOND_TEXT_PART_MARKER", "{{" + f"{SpanAttributes.INPUT_IMAGES}.0" + "}}", + ]), + "text/plain", + id="multiturn-multimodal", + ), + pytest.param( + [{"role": "system", "content": "SYSTEM_CONTEXT_MARKER"}] + [ + message + for i in range(40) + for message in ( + {"role": "user", "content": f"QUESTION_{i}"}, + { + "role": "assistant", + "content": None, + "tool_calls": [{ + "id": f"call_{i}", + "type": "function", + "function": { + "name": "lookup", + "arguments": json.dumps({"query": f"item_{i}"}), + }, + }], + }, + {"role": "tool", "tool_call_id": f"call_{i}", "content": f"RESULT_{i}"}, + {"role": "assistant", "content": f"ANSWER_{i}"}, + ) + ], + " \n ".join(["SYSTEM_CONTEXT_MARKER"] + [ + text for i in range(40) + for text in (f"QUESTION_{i}", "None", f"RESULT_{i}", f"ANSWER_{i}") + ]), + "text/plain", + id="conversation-exceeds-attribute-count", + ), + pytest.param( + [{"role": "user", "content": [ + {"type": "image_url", "image_url": {"url": "https://synthetic.invalid/a.png"}}, + {"type": "image_url", "image_url": {"url": "https://synthetic.invalid/b.png"}}, + ]}], + " \n ".join("{{" + f"{SpanAttributes.INPUT_IMAGES}.{i}" + "}}" for i in range(2)), + "text/plain", + id="images-only", + ), + pytest.param( + [{"role": "user", "content": [{ + "type": "input_audio", "input_audio": {"data": "AAECAw==", "format": "wav"}, + }]}], + "[non-text content]", + "text/plain", + id="audio-only", + ), + pytest.param( + [{"role": "user", "content": [{ + "type": "file", "file": {"file_data": "AAECAw==", "filename": "sample.bin"}, + }]}], + "[non-text content]", + "text/plain", + id="file-only", + ), + pytest.param( + [{"role": "user", "content": '{"task": "sum"}'}], + '{"task": "sum"}', + "application/json", + id="single-json-message", + ), + pytest.param( + [ + {"role": "user", "content": '{"task": "sum"}'}, + {"role": "user", "content": "now explain it"}, + ], + '{"task": "sum"} \n now explain it', + "text/plain", + id="json-followed-by-text", + ), + ], +) +def test_chat_input_value_preserves_full_conversation( + tracing, async_, stream, messages, expected_value, expected_mime, +): + original = deepcopy(messages) + requests = [] + + def handle(request): + body = json.loads(request.content) + requests.append(body) + assert body["messages"] == original + assert body["stream"] is stream + response = { + "id": "chatcmpl_synthetic", + "object": "chat.completion", + "created": 1, + "model": "gpt-4o-mini", + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "ANSWER_MARKER"}, + "finish_reason": "stop", + } + ], + } + if stream: + response["object"] = "chat.completion.chunk" + choice = response["choices"][0] + choice["delta"] = choice.pop("message") + return httpx.Response( + 200, + headers={"content-type": "text/event-stream"}, + content=f"data: {json.dumps(response)}\n\ndata: [DONE]\n\n", + ) + return httpx.Response(200, json=response) + + params = {"model": "gpt-4o-mini", "messages": messages, "stream": stream} + + async def run_async(): + async with openai.AsyncOpenAI( + api_key="synthetic-test-key", + base_url="https://synthetic.invalid/v1", + max_retries=0, + http_client=httpx.AsyncClient( + transport=httpx.MockTransport(handle), trust_env=False + ), + ) as client: + result = await client.chat.completions.create(**params) + if stream: + parts = [chunk.choices[0].delta.content or "" async for chunk in result] + return "".join(parts) + return result.choices[0].message.content + + if async_: + answer = asyncio.run(run_async()) + else: + with openai.OpenAI( + api_key="synthetic-test-key", + base_url="https://synthetic.invalid/v1", + max_retries=0, + http_client=httpx.Client( + transport=httpx.MockTransport(handle), trust_env=False + ), + ) as client: + result = client.chat.completions.create(**params) + if stream: + answer = "".join(chunk.choices[0].delta.content or "" for chunk in result) + else: + answer = result.choices[0].message.content + + assert answer == "ANSWER_MARKER" + assert len(requests) == 1 + assert messages == original + (span,) = tracing.get_finished_spans() + assert span.status.status_code is StatusCode.OK + attrs = span.attributes + assert attrs["input.value"] == expected_value + assert attrs["input.value"].strip() + assert attrs["input.mime_type"] == expected_mime + assert json.loads(attrs[SpanAttributes.GEN_AI_INPUT_MESSAGES]) == original + if len(messages) in (2, 5) and messages[0]["role"] == "system": + assert attrs["gen_ai.input.messages.1.message.content"] == "USER_QUESTION_MARKER" + image_urls = [ + part["image_url"]["url"] + for message in messages if isinstance(message.get("content"), list) + for part in message["content"] if part["type"] == "image_url" + ] + if image_urls: + assert json.loads(attrs[SpanAttributes.INPUT_IMAGES]) == image_urls + + +@pytest.mark.parametrize( + "flag", ["hide_inputs", "hide_input_messages", "hide_input_text", "hide_input_images"], +) +def test_aggregate_messages_respect_input_privacy(flag): + from types import SimpleNamespace + from traceai_openai._request import _WithOpenAI + + request = SimpleNamespace( + _config=TraceConfig(**{flag: True}), + _openai=openai, + _get_span_kind=lambda **kwargs: "LLM", + ) + attributes = dict(_WithOpenAI._get_attributes_from_request( + request, + openai.types.chat.ChatCompletion, + {"model": "gpt-4o-mini", "messages": [{"role": "user", "content": "PRIVATE_INPUT"}]}, + )) + assert SpanAttributes.GEN_AI_INPUT_MESSAGES not in attributes diff --git a/python/frameworks/openai/tests/test_chat_usage_details.py b/python/frameworks/openai/tests/test_chat_usage_details.py new file mode 100644 index 000000000..5e7439498 --- /dev/null +++ b/python/frameworks/openai/tests/test_chat_usage_details.py @@ -0,0 +1,209 @@ +"""Chat usage details reach spans without changing SDK results.""" + +import asyncio +import json + +import httpx +import openai +import pytest +from fi_instrumentation import TraceConfig +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter + +from traceai_openai import OpenAIInstrumentor + + +@pytest.fixture +def exporter(): + exporter = InMemorySpanExporter() + provider = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + instrumentor = OpenAIInstrumentor() + instrumentor.instrument(tracer_provider=provider, config=TraceConfig()) + try: + yield exporter + finally: + instrumentor.uninstrument() + provider.shutdown() + + +@pytest.mark.parametrize("use_async", [False, True], ids=["sync", "async"]) +@pytest.mark.parametrize("stream", [False, True], ids=["nonstream", "stream"]) +@pytest.mark.parametrize( + "details,cached,reasoning", + [ + pytest.param( + { + "prompt_tokens_details": {"cached_tokens": 80}, + "completion_tokens_details": {"reasoning_tokens": 10}, + }, + 80, + 10, + id="nonzero", + ), + pytest.param( + { + "prompt_tokens_details": {"cached_tokens": 0}, + "completion_tokens_details": {"reasoning_tokens": 0}, + }, + 0, + 0, + id="zero", + ), + pytest.param({}, None, None, id="missing-details"), + pytest.param( + {"prompt_tokens_details": None, "completion_tokens_details": None}, + None, + None, + id="null-details", + ), + pytest.param( + {"prompt_tokens_details": {}, "completion_tokens_details": {}}, + None, + None, + id="missing-counts", + ), + pytest.param( + { + "prompt_tokens_details": {"cached_tokens": None}, + "completion_tokens_details": {"reasoning_tokens": None}, + }, + None, + None, + id="null-counts", + ), + pytest.param( + {"prompt_tokens_details": {"cached_tokens": 80}}, + 80, + None, + id="cache-only", + ), + pytest.param( + {"completion_tokens_details": {"reasoning_tokens": 10}}, + None, + 10, + id="reasoning-only", + ), + ], +) +def test_chat_usage_details(exporter, use_async, stream, details, cached, reasoning): + usage = { + "prompt_tokens": 100, + "completion_tokens": 20, + "total_tokens": 120, + **details, + } + common = {"id": "chatcmpl-test", "created": 1, "model": "gpt-4o-mini"} + + def handler(_request): + if not stream: + return httpx.Response( + 200, + json={ + **common, + "object": "chat.completion", + "choices": [{ + "index": 0, + "finish_reason": "stop", + "message": {"role": "assistant", "content": "Hello"}, + }], + "usage": usage, + }, + ) + chunks = [ + { + **common, + "object": "chat.completion.chunk", + "choices": [{ + "index": 0, + "delta": {"role": "assistant", "content": "Hello"}, + "finish_reason": None, + }], + "usage": None, + }, + { + **common, + "object": "chat.completion.chunk", + "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}], + "usage": None, + }, + { + **common, + "object": "chat.completion.chunk", + "choices": [], + "usage": usage, + }, + ] + body = "".join(f"data: {json.dumps(chunk)}\n\n" for chunk in chunks) + return httpx.Response( + 200, + headers={"content-type": "text/event-stream"}, + content=(body + "data: [DONE]\n\n").encode(), + ) + + kwargs = { + "model": "gpt-4o-mini", + "messages": [{"role": "user", "content": "hello"}], + "stream": stream, + } + if stream: + kwargs["stream_options"] = {"include_usage": True} + client_kwargs = { + "api_key": "test-only-key", + "base_url": "https://api.openai.com/v1", + "max_retries": 0, + } + + async def run_async(): + async with openai.AsyncOpenAI( + **client_kwargs, + http_client=httpx.AsyncClient( + transport=httpx.MockTransport(handler), trust_env=False + ), + ) as client: + response = await client.chat.completions.create(**kwargs) + return [chunk async for chunk in response] if stream else response + + if use_async: + result = asyncio.run(run_async()) + else: + with openai.OpenAI( + **client_kwargs, + http_client=httpx.Client( + transport=httpx.MockTransport(handler), trust_env=False + ), + ) as client: + response = client.chat.completions.create(**kwargs) + result = list(response) if stream else response + + if stream: + assert len(result) == 3 + assert result[-1].choices == [] + assert "".join( + choice.delta.content or "" for chunk in result for choice in chunk.choices + ) == "Hello" + sdk_usage = result[-1].usage + else: + assert result.choices[0].message.content == "Hello" + sdk_usage = result.usage + assert sdk_usage.prompt_tokens == 100 + assert sdk_usage.completion_tokens == 20 + assert sdk_usage.total_tokens == 120 + assert getattr(sdk_usage.prompt_tokens_details, "cached_tokens", None) == cached + assert getattr(sdk_usage.completion_tokens_details, "reasoning_tokens", None) == reasoning + + (span,) = exporter.get_finished_spans() + expected = { + "gen_ai.usage.input_tokens": 100, + "gen_ai.usage.output_tokens": 20, + "gen_ai.usage.total_tokens": 120, + } + if cached is not None: + expected["gen_ai.usage.input_tokens.cache_read"] = cached + if reasoning is not None: + expected["gen_ai.usage.output_tokens.reasoning"] = reasoning + assert { + key: value for key, value in span.attributes.items() + if key.startswith("gen_ai.usage.") + } == expected diff --git a/python/frameworks/openai/traceai_openai/_request.py b/python/frameworks/openai/traceai_openai/_request.py index 2f87156b5..111cdc6e3 100644 --- a/python/frameworks/openai/traceai_openai/_request.py +++ b/python/frameworks/openai/traceai_openai/_request.py @@ -205,6 +205,16 @@ def _get_attributes_from_request( if isinstance(model := request_parameters.get("model"), str) and model: yield SpanAttributes.GEN_AI_REQUEST_MODEL, model try: + if ( + cast_to is self._openai.types.chat.ChatCompletion + and isinstance(messages := request_parameters.get("messages"), list) + and not ( + self._config.hide_inputs or self._config.hide_input_messages + or self._config.hide_input_text or self._config.hide_input_images + ) + ): + # Keep the complete conversation primary so indexed details cannot evict it. + yield SpanAttributes.GEN_AI_INPUT_MESSAGES, safe_json_dumps(messages) yield from _as_input_attributes( _io_value_and_type(request_parameters), ) diff --git a/python/frameworks/openai/traceai_openai/_response_accumulator.py b/python/frameworks/openai/traceai_openai/_response_accumulator.py index 820779055..01a6df777 100644 --- a/python/frameworks/openai/traceai_openai/_response_accumulator.py +++ b/python/frameworks/openai/traceai_openai/_response_accumulator.py @@ -257,6 +257,14 @@ def get_attributes(self) -> Iterator[Tuple[str, AttributeValue]]: yield SpanAttributes.GEN_AI_USAGE_INPUT_TOKENS, prompt_tokens if completion_tokens is not None: yield SpanAttributes.GEN_AI_USAGE_OUTPUT_TOKENS, completion_tokens + prompt_details = usage.get("prompt_tokens_details") + if isinstance(prompt_details, Mapping): + if (cached_tokens := prompt_details.get("cached_tokens")) is not None: + yield SpanAttributes.GEN_AI_USAGE_INPUT_TOKENS_CACHE_READ, cached_tokens + completion_details = usage.get("completion_tokens_details") + if isinstance(completion_details, Mapping): + if (reasoning_tokens := completion_details.get("reasoning_tokens")) is not None: + yield SpanAttributes.GEN_AI_USAGE_OUTPUT_TOKENS_REASONING, reasoning_tokens # A tool-only completion has no text, but still has useful output. if not output_value and (result := self._result()): diff --git a/python/frameworks/openai/traceai_openai/_response_attributes_extractor.py b/python/frameworks/openai/traceai_openai/_response_attributes_extractor.py index 39331b6b5..604198c7e 100644 --- a/python/frameworks/openai/traceai_openai/_response_attributes_extractor.py +++ b/python/frameworks/openai/traceai_openai/_response_attributes_extractor.py @@ -311,6 +311,12 @@ def _get_attributes_from_completion_usage( yield SpanAttributes.GEN_AI_USAGE_INPUT_TOKENS, prompt_tokens if (completion_tokens := getattr(usage, "completion_tokens", None)) is not None: yield SpanAttributes.GEN_AI_USAGE_OUTPUT_TOKENS, completion_tokens + prompt_details = getattr(usage, "prompt_tokens_details", None) + if (cached_tokens := getattr(prompt_details, "cached_tokens", None)) is not None: + yield SpanAttributes.GEN_AI_USAGE_INPUT_TOKENS_CACHE_READ, cached_tokens + completion_details = getattr(usage, "completion_tokens_details", None) + if (reasoning_tokens := getattr(completion_details, "reasoning_tokens", None)) is not None: + yield SpanAttributes.GEN_AI_USAGE_OUTPUT_TOKENS_REASONING, reasoning_tokens def _get_attributes_from_embedding_usage( self, diff --git a/python/frameworks/openai/traceai_openai/_span_io_handler.py b/python/frameworks/openai/traceai_openai/_span_io_handler.py index 6898f4b23..7f7c35e0d 100644 --- a/python/frameworks/openai/traceai_openai/_span_io_handler.py +++ b/python/frameworks/openai/traceai_openai/_span_io_handler.py @@ -22,6 +22,7 @@ def _process_input_data(input_data: Any, span: _WithSpan) -> None: input_content = [] input_images = [] eval_input = [] + has_non_text_content = False for msg in input_data: if isinstance(msg, dict): msg_content = msg.get("content", "") @@ -32,7 +33,9 @@ def _process_input_data(input_data: Any, span: _WithSpan) -> None: if item.get("type") == "text": filtered_content.append(item) eval_input.append(item.get("text", "")) - elif item.get("type") == "image_url": + else: + has_non_text_content = True + if item.get("type") == "image_url": url_data = item.get("image_url", {}) url = url_data.get("url") if url: @@ -58,8 +61,10 @@ def _process_input_data(input_data: Any, span: _WithSpan) -> None: if input_images: images_value = json.dumps(input_images, ensure_ascii=False) span.set_attribute(SpanAttributes.INPUT_IMAGES, images_value) - if eval_input: - eval_input_str = " \n ".join(map(str, eval_input)) + eval_input_str = " \n ".join(map(str, eval_input)) + if not eval_input_str.strip() and has_non_text_content: + eval_input_str = "[non-text content]" + if eval_input or has_non_text_content: span.set_attribute(SpanAttributes.INPUT_VALUE, eval_input_str) # Describe the value actually exported (all messages joined), not the first message. _set_mime_type(span, SpanAttributes.INPUT_MIME_TYPE, eval_input_str) diff --git a/python/frameworks/openai/traceai_openai/version.py b/python/frameworks/openai/traceai_openai/version.py index af77e1197..3e9cf647d 100644 --- a/python/frameworks/openai/traceai_openai/version.py +++ b/python/frameworks/openai/traceai_openai/version.py @@ -1 +1 @@ -__version__ = "0.1.14" # x-release-please-version +__version__ = "0.1.15" # x-release-please-version diff --git a/python/frameworks/pipecat/CHANGELOG.md b/python/frameworks/pipecat/CHANGELOG.md index 09083a28b..710c5ce16 100644 --- a/python/frameworks/pipecat/CHANGELOG.md +++ b/python/frameworks/pipecat/CHANGELOG.md @@ -1,5 +1,10 @@ +## [0.1.3] - pending release + +### Changed +- traceai-pipecat now requires fi-instrumentation-otel 1.0.0 or later. Action: Upgrade fi-instrumentation-otel if your environment pins an earlier version. + ## [0.1.2] - 2026-10-29 ### Fixed diff --git a/python/frameworks/pipecat/poetry.lock b/python/frameworks/pipecat/poetry.lock index fa744960e..d46f6f1fc 100644 --- a/python/frameworks/pipecat/poetry.lock +++ b/python/frameworks/pipecat/poetry.lock @@ -900,4 +900,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.1" python-versions = ">3.9,<3.14" -content-hash = "7e76581228e27cd95e9956919555b7f30a8fc2e032ed5cd40ae12667e2510f66" +content-hash = "4b737cde85622ae5446064d605b28d51d270893abadcf99719c49821de8be34b" diff --git a/python/frameworks/pipecat/pyproject.toml b/python/frameworks/pipecat/pyproject.toml index a796d6b48..779fc299d 100644 --- a/python/frameworks/pipecat/pyproject.toml +++ b/python/frameworks/pipecat/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "traceAI-pipecat" -version = "0.1.2" +version = "0.1.3" description = "OpenTelemetry instrumentation for Pipecat" authors = ["Future AGI "] readme = "README.md" @@ -28,7 +28,7 @@ classifiers = [ [tool.poetry.dependencies] python = ">3.9,<3.14" -fi-instrumentation-otel = ">=0.1.14" +fi-instrumentation-otel = ">=1.0.0" [tool.poetry.urls] Homepage = "https://futureagi.com" diff --git a/python/frameworks/pipecat/traceai_pipecat/__init__.py b/python/frameworks/pipecat/traceai_pipecat/__init__.py index ec1cc7526..bcc4c5ce3 100644 --- a/python/frameworks/pipecat/traceai_pipecat/__init__.py +++ b/python/frameworks/pipecat/traceai_pipecat/__init__.py @@ -18,7 +18,7 @@ enable_http_attribute_mapping, ) -__version__ = "0.1.2" # x-release-please-version +__version__ = "0.1.3" # x-release-please-version __all__ = [ "enable_fi_attribute_mapping", diff --git a/python/frameworks/pipecat/traceai_pipecat/version.py b/python/frameworks/pipecat/traceai_pipecat/version.py index 0d33eef3f..97c339acc 100644 --- a/python/frameworks/pipecat/traceai_pipecat/version.py +++ b/python/frameworks/pipecat/traceai_pipecat/version.py @@ -1 +1 @@ -__version__ = "0.1.2" # x-release-please-version +__version__ = "0.1.3" # x-release-please-version diff --git a/python/frameworks/portkey/CHANGELOG.md b/python/frameworks/portkey/CHANGELOG.md index 16425b06d..fcf26c3e7 100644 --- a/python/frameworks/portkey/CHANGELOG.md +++ b/python/frameworks/portkey/CHANGELOG.md @@ -1,5 +1,10 @@ # Changelog +## [0.1.4] - pending release + +### Changed +- traceai-portkey now requires fi-instrumentation-otel 1.0.0 or later. Action: Upgrade fi-instrumentation-otel if your environment pins an earlier version. + ## [0.1.3] - 2026-10-29 ### Fixed diff --git a/python/frameworks/portkey/poetry.lock b/python/frameworks/portkey/poetry.lock index 69895a47b..e5460751c 100644 --- a/python/frameworks/portkey/poetry.lock +++ b/python/frameworks/portkey/poetry.lock @@ -1197,4 +1197,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.1" python-versions = ">3.9, <3.13" -content-hash = "ba50e521ef6608709bfaa78421de3952af51b4a62094c2a2524d1169a91fe264" +content-hash = "e9c70f414f4c9b822c62b4ae335e7596c4a916d8e58da73c6918fe1f626caf3f" diff --git a/python/frameworks/portkey/pyproject.toml b/python/frameworks/portkey/pyproject.toml index 6ff4a49ed..d83620582 100644 --- a/python/frameworks/portkey/pyproject.toml +++ b/python/frameworks/portkey/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "traceAI-portkey" -version = "0.1.3" +version = "0.1.4" description = "OpenTelemetry instrumentation for Portkey AI" authors = ["Future AGI "] readme = "README.md" @@ -29,7 +29,7 @@ classifiers = [ [tool.poetry.dependencies] python = ">3.9, <3.13" portkey-ai = ">0.1.0" -fi-instrumentation-otel = ">=0.1.11" +fi-instrumentation-otel = ">=1.0.0" [tool.poetry.urls] diff --git a/python/frameworks/portkey/traceai_portkey/version.py b/python/frameworks/portkey/traceai_portkey/version.py index 499255216..81d99b6cf 100644 --- a/python/frameworks/portkey/traceai_portkey/version.py +++ b/python/frameworks/portkey/traceai_portkey/version.py @@ -1 +1 @@ -__version__ = "0.1.3" # x-release-please-version \ No newline at end of file +__version__ = "0.1.4" # x-release-please-version \ No newline at end of file diff --git a/python/frameworks/pydantic-ai/CHANGELOG.md b/python/frameworks/pydantic-ai/CHANGELOG.md new file mode 100644 index 000000000..9dd227af2 --- /dev/null +++ b/python/frameworks/pydantic-ai/CHANGELOG.md @@ -0,0 +1,8 @@ +## [0.2.0] - pending release + +### Changed +- traceai-pydantic-ai now requires Python 3.10 or later. Action: Stay on the previous traceai-pydantic-ai release if you run Python 3.9. + +### Fixed +- Instrumented run_stream works as an async context manager, the tracing content privacy settings apply to captured content, and spans use the standard span kinds and content attributes. +- Streamed agent runs (run_stream) record their input and, once the stream is consumed, their output, like non-streamed runs. diff --git a/python/frameworks/pydantic-ai/pyproject.toml b/python/frameworks/pydantic-ai/pyproject.toml index 378c80682..81f0bb512 100644 --- a/python/frameworks/pydantic-ai/pyproject.toml +++ b/python/frameworks/pydantic-ai/pyproject.toml @@ -4,11 +4,11 @@ build-backend = "hatchling.build" [project] name = "traceai-pydantic-ai" -version = "0.1.0" +version = "0.2.0" description = "OpenTelemetry instrumentation for Pydantic AI" readme = "README.md" license = "MIT" -requires-python = ">=3.9" +requires-python = ">=3.10" authors = [ { name = "FutureAGI", email = "support@futureagi.com" } ] @@ -21,7 +21,6 @@ classifiers = [ "Programming Language :: Python", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3 :: Only", - "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", @@ -34,6 +33,8 @@ classifiers = [ keywords = ["pydantic-ai", "opentelemetry", "tracing", "observability", "llm", "agents", "traceai", "futureagi"] dependencies = [ + "fi-instrumentation-otel>=1.3.0", + "wrapt>=1.15.0,<2", "opentelemetry-api>=1.0.0", "opentelemetry-sdk>=1.0.0", "opentelemetry-instrumentation>=0.40b0", diff --git a/python/frameworks/pydantic-ai/tests/test_content_privacy.py b/python/frameworks/pydantic-ai/tests/test_content_privacy.py new file mode 100644 index 000000000..fe8a713f1 --- /dev/null +++ b/python/frameworks/pydantic-ai/tests/test_content_privacy.py @@ -0,0 +1,379 @@ +"""T5-PY-002: fake Pydantic operations, real SDK capture/export boundaries. + +No model requests or installed Pydantic AI are needed. Assert exported records, +including exception events and status descriptions, rather than display fields. +""" + +import asyncio +import sys +from types import ModuleType, SimpleNamespace + +import pytest +from fi_instrumentation import TraceConfig, suppress_tracing +from opentelemetry.sdk.trace import SpanProcessor, TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +from opentelemetry.trace import StatusCode + +from traceai_pydantic_ai import ( + PydanticAIInstrumentor, instrument_pydantic_ai, wrap_tool_function, +) +from traceai_pydantic_ai._attributes import PydanticAIAttributes as A + + +INPUT = "PRIVATE_INPUT_MARKER" +INSTRUCTIONS = "PRIVATE_INSTRUCTIONS_MARKER" +OUTPUT = "PRIVATE_OUTPUT_MARKER" +ERROR = "PRIVATE_ERROR_MARKER" + + +class CaptureStarts(SpanProcessor): + def __init__(self): + self.attributes = [] + + def on_start(self, span, parent_context=None): + self.attributes.append(dict(span.attributes)) + + def on_end(self, span): + pass + + def shutdown(self): + pass + + def force_flush(self, timeout_millis=30000): + return True + + +@pytest.fixture +def traced(monkeypatch): + for name in ( + "HIDE_INPUTS", "HIDE_OUTPUTS", "HIDE_INPUT_MESSAGES", "HIDE_OUTPUT_MESSAGES", + "HIDE_INPUT_TEXT", "HIDE_OUTPUT_TEXT", "HIDE_INPUT_IMAGES", "HIDE_ERROR_DETAILS", + "HIDE_LLM_INVOCATION_PARAMETERS", "HIDE_EMBEDDING_VECTORS", "PII_REDACTION", + "BASE64_IMAGE_MAX_LENGTH", + ): + monkeypatch.delenv("FI_" + name, raising=False) + + class FakeStream: + async def __aenter__(self): + return self + + async def __aexit__(self, *args): + return False + + def __aiter__(self): + return self + + async def __anext__(self): + raise StopAsyncIteration + + class FakeAgent: + name = "privacy-agent" + model = "openai:test-model" + instructions = INSTRUCTIONS + output_type = str + builtin_calls = [] + + def __init__(self): + self.error = None + self.received = None + self.result = SimpleNamespace( + output=OUTPUT, + usage=SimpleNamespace(input_tokens=2, output_tokens=3, total_tokens=5), + ) + + async def run(self, user_prompt, **kwargs): + self.received = (user_prompt, kwargs) + if self.error is not None: + raise self.error + return self.result + + def run_sync(self, user_prompt, **kwargs): + return asyncio.run(self.run(user_prompt, **kwargs)) + + async def run_stream(self, user_prompt, **kwargs): + self.received = (user_prompt, kwargs) + if self.error is not None: + raise self.error + return FakeStream() + + @classmethod + def instrument_all(cls, *args, **kwargs): + cls.builtin_calls.append((args, kwargs)) + + module = ModuleType("pydantic_ai") + module.Agent = FakeAgent + module.usage = SimpleNamespace(RequestUsage=object) + monkeypatch.setitem(sys.modules, "pydantic_ai", module) + monkeypatch.setattr(PydanticAIInstrumentor, "_instance", None) + monkeypatch.setattr(PydanticAIInstrumentor, "_is_instrumented", False) + monkeypatch.setattr(PydanticAIInstrumentor, "_is_instrumented_by_opentelemetry", False) + monkeypatch.setattr(PydanticAIInstrumentor, "instrumentation_dependencies", lambda self: ()) + + exporter = InMemorySpanExporter() + starts = CaptureStarts() + provider = TracerProvider() + provider.add_span_processor(starts) + provider.add_span_processor(SimpleSpanProcessor(exporter)) + instrumentor = PydanticAIInstrumentor() + + def start(stream_method=None, convenience=False, **kwargs): + if stream_method is not None: + FakeAgent.run_stream = stream_method + if convenience: + assert instrument_pydantic_ai(tracer_provider=provider, **kwargs) is instrumentor + else: + instrumentor.instrument(tracer_provider=provider, **kwargs) + return SimpleNamespace( + agent=FakeAgent(), exporter=exporter, starts=starts, instrumentor=instrumentor, + ) + + yield start + instrumentor.uninstrument() + provider.shutdown() + + +def run_agent(state, method): + options = {"message_history": ["harmless history"]} + if method == "run_sync": + result = state.agent.run_sync(INPUT, **options) + else: + result = asyncio.run(state.agent.run(INPUT, **options)) + assert result is state.agent.result + assert state.agent.received == (INPUT, options) + return state.exporter.get_finished_spans() + + +def captured(state): + return repr(state.starts.attributes) + "\n" + "\n".join( + span.to_json() for span in state.exporter.get_finished_spans() + ) + + +@pytest.mark.parametrize("method", ["run_sync", "run"]) +@pytest.mark.parametrize("source", ["config", "environment"]) +@pytest.mark.parametrize("flag", [ + "hide_inputs", "hide_input_messages", "hide_input_text", + "hide_outputs", "hide_output_messages", "hide_output_text", +]) +def test_directional_privacy_reaches_exporter(traced, monkeypatch, method, source, flag): + if source == "environment": + monkeypatch.setenv("FI_" + flag.upper(), "true") + state = traced() + else: + state = traced(config=TraceConfig(**{flag: True})) + spans = run_agent(state, method) + assert len(spans) == 1 + assert spans[0].attributes[A.MODEL_NAME] == "openai:test-model" + assert spans[0].attributes[A.USAGE_TOTAL_TOKENS] == 5 + text = captured(state) + if flag.startswith("hide_input"): + assert INPUT not in text + assert INSTRUCTIONS not in text + assert OUTPUT in text + else: + assert INPUT in text + assert INSTRUCTIONS in text + assert OUTPUT not in text + + +@pytest.mark.parametrize("method", ["run_sync", "run"]) +def test_explicit_false_overrides_environment_without_changing_results(traced, monkeypatch, method): + monkeypatch.setenv("FI_HIDE_INPUTS", "true") + monkeypatch.setenv("FI_HIDE_OUTPUTS", "true") + state = traced(config=TraceConfig(hide_inputs=False, hide_outputs=False)) + assert len(run_agent(state, method)) == 1 + for marker in (INPUT, INSTRUCTIONS, OUTPUT): + assert marker in captured(state) + + +@pytest.mark.parametrize("method", ["run_sync", "run"]) +@pytest.mark.parametrize("error_details", [None, False, True]) +def test_exception_policy_covers_aliases_events_and_status(traced, method, error_details): + state = traced(config=TraceConfig( + hide_inputs=True, hide_outputs=True, hide_error_details=error_details, + )) + error = ValueError(ERROR) + state.agent.error = error + with pytest.raises(ValueError) as caught: + run_agent(state, method) + assert caught.value is error + span, = state.exporter.get_finished_spans() + assert span.status.status_code is StatusCode.ERROR + assert span.events + assert span.attributes[A.ERROR_TYPE] == "ValueError" + if error_details is False: + assert ERROR in captured(state) + else: + assert ERROR not in captured(state) + assert span.status.description is None + assert all("exception.message" not in event.attributes for event in span.events) + + +@pytest.mark.parametrize("asynchronous", [False, True]) +@pytest.mark.parametrize("failure", [False, True]) +def test_tool_aliases_and_errors_obey_policy(traced, asynchronous, failure): + state = traced(config=TraceConfig(hide_inputs=True, hide_outputs=True)) + error = ValueError(ERROR) + + def tool(context, value): + assert value == INPUT + if failure: + raise error + return OUTPUT + + async def async_tool(context, value): + return tool(context, value) + + wrapped = wrap_tool_function( + async_tool if asynchronous else tool, + state.instrumentor._tracer, + "local_tool", + tool_description=INSTRUCTIONS, + ) + + def call(): + result = wrapped(None, value=INPUT) + return asyncio.run(result) if asynchronous else result + + if failure: + with pytest.raises(ValueError) as caught: + call() + assert caught.value is error + else: + assert call() == OUTPUT + assert len(state.exporter.get_finished_spans()) == 1 + for marker in (INPUT, INSTRUCTIONS, OUTPUT, ERROR): + assert marker not in captured(state) + + +@pytest.mark.parametrize("failure", ["none", "start", "body"]) +def test_stream_prompt_and_errors_obey_policy(traced, failure): + state = traced(config=TraceConfig(hide_inputs=True, hide_outputs=True)) + error = ValueError(ERROR) + if failure == "start": + state.agent.error = error + + async def consume(): + async with await state.agent.run_stream(INPUT): + if failure == "body": + raise error + + if failure == "none": + asyncio.run(consume()) + else: + with pytest.raises(ValueError) as caught: + asyncio.run(consume()) + assert caught.value is error + assert len(state.exporter.get_finished_spans()) == 1 + for marker in (INPUT, OUTPUT, ERROR): + assert marker not in captured(state) + + +def test_suppression_skips_export_but_not_application(traced): + state = traced(config=TraceConfig(hide_inputs=True, hide_outputs=True)) + with suppress_tracing(): + assert run_agent(state, "run_sync") == () + assert run_agent(state, "run") == () + assert state.starts.attributes == [] + assert len(run_agent(state, "run_sync")) == 1 + + +def test_suppression_covers_tools_and_streams(traced): + state = traced() + tool = wrap_tool_function(lambda: OUTPUT, state.instrumentor._tracer, "local_tool") + + async def consume(): + async with await state.agent.run_stream(INPUT): + pass + + with suppress_tracing(): + assert tool() == OUTPUT + asyncio.run(consume()) + assert state.exporter.get_finished_spans() == () + assert state.starts.attributes == [] + + +def test_convenience_function_forwards_config(traced): + state = traced(convenience=True, config=TraceConfig(hide_inputs=True, hide_outputs=True)) + assert len(run_agent(state, "run_sync")) == 1 + for marker in (INPUT, INSTRUCTIONS, OUTPUT): + assert marker not in captured(state) + + +def test_builtin_option_cannot_bypass_content_policy(traced): + state = traced(use_builtin=True, config=TraceConfig(hide_inputs=True, hide_outputs=True)) + assert len(run_agent(state, "run_sync")) == 1 + for marker in (INPUT, INSTRUCTIONS, OUTPUT): + assert marker not in captured(state) + assert state.agent.builtin_calls == [] + + +@pytest.mark.parametrize("use_builtin", [False, True]) +@pytest.mark.parametrize("failure", ["none", "enter", "exit", "body", "suppressed"]) +def test_native_stream_protocol_keeps_cleanup_and_exception_behavior(traced, use_builtin, failure): + from contextlib import asynccontextmanager + + error = ValueError(ERROR) + closed = [] + + @asynccontextmanager + async def native_stream(self, user_prompt): + assert user_prompt == INPUT + if failure == "enter": + raise error + try: + yield SimpleNamespace(content=OUTPUT) + except ValueError: + if failure != "suppressed": + raise + finally: + closed.append(True) + if failure == "exit": + raise error + + state = traced( + stream_method=native_stream, use_builtin=use_builtin, + config=TraceConfig(hide_inputs=True, hide_outputs=True), + ) + + async def consume(): + async with state.agent.run_stream(INPUT) as result: + assert result.content == OUTPUT + if failure in ("body", "suppressed"): + raise error + + if failure in ("enter", "exit", "body"): + with pytest.raises(ValueError) as caught: + asyncio.run(consume()) + assert caught.value is error + else: + asyncio.run(consume()) + assert closed == ([] if failure == "enter" else [True]) + assert len(state.exporter.get_finished_spans()) == 1 + for marker in (INPUT, OUTPUT, ERROR): + assert marker not in captured(state) + + +def test_alias_policy_applies_at_start_and_late_updates(traced): + state = traced(config=TraceConfig(hide_inputs=True, hide_outputs=True)) + private = { + A.RUN_PROMPT: INPUT, + A.AGENT_INSTRUCTIONS: INSTRUCTIONS, + A.RUN_RESULT: OUTPUT, + A.TOOL_ARGS: INPUT, + A.GEN_AI_TOOL_DESCRIPTION: INSTRUCTIONS, + A.TOOL_RESULT: OUTPUT, + A.MESSAGE_CONTENT: OUTPUT, + A.MESSAGE_TOOL_CALLS: INPUT, + A.ERROR_MESSAGE: ERROR, + A.TOOL_ERROR_MESSAGE: ERROR, + A.RETRY_REASON: ERROR, + A.VALIDATION_ERROR: ERROR, + } + with state.instrumentor._tracer.start_as_current_span("aliases", attributes=private) as span: + span.set_attributes(private) + span.add_event("aliases", attributes=private) + assert len(state.exporter.get_finished_spans()) == 1 + for marker in (INPUT, INSTRUCTIONS, OUTPUT, ERROR): + assert marker not in captured(state) diff --git a/python/frameworks/pydantic-ai/tests/test_default_execution_hooks.py b/python/frameworks/pydantic-ai/tests/test_default_execution_hooks.py new file mode 100644 index 000000000..2c9a61d75 --- /dev/null +++ b/python/frameworks/pydantic-ai/tests/test_default_execution_hooks.py @@ -0,0 +1,376 @@ +"""T5-PY-004: default registration must trace execution, not just Agent.run. + +These SDK-shaped fakes exercise the public Model request methods and the +FunctionToolset.call_tool (1.0) / Tool.run (historical) dispatch boundaries. +No provider, credentials, native instrument_all setting, or network is used. +""" + +import asyncio +import inspect +import sys +from contextlib import asynccontextmanager +from types import ModuleType, SimpleNamespace + +import pytest +from opentelemetry import trace +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter + +from traceai_pydantic_ai import PydanticAIInstrumentor + + +@pytest.fixture(params=["modern", "legacy"]) +def sdk(request, monkeypatch): + modern = request.param == "modern" + + class ModelRetry(Exception): + pass + + class Model: + model_name = "local-test-model" + system = "test" + + def __init_subclass__(cls, **kwargs): + super().__init_subclass__(**kwargs) + cls.subclass_initialized = True + + async def request(self, messages, model_settings, model_request_parameters): + raise NotImplementedError + + class LocalModel(Model): + def __init__(self): + self.calls = [] + self.responses = [] + self.failure = None + + async def request(self, messages, model_settings, model_request_parameters): + self.calls.append((messages, model_settings, model_request_parameters)) + if self.failure is not None: + raise self.failure + response = SimpleNamespace( + kind="response", output="done", + usage=SimpleNamespace(input_tokens=2, output_tokens=1, total_tokens=3), + ) + self.responses.append(response) + return response + + @asynccontextmanager + async def request_stream(self, messages, model_settings, model_request_parameters): + self.stream_entered = True + stream = SimpleNamespace(usage=lambda: SimpleNamespace( + input_tokens=2, output_tokens=1, total_tokens=3)) + self.stream = stream + try: + yield stream + finally: + self.stream_exited = True + + class FunctionToolset: + async def call_tool(self, name, tool_args, ctx, tool): + result = tool.function(**tool_args) + return await result if inspect.isawaitable(result) else result + + class Tool: + def __init__(self, function): + self.function = function + self.name = "local_tool" + self.description = "Return the supplied value." + self.current_retry = 0 + + async def run(self, message, run_context): + try: + result = self.function(**message.args) + if inspect.isawaitable(result): + result = await result + return SimpleNamespace(part_kind="tool-return", content=result) + except ModelRetry as exc: + self.current_retry += 1 + return SimpleNamespace(part_kind="retry-prompt", content=str(exc)) + + class Agent: + def __init__(self, model, function=None): + self.model = model + self.tool = Tool(function) if function else None + self.result = SimpleNamespace(output="done", new_messages=lambda: model.responses) + + async def run(self, user_prompt): + # Disabled native tracing may temporarily install an invalid span. + # Child hooks must still use the traceAI run as their parent. + with trace.use_span(trace.INVALID_SPAN): + await self.model.request([user_prompt], None, None) + if self.tool: + for retry in range(2): + ctx = SimpleNamespace(retry=retry) + try: + if modern: + self.tool_result = await FunctionToolset().call_tool( + name="local_tool", tool_args={"value": 7}, + ctx=ctx, tool=self.tool, + ) + else: + part = await self.tool.run( + SimpleNamespace(args={"value": 7}), ctx) + if part.part_kind == "retry-prompt": + raise ModelRetry(part.content) + self.tool_result = part.content + break + except ModelRetry: + await self.model.request(["retry"], None, None) + await self.model.request(["tool result"], None, None) + return self.result + + def run_sync(self, user_prompt): + return asyncio.run(self.run(user_prompt)) + + package = ModuleType("pydantic_ai") + package.__path__ = [] + package.Agent = Agent + models = ModuleType("pydantic_ai.models") + models.Model = Model + usage = ModuleType("pydantic_ai.usage") + # Keep these tests on response-local accounting in both dispatch variants. + usage.RequestUsage = SimpleNamespace + package.models, package.usage = models, usage + toolsets = ModuleType("pydantic_ai.toolsets") + toolsets.__path__ = [] + function = ModuleType("pydantic_ai.toolsets.function") + function.FunctionToolset = FunctionToolset + tools = ModuleType("pydantic_ai.tools") + tools.Tool = Tool + for name, module in { + "pydantic_ai": package, "pydantic_ai.models": models, + "pydantic_ai.usage": usage, "pydantic_ai.toolsets": toolsets, + "pydantic_ai.toolsets.function": function if modern else None, + "pydantic_ai.tools": tools, + }.items(): + monkeypatch.setitem(sys.modules, name, module) + + # Isolate BaseInstrumentor's singleton and enable through the public API. + monkeypatch.setattr(PydanticAIInstrumentor, "_instance", None) + monkeypatch.setattr(PydanticAIInstrumentor, "_is_instrumented", False) + instrumentor = PydanticAIInstrumentor() + monkeypatch.setattr(instrumentor, "instrumentation_dependencies", lambda: []) + exporter = InMemorySpanExporter() + provider = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + + def enable(): + instrumentor.instrument(tracer_provider=provider) + + try: + yield SimpleNamespace( + Agent=Agent, Model=Model, LocalModel=LocalModel, Tool=Tool, + FunctionToolset=FunctionToolset, ModelRetry=ModelRetry, + instrumentor=instrumentor, exporter=exporter, enable=enable, + ) + finally: + instrumentor.uninstrument() + provider.shutdown() + + +def execution_spans(sdk): + spans = sdk.exporter.get_finished_spans() + runs = [span for span in spans if span.name == "pydantic_ai.agent.run"] + tools = [span for span in spans if span.name.startswith("pydantic_ai.tool.call.")] + models = [span for span in spans if span.name == "pydantic_ai.model.request"] + return runs, tools, models + + +@pytest.mark.parametrize("method", ["run", "run_sync"]) +@pytest.mark.parametrize("async_tool", [False, True]) +def test_default_registration_traces_actual_tools_and_each_request(sdk, method, async_tool): + calls = [] + returned = {"answer": 7} + + def local_tool(value): + calls.append(value) + return returned + + async def async_local_tool(value): + return local_tool(value) + + # Existing models and tools must be covered without re-registration. + model = sdk.LocalModel() + agent = sdk.Agent(model, async_local_tool if async_tool else local_tool) + sdk.enable() + result = (asyncio.run(agent.run("use tool")) if method == "run" + else agent.run_sync("use tool")) + assert result is agent.result + assert agent.tool_result is returned + assert calls == [7] + assert model.calls == [(["use tool"], None, None), (["tool result"], None, None)] + runs, tools, models = execution_spans(sdk) + assert len(runs) == 1 + assert len(tools) == 1 # RED: the default instrumentor installs no tool hook. + assert len(models) == 2 + for child in tools + models: + assert child.parent.span_id == runs[0].context.span_id + assert child.context.trace_id == runs[0].context.trace_id + assert child.status.status_code.name == "OK" + assert tools[0].attributes["pydantic_ai.tool.name"] == "local_tool" + assert tools[0].attributes["pydantic_ai.tool.args"] == '{"value": 7}' + assert tools[0].attributes["pydantic_ai.tool.result"] == '{"answer": 7}' + assert all(span.attributes["gen_ai.request.model"] == "local-test-model" for span in models) + assert all(span.attributes["gen_ai.usage.input_tokens"] == 2 for span in models) + + +def test_caught_model_retry_keeps_success_and_records_each_attempt(sdk): + calls = [] + retry = sdk.ModelRetry("try once more") + + def local_tool(value): + calls.append(value) + if len(calls) == 1: + raise retry + return value + + agent = sdk.Agent(sdk.LocalModel(), local_tool) + sdk.enable() + assert agent.run_sync("retry") is agent.result + assert calls == [7, 7] + runs, tools, models = execution_spans(sdk) + assert len(runs) == 1 + assert runs[0].status.status_code.name == "OK" + assert len(models) == 3 + assert len(tools) == 2 + assert tools[0].attributes["pydantic_ai.tool.is_error"] is True + assert tools[0].attributes["pydantic_ai.tool.error_message"] == str(retry) + assert tools[0].status.status_code.name == "ERROR" + assert tools[1].attributes["pydantic_ai.tool.is_error"] is False + assert tools[1].attributes["pydantic_ai.tool.retry_count"] == 1 + assert tools[1].status.status_code.name == "OK" + assert all(span.parent.span_id == runs[0].context.span_id for span in tools + models) + + +@pytest.mark.parametrize("failure_site", ["tool", "model"]) +def test_execution_errors_preserve_exception_identity(sdk, failure_site): + error = ValueError("local execution failed") + + def fail(value): + raise error + + model = sdk.LocalModel() + if failure_site == "model": + model.failure = error + agent = sdk.Agent(model, fail if failure_site == "tool" else None) + sdk.enable() + with pytest.raises(ValueError) as caught: + agent.run_sync("fail") + assert caught.value is error + runs, tools, models = execution_spans(sdk) + child = tools if failure_site == "tool" else models + assert len(child) == 1 + assert child[0].status.status_code.name == "ERROR" + assert child[0].events[0].name == "exception" + assert child[0].parent.span_id == runs[0].context.span_id + assert runs[0].status.status_code.name == "ERROR" + + +def test_late_model_classes_and_teardown_are_not_missed(sdk): + original_request = sdk.LocalModel.request + original_stream = sdk.LocalModel.request_stream + original_init_subclass = sdk.Model.__dict__["__init_subclass__"] + original_tool = sdk.FunctionToolset.call_tool + original_legacy_tool = sdk.Tool.run + sdk.enable() + + async def request(self, messages, model_settings, model_request_parameters): + # Super delegation is still one model request, not two child spans. + return await super(LateModel, self).request( + messages, model_settings, model_request_parameters) + + LateModel = type("LateModel", (sdk.LocalModel,), {"request": request}) + assert LateModel.subclass_initialized + agent = sdk.Agent(LateModel()) + assert agent.run_sync("late") is agent.result + assert len(execution_spans(sdk)[2]) == 1 + sdk.instrumentor.uninstrument() + assert sdk.LocalModel.request is original_request + assert sdk.LocalModel.request_stream is original_stream + assert sdk.Model.__dict__["__init_subclass__"] is original_init_subclass + assert LateModel.request is request + assert sdk.FunctionToolset.call_tool is original_tool + assert sdk.Tool.run is original_legacy_tool + sdk.exporter.clear() + agent.run_sync("not traced") + assert sdk.exporter.get_finished_spans() == () + sdk.enable() + agent.run_sync("traced again") + assert len(execution_spans(sdk)[2]) == 1 + + +def test_request_preserves_arguments_and_response_identity(sdk): + sdk.enable() + model = sdk.LocalModel() + messages, options, parameters = [], {}, object() + response = asyncio.run(model.request( + messages, model_settings=options, model_request_parameters=parameters)) + assert response is model.responses[0] + observed = model.calls[0] + assert observed[0] is messages + assert observed[1] is options + assert observed[2] is parameters + assert len(execution_spans(sdk)[2]) == 1 + + +def test_model_without_own_subclass_hook_is_restored(sdk): + del sdk.Model.__init_subclass__ + sdk.enable() + + class LateModel(sdk.LocalModel): + async def request(self, *args, **kwargs): + return await super().request(*args, **kwargs) + + agent = sdk.Agent(LateModel()) + assert agent.run_sync("late") is agent.result + assert len(execution_spans(sdk)[2]) == 1 + sdk.instrumentor.uninstrument() + assert "__init_subclass__" not in sdk.Model.__dict__ + + +def test_stream_request_keeps_context_manager_and_result_identity(sdk): + model = sdk.LocalModel() + sdk.enable() + + async def execute(): + async with model.request_stream([], None, None) as stream: + assert stream is model.stream + assert model.stream_entered + assert not sdk.exporter.get_finished_spans() + assert model.stream_exited + + asyncio.run(execute()) + models = execution_spans(sdk)[2] + assert len(models) == 1 + assert models[0].attributes["gen_ai.usage.total_tokens"] == 3 + assert models[0].status.status_code.name == "OK" + + +@pytest.mark.parametrize("failure", [False, True]) +def test_stream_manager_can_enter_and_exit_in_different_tasks(sdk, failure): + model = sdk.LocalModel() + sdk.enable() + error = ValueError("consumer failed") if failure else None + + async def execute(): + previous = trace.get_current_span() + manager = model.request_stream([], None, None) + stream = await asyncio.create_task(manager.__aenter__()) + assert stream is model.stream + assert model.stream_entered + assert trace.get_current_span() is previous + assert sdk.exporter.get_finished_spans() == () + suppressed = await asyncio.create_task(manager.__aexit__( + type(error) if error else None, error, None)) + assert not suppressed + assert model.stream_exited + assert trace.get_current_span() is previous + + asyncio.run(execute()) + span, = execution_spans(sdk)[2] + assert span.status.status_code.name == ("ERROR" if failure else "OK") + if error: + assert span.attributes["pydantic_ai.error.message"] == str(error) + else: + assert span.attributes["gen_ai.usage.total_tokens"] == 3 diff --git a/python/frameworks/pydantic-ai/tests/test_explicit_provider_routing.py b/python/frameworks/pydantic-ai/tests/test_explicit_provider_routing.py new file mode 100644 index 000000000..eb3459969 --- /dev/null +++ b/python/frameworks/pydantic-ai/tests/test_explicit_provider_routing.py @@ -0,0 +1,175 @@ +"""Real offline agent runs must honor a non-global provider in either mode. + +Each case gets a fresh interpreter because OpenTelemetry's global provider can +only be assigned once. No private provider reset or fake agent is involved. +""" + +import json +import os +from pathlib import Path +import subprocess +import sys + +import pytest + + +@pytest.mark.parametrize("global_state", ["unset", "different"]) +@pytest.mark.parametrize("method", ["run", "run_sync"]) +@pytest.mark.parametrize("hidden", [False, True], ids=["visible", "hidden"]) +@pytest.mark.parametrize("use_builtin", [True, False], ids=["builtin-alias", "wrapper"]) +def test_explicit_provider_routing(global_state, method, hidden, use_builtin, tmp_path): + # Do not pass application credentials, native tracing settings, or content + # policy environment variables from the parent test process into the probe. + env = {key: os.environ[key] for key in ( + "PATH", "SYSTEMROOT", "WINDIR", "TMP", "TEMP", "TMPDIR", + ) if key in os.environ} + env["HOME"] = str(tmp_path) + completed = subprocess.run( + [sys.executable, "-I", "-B", str(Path(__file__).resolve()), + global_state, method, str(int(hidden)), str(int(use_builtin))], + cwd=tmp_path, env=env, capture_output=True, text=True, timeout=60, + ) + assert completed.returncode == 0, completed.stdout + completed.stderr + result = json.loads(completed.stdout) + assert result["agent_spans"] == 1 + assert result["model_spans"] == 1 + assert result["global_spans"] == 0 + assert result["global_unchanged"] is True + print("ROUTING_PROBE " + completed.stdout.strip()) + + +def _probe(global_state, method, hidden, use_builtin): + import asyncio + from importlib.metadata import version + import socket + + # A model regression must fail locally rather than attempt an API call. + def no_network(*args, **kwargs): + raise AssertionError("The routing probe must not use the network") + + socket.socket.connect = no_network + socket.socket.connect_ex = no_network + socket.create_connection = no_network + + from fi_instrumentation import TraceConfig + from opentelemetry import trace + from opentelemetry.sdk.trace import SpanProcessor, TracerProvider + from opentelemetry.sdk.trace.export import SimpleSpanProcessor + from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter + from pydantic_ai import Agent, models + from pydantic_ai.models.test import TestModel + import traceai_pydantic_ai + from traceai_pydantic_ai import instrument_pydantic_ai + + models.ALLOW_MODEL_REQUESTS = False + prompt = "SYNTHETIC_ROUTING_INPUT" + output = "SYNTHETIC_ROUTING_OUTPUT" + + class CaptureStarts(SpanProcessor): + def __init__(self): + self.attributes = [] + + def on_start(self, span, parent_context=None): + self.attributes.append(dict(span.attributes)) + + def on_end(self, span): + pass + + def shutdown(self): + pass + + def force_flush(self, timeout_millis=30000): + return True + + exporter = InMemorySpanExporter() + explicit_provider = TracerProvider() + starts = CaptureStarts() + explicit_provider.add_span_processor(starts) + explicit_provider.add_span_processor(SimpleSpanProcessor(exporter)) + global_exporter = InMemorySpanExporter() + other_provider = TracerProvider() + other_provider.add_span_processor(SimpleSpanProcessor(global_exporter)) + original_global = trace.get_tracer_provider() + assert isinstance(original_global, trace.ProxyTracerProvider) + if global_state == "different": + trace.set_tracer_provider(other_provider) + global_before = trace.get_tracer_provider() + assert global_before is not explicit_provider + + # Positive control: a separately configured global exporter really works. + # With no global set, creating a tracer must leave it a no-op instead. + with trace.get_tracer("routing-control").start_as_current_span("control") as span: + assert span.is_recording() is (global_state == "different") + assert len(global_exporter.get_finished_spans()) == int(global_state == "different") + global_exporter.clear() + + agent = Agent(TestModel(custom_output_text=output)) + + def run(): + result = (asyncio.run(agent.run(prompt)) if method == "run" + else agent.run_sync(prompt)) + assert result.output == output + + # Existing instances must be traced, and teardown must stop exporting. + run() + assert not exporter.get_finished_spans() + assert not global_exporter.get_finished_spans() + instrumentor = instrument_pydantic_ai( + tracer_provider=explicit_provider, use_builtin=use_builtin, + config=TraceConfig(hide_inputs=hidden, hide_outputs=hidden), + ) + try: + assert instrumentor.is_instrumented + assert trace.get_tracer_provider() is global_before + run() + explicit_provider.force_flush() + other_provider.force_flush() + spans = exporter.get_finished_spans() + runs = [span for span in spans if span.name == "pydantic_ai.agent.run"] + requests = [span for span in spans if span.name == "pydantic_ai.model.request"] + assert len(runs) == 1, [span.name for span in spans] + assert len(requests) == 1, [span.name for span in spans] + assert len(spans) == 2, [span.name for span in spans] + assert requests[0].parent.span_id == runs[0].context.span_id + assert requests[0].context.trace_id == runs[0].context.trace_id + assert all(span.status.status_code.name == "OK" for span in spans) + assert not global_exporter.get_finished_spans() + assert trace.get_tracer_provider() is global_before + captured = repr(starts.attributes) + "\n".join(span.to_json() for span in spans) + if hidden: + assert prompt not in captured + assert output not in captured + else: + assert prompt in captured + assert output in captured + + instrumentor.uninstrument() + exporter.clear() + run() + assert not exporter.get_finished_spans() + assert not global_exporter.get_finished_spans() + assert trace.get_tracer_provider() is global_before + print(json.dumps({ + "global_state": global_state, "method": method, + "hidden": hidden, "use_builtin": use_builtin, + "agent_spans": len(runs), "model_spans": len(requests), + "global_spans": len(global_exporter.get_finished_spans()), + "global_unchanged": True, "privacy_checked": True, + "uninstrumented_and_teardown_controls": True, + "instrumentation_file": traceai_pydantic_ai.__file__, + "versions": {name: version(name) for name in ( + "pydantic-ai", "pydantic-ai-slim", "pydantic", + "opentelemetry-sdk", "fi-instrumentation-otel", + )}, + })) + finally: + if instrumentor.is_instrumented: + instrumentor.uninstrument() + explicit_provider.shutdown() + other_provider.shutdown() + + +if __name__ == "__main__": + # Resolve this checkout, not an editable install from a different tree. + sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + _probe(sys.argv[1], sys.argv[2], bool(int(sys.argv[3])), bool(int(sys.argv[4]))) diff --git a/python/frameworks/pydantic-ai/tests/test_integration.py b/python/frameworks/pydantic-ai/tests/test_integration.py index 6a825f13a..251732c30 100644 --- a/python/frameworks/pydantic-ai/tests/test_integration.py +++ b/python/frameworks/pydantic-ai/tests/test_integration.py @@ -262,10 +262,12 @@ def test_agent_with_tools(self, memory_exporter): try: agent = Agent(TestModel()) + calls = [] @agent.tool def add_numbers(ctx: RunContext, a: int, b: int) -> int: """Add two numbers together.""" + calls.append((a, b)) return a + b result = agent.run_sync( @@ -274,9 +276,22 @@ def add_numbers(ctx: RunContext, a: int, b: int) -> int: spans = exporter.get_finished_spans() - # Should have agent run span - agent_spans = [s for s in spans if "agent.run" in s.name] - assert len(agent_spans) >= 1 + # Default instrumentation must observe execution, not registration. + import json + assert len(calls) == 1 + agent_spans = [s for s in spans if s.name == "pydantic_ai.agent.run"] + tool_spans = [s for s in spans if s.name == "pydantic_ai.tool.call.add_numbers"] + model_spans = [s for s in spans if s.name == "pydantic_ai.model.request"] + assert len(agent_spans) == 1 + assert len(tool_spans) == 1 + assert len(model_spans) == 2 + for child in tool_spans + model_spans: + assert child.parent.span_id == agent_spans[0].context.span_id + assert child.context.trace_id == agent_spans[0].context.trace_id + a, b = calls[0] + assert json.loads(tool_spans[0].attributes["pydantic_ai.tool.args"]) == {"a": a, "b": b} + assert tool_spans[0].attributes["pydantic_ai.tool.result"] == str(a + b) + assert tool_spans[0].attributes["pydantic_ai.tool.is_error"] is False finally: instrumentor.uninstrument() diff --git a/python/frameworks/pydantic-ai/tests/test_run_content_contract.py b/python/frameworks/pydantic-ai/tests/test_run_content_contract.py new file mode 100644 index 000000000..25f95ac51 --- /dev/null +++ b/python/frameworks/pydantic-ai/tests/test_run_content_contract.py @@ -0,0 +1,169 @@ +"""T5-PY-003: exported run attributes satisfy FutureAGI's content contract. + +Only the wrapped library is faked; spans use the real OTel SDK/exporter. Literal +canonical keys pin the collector boundary without a network or a collector. +""" + +import asyncio +import sys +from types import ModuleType, SimpleNamespace + +import pytest +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter + +from traceai_pydantic_ai import PydanticAIInstrumentor + + +@pytest.fixture +def traced_agent(monkeypatch): + class FakeAgent: + def __init__(self, result, child=None, error=None): + self.result = result + self.child = child + self.error = error + self.calls = [] + + async def run(self, user_prompt, **kwargs): + self.calls.append((user_prompt, kwargs)) + if self.error is not None: + raise self.error + if self.child is not None: + self.child_result = await self.child.run(user_prompt="child question") + return self.result + + def run_sync(self, *args, **kwargs): + # Like the SDK, run_sync delegates to the instrumented async run. + return asyncio.run(self.run(*args, **kwargs)) + + library = ModuleType("pydantic_ai") + library.Agent = FakeAgent + library.usage = SimpleNamespace(RequestUsage=object) + monkeypatch.setitem(sys.modules, "pydantic_ai", library) + monkeypatch.setattr(PydanticAIInstrumentor, "_instance", None) + monkeypatch.setattr(PydanticAIInstrumentor, "_is_instrumented", False) + monkeypatch.setattr( + PydanticAIInstrumentor, "instrumentation_dependencies", lambda self: () + ) + + exporter = InMemorySpanExporter() + provider = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + instrumentor = PydanticAIInstrumentor() + try: + instrumentor.instrument(tracer_provider=provider) + assert instrumentor.is_instrumented + yield FakeAgent, exporter + finally: + instrumentor.uninstrument() + provider.shutdown() + + +def invoke(agent, method, *args, **kwargs): + result = getattr(agent, method)(*args, **kwargs) + return asyncio.run(result) if method == "run" else result + + +def assert_content_contract(span, prompt, output, method): + attributes = span.attributes + assert span.name == "pydantic_ai.agent.run" + assert attributes["pydantic_ai.run.method"] == method + assert attributes["pydantic_ai.span_kind"] == "agent_run" + assert attributes["pydantic_ai.run.prompt"] == prompt + assert attributes["pydantic_ai.run.result"] == output + assert attributes["input.value"] == prompt + assert attributes["output.value"] == output + # safe_serialize produces bounded text (even structured values may truncate). + assert attributes["input.mime_type"] == "text/plain" + assert attributes["output.mime_type"] == "text/plain" + assert attributes["gen_ai.span.kind"] == "AGENT" + + +@pytest.mark.parametrize("method", ["run_sync", "run"]) +@pytest.mark.parametrize("result_field", ["output", "data"]) +@pytest.mark.parametrize("keyword_prompt", [False, True]) +def test_root_run_exports_collector_content( + traced_agent, method, result_field, keyword_prompt +): + Agent, exporter = traced_agent + expected = SimpleNamespace(**{result_field: "answer"}) + agent = Agent(expected) + deps = object() + if keyword_prompt: + result = invoke(agent, method, user_prompt="question", deps=deps) + else: + result = invoke(agent, method, "question", deps=deps) + + assert result is expected + assert agent.calls == [("question", {"deps": deps})] + (span,) = exporter.get_finished_spans() + assert span.parent is None + assert_content_contract(span, "question", "answer", method) + + +@pytest.mark.parametrize("method", ["run_sync", "run"]) +def test_nested_runs_export_independent_collector_content(traced_agent, method): + Agent, exporter = traced_agent + child_result = SimpleNamespace(output="child answer") + parent_result = SimpleNamespace(output="parent answer") + child = Agent(child_result) + parent = Agent(parent_result, child=child) + + assert invoke(parent, method, "parent question") is parent_result + assert parent.child_result is child_result + nested, outer = exporter.get_finished_spans() + assert outer.parent is None + assert nested.parent.span_id == outer.context.span_id + assert nested.context.trace_id == outer.context.trace_id + assert_content_contract(nested, "child question", "child answer", "run") + assert_content_contract(outer, "parent question", "parent answer", method) + + +@pytest.mark.parametrize("method", ["run_sync", "run"]) +@pytest.mark.parametrize( + "output, serialized", + [ + pytest.param("", "", id="empty-text"), + pytest.param(False, "False", id="false"), + pytest.param(0, "0", id="zero"), + pytest.param({"answer": 42}, '{"answer": 42}', id="mapping"), + pytest.param(["answer"], '["answer"]', id="list"), + pytest.param("a" * 2500, "a" * 1997 + "...", id="bounded-text"), + ], +) +def test_canonical_content_reuses_compatibility_serialization( + traced_agent, method, output, serialized +): + Agent, exporter = traced_agent + expected = SimpleNamespace(output=output) + agent = Agent(expected) + prompt = "q" * 2500 + + assert invoke(agent, method, user_prompt=prompt) is expected + assert agent.calls == [(prompt, {})] + (span,) = exporter.get_finished_spans() + assert_content_contract(span, "q" * 1997 + "...", serialized, method) + + +@pytest.mark.parametrize("method", ["run_sync", "run"]) +def test_failed_run_exports_input_and_kind_without_inventing_output( + traced_agent, method +): + Agent, exporter = traced_agent + error = ValueError("fake agent failure") + agent = Agent(None, error=error) + + with pytest.raises(ValueError) as caught: + invoke(agent, method, "question") + + assert caught.value is error + (span,) = exporter.get_finished_spans() + assert span.status.status_code.name == "ERROR" + assert span.attributes["pydantic_ai.run.prompt"] == "question" + assert span.attributes["input.value"] == "question" + assert span.attributes["input.mime_type"] == "text/plain" + assert span.attributes["gen_ai.span.kind"] == "AGENT" + assert "output.value" not in span.attributes + assert "output.mime_type" not in span.attributes + assert "pydantic_ai.run.result" not in span.attributes diff --git a/python/frameworks/pydantic-ai/tests/test_run_stream.py b/python/frameworks/pydantic-ai/tests/test_run_stream.py new file mode 100644 index 000000000..88159ee62 --- /dev/null +++ b/python/frameworks/pydantic-ai/tests/test_run_stream.py @@ -0,0 +1,239 @@ +"""T5-PY-001: preserve the native streaming context-manager protocol.""" + +import asyncio +from contextlib import asynccontextmanager +from types import SimpleNamespace + +import pytest +from opentelemetry import trace as trace_api +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter + +from traceai_pydantic_ai import PydanticAIInstrumentor +from traceai_pydantic_ai._attributes import PydanticAIAttributes + + +@pytest.fixture +def tracing(monkeypatch): + monkeypatch.setattr(PydanticAIInstrumentor, "_instance", None) + monkeypatch.setattr(PydanticAIInstrumentor, "_is_instrumented", False) + provider = TracerProvider() + exporter = InMemorySpanExporter() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + instrumentor = PydanticAIInstrumentor() + instrumentor._tracer = provider.get_tracer(__name__) + try: + yield instrumentor, exporter, provider + finally: + if instrumentor.is_instrumented: + instrumentor.uninstrument() + provider.shutdown() + + +def wrapped_agent(instrumentor, original): + class FakeAgent: + model = None + + FakeAgent.run_stream = instrumentor._wrap_run_stream(original) + return FakeAgent() + + +def test_run_stream_supports_async_with_and_keeps_span_current(tracing): + instrumentor, exporter, _ = tracing + events = [] + active_spans = [] + dependency = object() + + def observe(event): + span = trace_api.get_current_span() + assert span.is_recording() + assert not exporter.get_finished_spans() + events.append(event) + active_spans.append(span) + + async def stream_text(): + observe("consume") + yield "hello" + + @asynccontextmanager + async def run_stream(self, prompt, *, deps): + assert self is agent + assert prompt == "hello" + assert deps is dependency + observe("enter") + try: + yield SimpleNamespace(stream_text=stream_text) + finally: + observe("exit") + + agent = wrapped_agent(instrumentor, run_stream) + + async def execute(): + previous_span = trace_api.get_current_span() + async with agent.run_stream("hello", deps=dependency) as result: + assert [part async for part in result.stream_text()] == ["hello"] + assert trace_api.get_current_span() is previous_span + + asyncio.run(execute()) + assert events == ["enter", "consume", "exit"] + assert all(span is active_spans[0] for span in active_spans) + spans = exporter.get_finished_spans() + assert len(spans) == 1 + assert spans[0].name == "pydantic_ai.agent.run_stream" + assert spans[0].status.status_code is trace_api.StatusCode.OK + assert spans[0].attributes[PydanticAIAttributes.IS_ERROR] is False + assert spans[0].attributes[PydanticAIAttributes.DURATION_MS] >= 0 + + +@pytest.mark.parametrize("suppress", [False, True], ids=["propagate", "suppress"]) +def test_run_stream_preserves_manager_exit_semantics(tracing, suppress): + instrumentor, exporter, _ = tracing + error = ValueError("consumer failed") + exits = [] + entered_result = SimpleNamespace(value="hello") + + class Manager: + async def __aenter__(self): + return entered_result + + async def __aexit__(self, exc_type, exc_value, traceback): + assert trace_api.get_current_span().is_recording() + exits.append((exc_type, exc_value, traceback)) + return suppress + + agent = wrapped_agent(instrumentor, lambda self, prompt: Manager()) + + async def execute(): + async with agent.run_stream("hello") as result: + assert result.value == "hello" + raise error + + if suppress: + asyncio.run(execute()) + else: + with pytest.raises(ValueError) as caught: + asyncio.run(execute()) + assert caught.value is error + + assert len(exits) == 1 + assert exits[0][0] is ValueError + assert exits[0][1] is error + assert exits[0][2] is not None + spans = exporter.get_finished_spans() + assert len(spans) == 1 + assert spans[0].attributes[PydanticAIAttributes.IS_ERROR] is (not suppress) + + +@pytest.mark.parametrize("phase", ["factory", "enter", "consume", "exit"]) +@pytest.mark.parametrize("error_type", [ValueError, asyncio.CancelledError]) +def test_run_stream_ends_span_on_failure(tracing, phase, error_type): + instrumentor, exporter, _ = tracing + error = error_type("stream failed") + exits = [] + + async def stream_text(): + if phase == "consume": + raise error + yield "hello" + + class Manager: + async def __aenter__(self): + if phase == "enter": + raise error + return SimpleNamespace(stream_text=stream_text) + + async def __aexit__(self, exc_type, exc_value, traceback): + assert trace_api.get_current_span().is_recording() + exits.append(exc_value) + if phase == "exit": + raise error + return False + + def run_stream(self, prompt): + if phase == "factory": + raise error + return Manager() + + agent = wrapped_agent(instrumentor, run_stream) + + async def execute(): + previous_span = trace_api.get_current_span() + with pytest.raises(error_type) as caught: + async with agent.run_stream("hello") as result: + assert [part async for part in result.stream_text()] == ["hello"] + assert caught.value is error + assert trace_api.get_current_span() is previous_span + + asyncio.run(execute()) + assert exits == ([] if phase in ("factory", "enter") else + [error if phase == "consume" else None]) + spans = exporter.get_finished_spans() + assert len(spans) == 1 + assert spans[0].status.status_code is trace_api.StatusCode.ERROR + assert spans[0].attributes[PydanticAIAttributes.IS_ERROR] is True + assert spans[0].attributes[PydanticAIAttributes.ERROR_MESSAGE] == "stream failed" + assert spans[0].attributes[PydanticAIAttributes.DURATION_MS] >= 0 + assert len([event for event in spans[0].events if event.name == "exception"]) == 1 + + +def test_run_stream_test_model_native_async_with(tracing, monkeypatch): + pydantic_ai = pytest.importorskip("pydantic_ai") + from pydantic_ai import models + from pydantic_ai.models.test import TestModel + + # TestModel is local; prohibit real model requests as an additional guard. + monkeypatch.setattr(models, "ALLOW_MODEL_REQUESTS", False) + instrumentor, exporter, provider = tracing + instrumentor.instrument(tracer_provider=provider) + agent = pydantic_ai.Agent(TestModel(custom_output_text="hello")) + + async def execute(): + async with agent.run_stream("hello") as result: + parts = [part async for part in result.stream_text()] + return parts + + assert asyncio.run(execute())[-1] == "hello" + spans = [span for span in exporter.get_finished_spans() + if span.name == "pydantic_ai.agent.run_stream"] + assert len(spans) == 1 + assert spans[0].status.status_code is trace_api.StatusCode.OK + + +def test_synchronous_stream_factory_can_return_awaitable_manager(tracing): + instrumentor, exporter, _ = tracing + events = [] + result = SimpleNamespace(content="hello") + + @asynccontextmanager + async def manager(): + assert trace_api.get_current_span().is_recording() + events.append("enter") + try: + yield result + finally: + assert trace_api.get_current_span().is_recording() + events.append("exit") + + async def create_manager(): + events.append("await") + return manager() + + def run_stream(self, prompt): + assert prompt == "hello" + events.append("factory") + return create_manager() + + agent = wrapped_agent(instrumentor, run_stream) + + async def execute(): + previous = trace_api.get_current_span() + async with agent.run_stream("hello") as stream: + assert stream.content == result.content + assert trace_api.get_current_span().is_recording() + assert trace_api.get_current_span() is previous + + asyncio.run(execute()) + assert events == ["factory", "await", "enter", "exit"] + span, = exporter.get_finished_spans() + assert span.status.status_code is trace_api.StatusCode.OK diff --git a/python/frameworks/pydantic-ai/tests/test_run_stream_content.py b/python/frameworks/pydantic-ai/tests/test_run_stream_content.py new file mode 100644 index 000000000..a8783ba6e --- /dev/null +++ b/python/frameworks/pydantic-ai/tests/test_run_stream_content.py @@ -0,0 +1,287 @@ +"""A streamed run exports the same canonical content as a non-streamed run. + +Spans use the real OTel SDK and an in-memory exporter. The model-backed cases +use Pydantic AI's local TestModel; the others fake only the streamed result. +""" + +import asyncio +import collections.abc +from contextlib import asynccontextmanager +from types import SimpleNamespace + +import pytest +from fi_instrumentation import TraceConfig +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +from opentelemetry.trace import StatusCode + +from traceai_pydantic_ai import PydanticAIInstrumentor + + +PROMPT = "PRIVATE_STREAM_PROMPT" +REPLY = "PRIVATE_STREAM_REPLY across several words" +STREAM_SPAN = "pydantic_ai.agent.run_stream" + + +@pytest.fixture +def exporter(): + exporter = InMemorySpanExporter() + provider = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + exporter.provider = provider + yield exporter + provider.shutdown() + + +@pytest.fixture +def instrumentor(monkeypatch, exporter): + for name in ( + "HIDE_INPUTS", "HIDE_OUTPUTS", "HIDE_INPUT_MESSAGES", "HIDE_OUTPUT_MESSAGES", + "HIDE_INPUT_TEXT", "HIDE_OUTPUT_TEXT", "HIDE_INPUT_IMAGES", "HIDE_ERROR_DETAILS", + "HIDE_LLM_INVOCATION_PARAMETERS", "HIDE_EMBEDDING_VECTORS", "PII_REDACTION", + "BASE64_IMAGE_MAX_LENGTH", + ): + monkeypatch.delenv("FI_" + name, raising=False) + monkeypatch.setattr(PydanticAIInstrumentor, "_instance", None) + monkeypatch.setattr(PydanticAIInstrumentor, "_is_instrumented", False) + monkeypatch.setattr(PydanticAIInstrumentor, "_is_instrumented_by_opentelemetry", False) + monkeypatch.setattr(PydanticAIInstrumentor, "instrumentation_dependencies", lambda self: ()) + instrumentor = PydanticAIInstrumentor() + instrumentor._tracer = exporter.provider.get_tracer(__name__) + yield instrumentor + if instrumentor.is_instrumented: + instrumentor.uninstrument() + + +@pytest.fixture +def make_agent(monkeypatch): + pydantic_ai = pytest.importorskip("pydantic_ai") + from pydantic_ai import models + from pydantic_ai.models.test import TestModel + + # TestModel is local; refuse real model requests as an additional guard. + monkeypatch.setattr(models, "ALLOW_MODEL_REQUESTS", False) + + def make(with_tool=False): + model = TestModel(custom_output_text=REPLY, call_tools="all" if with_tool else []) + agent = pydantic_ai.Agent(model) + if with_tool: + @agent.tool_plain + def lookup(q: str) -> str: + return "looked up " + q + return agent + + return make + + +async def text_deltas(result): + return "".join([part async for part in result.stream_text(delta=True)]) + + +async def full_text(result): + return [part async for part in result.stream_text()][-1] + + +async def outputs(result): + return [output async for output in result.stream_output()][-1] + + +async def get_output(result): + return await result.get_output() + + +CONSUMERS = { + "text-deltas": text_deltas, + "full-text": full_text, + "outputs": outputs, + "get-output": get_output, +} + + +def stream(agent, consume, prompt=PROMPT): + async def execute(): + async with agent.run_stream(prompt) as result: + return await consume(result) + + return asyncio.run(execute()) + + +def stream_span(exporter): + (span,) = [span for span in exporter.get_finished_spans() if span.name == STREAM_SPAN] + return span + + +def exported(exporter): + return "\n".join(span.to_json() for span in exporter.get_finished_spans()) + + +def assert_canonical_content(span, prompt, output): + attributes = span.attributes + assert attributes["gen_ai.span.kind"] == "AGENT" + assert attributes["input.value"] == prompt + assert attributes["input.mime_type"] == "text/plain" + assert attributes["pydantic_ai.run.prompt"] == prompt + assert attributes["output.value"] == output + assert attributes["output.mime_type"] == "text/plain" + assert attributes["pydantic_ai.run.result"] == output + # The framework's own fields keep their existing values. + assert attributes["pydantic_ai.span_kind"] == "stream" + assert attributes["pydantic_ai.run.method"] == "run_stream" + + +@pytest.mark.parametrize("consumer", sorted(CONSUMERS)) +def test_streamed_run_exports_prompt_reply_and_kind(instrumentor, exporter, make_agent, consumer): + instrumentor.instrument(tracer_provider=exporter.provider) + assert stream(make_agent(), CONSUMERS[consumer]) == REPLY + + span = stream_span(exporter) + assert span.parent is None + assert span.status.status_code is StatusCode.OK + assert_canonical_content(span, PROMPT, REPLY) + + +def test_streamed_run_with_tool_keeps_execution_spans_under_the_run(instrumentor, exporter, make_agent): + instrumentor.instrument(tracer_provider=exporter.provider) + assert stream(make_agent(with_tool=True), text_deltas) == REPLY + + span = stream_span(exporter) + children = [child for child in exporter.get_finished_spans() if child is not span] + assert {child.name for child in children} >= { + "pydantic_ai.model.request", "pydantic_ai.tool.call.lookup", + } + assert all(child.parent.span_id == span.context.span_id for child in children) + assert_canonical_content(span, PROMPT, REPLY) + + +@pytest.mark.parametrize( + "flags, hidden", + [ + ({"hide_inputs": True}, {PROMPT}), + ({"hide_outputs": True}, {REPLY}), + ({"hide_inputs": True, "hide_outputs": True}, {PROMPT, REPLY}), + ], + ids=["inputs", "outputs", "both"], +) +def test_streamed_run_content_obeys_privacy(instrumentor, exporter, make_agent, flags, hidden): + instrumentor.instrument(tracer_provider=exporter.provider, config=TraceConfig(**flags)) + # Hiding content never changes what the application receives. + assert stream(make_agent(with_tool=True), text_deltas) == REPLY + + text = exported(exporter) + for marker in (PROMPT, REPLY): + assert (marker in text) is (marker not in hidden) + attributes = stream_span(exporter).attributes + assert attributes["gen_ai.span.kind"] == "AGENT" + if PROMPT not in hidden: + assert attributes["input.value"] == PROMPT + if REPLY not in hidden: + assert attributes["output.value"] == REPLY + + +def chunked_agent(instrumentor, chunks, events): + """An agent whose stream yields text chunks, like StreamedRunResult.""" + + @asynccontextmanager + async def run_stream(self, user_prompt): + async def stream_text(*, delta=False): + text = "" + try: + for chunk in chunks: + text += chunk + yield chunk if delta else text + finally: + events.append("closed") + + async def stream_output(): + async for text in stream_text(): + yield text + + async def get_output(): + return await full_text(result) + + result = SimpleNamespace( + stream_text=stream_text, stream_output=stream_output, get_output=get_output, + ) + yield result + + class Agent: + model = None + + Agent.run_stream = instrumentor._wrap_run_stream(run_stream) + return Agent() + + +@pytest.mark.parametrize("consumer", sorted(CONSUMERS)) +def test_completed_stream_records_the_final_output(instrumentor, exporter, consumer): + events = [] + agent = chunked_agent(instrumentor, ["first ", "second ", "third"], events) + + assert stream(agent, CONSUMERS[consumer], prompt="question") == "first second third" + assert events == ["closed"] + assert_canonical_content(stream_span(exporter), "question", "first second third") + + +@pytest.mark.parametrize("delta", [False, True], ids=["full-text", "text-deltas"]) +def test_incomplete_stream_records_input_without_inventing_output(instrumentor, exporter, delta): + events = [] + agent = chunked_agent(instrumentor, ["first ", "second ", "third"], events) + + async def first_chunk(result): + async for part in result.stream_text(delta=delta): + return part + + assert stream(agent, first_chunk, prompt="question") == "first " + attributes = stream_span(exporter).attributes + assert attributes["gen_ai.span.kind"] == "AGENT" + assert attributes["input.value"] == "question" + assert attributes["input.mime_type"] == "text/plain" + for key in ("output.value", "output.mime_type", "pydantic_ai.run.result"): + assert key not in attributes + + +def test_closing_an_observed_stream_closes_the_native_stream(instrumentor, exporter): + events = [] + agent = chunked_agent(instrumentor, ["first ", "second "], events) + + async def execute(): + async with agent.run_stream("question") as result: + parts = result.stream_text(delta=True) + # Callers (and Pydantic AI's sync bridge) close async generators explicitly. + assert isinstance(parts, collections.abc.AsyncGenerator) + assert await parts.__anext__() == "first " + await parts.aclose() + assert events == ["closed"] + + asyncio.run(execute()) + assert "output.value" not in stream_span(exporter).attributes + + +def test_awaitable_stream_factory_exports_canonical_content(instrumentor, exporter): + class Result: + async def get_output(self): + return "answer" + + class Manager: + async def __aenter__(self): + return Result() + + async def __aexit__(self, *exc_info): + return False + + class Agent: + model = None + + async def run_stream(self, user_prompt): + return Manager() + + Agent.run_stream = instrumentor._wrap_run_stream(Agent.run_stream) + + async def execute(): + async with await Agent().run_stream(user_prompt="question") as result: + return await result.get_output() + + assert asyncio.run(execute()) == "answer" + span = stream_span(exporter) + assert span.status.status_code is StatusCode.OK + assert_canonical_content(span, "question", "answer") diff --git a/python/frameworks/pydantic-ai/tests/test_usage_accounting.py b/python/frameworks/pydantic-ai/tests/test_usage_accounting.py index da643707e..304b4c014 100644 --- a/python/frameworks/pydantic-ai/tests/test_usage_accounting.py +++ b/python/frameworks/pydantic-ai/tests/test_usage_accounting.py @@ -30,6 +30,21 @@ def traced(monkeypatch): provider.shutdown() +def run_spans(exporter): + # Per-request usage belongs to child spans as well. Continue to require + # exactly one aggregate span (and the exact delta) per logical agent run. + return [span for span in exporter.get_finished_spans() + if span.name == "pydantic_ai.agent.run"] + + +def assert_execution_parent(exporter, nested, outer, kind): + spans = {span.context.span_id: span for span in exporter.get_finished_spans()} + parent = spans[nested.parent.span_id] + assert parent.attributes["pydantic_ai.span_kind"] == kind + assert parent.parent.span_id == outer.context.span_id + assert nested.context.trace_id == outer.context.trace_id + + def counters(usage): input_name = "input_tokens" if hasattr(usage, "input_tokens") else "request_tokens" output_name = "output_tokens" if hasattr(usage, "output_tokens") else "response_tokens" @@ -69,7 +84,7 @@ async def run_twice(): tuple(after - prior for after, prior in zip(counters(shared), before)) ) - spans = traced.get_finished_spans() + spans = run_spans(traced) assert len(spans) == 2 keys = ("input_tokens", "output_tokens", "total_tokens") for span, delta in zip(spans, expected): @@ -102,12 +117,12 @@ async def invoke_child(messages, info): parent = Agent(FunctionModel(invoke_child)) assert parent.run_sync("parent prompt").output == "parent answer" - spans = traced.get_finished_spans() + spans = run_spans(traced) assert len(spans) == 2 nested, outer = spans assert nested.attributes["pydantic_ai.run.method"] == "run" assert outer.attributes["pydantic_ai.run.method"] == "run_sync" - assert nested.parent.span_id == outer.context.span_id + assert_execution_parent(traced, nested, outer, "model_request") assert nested.attributes["gen_ai.usage.input_tokens"] > 0 @@ -139,8 +154,8 @@ async def invoke(ctx: RunContext[None]) -> str: assert actual_usage is budgets[0] if explicit_budget: assert actual_usage is kwargs["usage"] - nested, outer = traced.get_finished_spans() # Exactly one span per logical operation. - assert nested.parent.span_id == outer.context.span_id + nested, outer = run_spans(traced) # Exactly one aggregate per logical operation. + assert_execution_parent(traced, nested, outer, "tool_call") keys = ("input_tokens", "output_tokens", "total_tokens") actual = [tuple(span.attributes[f"gen_ai.usage.{key}"] for key in keys) for span in (nested, outer)] @@ -170,7 +185,7 @@ def fail(messages, info): ).output == "ok" ) - failed, recovered = traced.get_finished_spans() + failed, recovered = run_spans(traced) assert failed.status.status_code.name == "ERROR" assert recovered.status.status_code.name == "OK" assert recovered.attributes["pydantic_ai.run.method"] == "run" diff --git a/python/frameworks/pydantic-ai/tests/test_usage_isolation.py b/python/frameworks/pydantic-ai/tests/test_usage_isolation.py index f4166cbb9..2fa7f075a 100644 --- a/python/frameworks/pydantic-ai/tests/test_usage_isolation.py +++ b/python/frameworks/pydantic-ai/tests/test_usage_isolation.py @@ -55,10 +55,17 @@ def counts(value): "total_tokens")) +def run_spans(exporter): + # Model requests now carry their own usage. These assertions guard the + # aggregate logical run counters, not a sum across parent and child spans. + return [span for span in exporter.get_finished_spans() + if span.name == "pydantic_ai.agent.run"] + + def span_counts(exporter): return [tuple(span.attributes.get(f"gen_ai.usage.{key}") for key in ( "input_tokens", "output_tokens", "total_tokens")) - for span in exporter.get_finished_spans()] + for span in run_spans(exporter)] @pytest.mark.parametrize("method", ["run_sync", "run"]) @@ -71,8 +78,8 @@ async def execute(): assert get_usage(result) is shared expected = counts(shared) assert span_counts(traces) == [expected] - assert len(traces.get_finished_spans()) == 1 - assert traces.get_finished_spans()[0].status.status_code.name == "OK" + assert len(run_spans(traces)) == 1 + assert run_spans(traces)[0].status.status_code.name == "OK" def test_reconstructing_singleton_does_not_lose_owned_hooks(traces): @@ -163,7 +170,7 @@ async def execute(): if hasattr(first, "conversation") else {"message_history": first.all_messages()}) second = agent.run_sync("second", **continuation) assert second.all_messages()[:len(first.all_messages())] == first.all_messages() - assert len(traces.get_finished_spans()) == 2 + assert len(run_spans(traces)) == 2 actual = span_counts(traces) if hasattr(first, "conversation"): assert second.conversation.conversation_id == first.conversation.conversation_id diff --git a/python/frameworks/pydantic-ai/traceai_pydantic_ai/__init__.py b/python/frameworks/pydantic-ai/traceai_pydantic_ai/__init__.py index 537bf6110..3c3dba335 100644 --- a/python/frameworks/pydantic-ai/traceai_pydantic_ai/__init__.py +++ b/python/frameworks/pydantic-ai/traceai_pydantic_ai/__init__.py @@ -32,7 +32,7 @@ wrap_tool_function, ) -__version__ = "0.1.0" # x-release-please-version +__version__ = "0.2.0" # x-release-please-version __all__ = [ # Main API diff --git a/python/frameworks/pydantic-ai/traceai_pydantic_ai/_agent_wrapper.py b/python/frameworks/pydantic-ai/traceai_pydantic_ai/_agent_wrapper.py index 0ca102588..21d064f30 100644 --- a/python/frameworks/pydantic-ai/traceai_pydantic_ai/_agent_wrapper.py +++ b/python/frameworks/pydantic-ai/traceai_pydantic_ai/_agent_wrapper.py @@ -210,6 +210,23 @@ def _record_usage(span, usage): span.set_attribute(f"gen_ai.usage.{key}", usage[key]) +def _record_output(span: Span, result: Any) -> None: + """Record the same bounded text for canonical and compatibility consumers.""" + output = getattr(result, "output", None) + if output is None: + output = getattr(result, "data", None) + _record_output_value(span, output) + + +def _record_output_value(span: Span, output: Any) -> None: + if output is not None: + serialized = safe_serialize(output) + span.set_attribute(PydanticAIAttributes.RUN_RESULT, serialized) + span.set_attribute(PydanticAIAttributes.OUTPUT_VALUE, serialized) + # Even structured values are text: truncation may make them invalid JSON. + span.set_attribute(PydanticAIAttributes.OUTPUT_MIME_TYPE, "text/plain") + + def _run_attributes( agent: Any, args: tuple, kwargs: dict, method_name: str ) -> Dict[str, Any]: @@ -230,9 +247,12 @@ def _run_attributes( # Build attributes attributes = { PydanticAIAttributes.SPAN_KIND: PydanticAISpanKind.AGENT_RUN.value, + PydanticAIAttributes.GEN_AI_SPAN_KIND: "AGENT", PydanticAIAttributes.RUN_ID: run_id, PydanticAIAttributes.RUN_METHOD: method_name, PydanticAIAttributes.RUN_PROMPT: prompt, + PydanticAIAttributes.INPUT_VALUE: prompt, + PydanticAIAttributes.INPUT_MIME_TYPE: "text/plain", } # Add agent info @@ -332,12 +352,7 @@ async def async_wrapper(self, *args, **kwargs): _record_usage(span, operation_usage.extract(result)) # Record output - output = getattr(result, "output", None) or getattr(result, "data", None) - if output is not None: - span.set_attribute( - PydanticAIAttributes.RUN_RESULT, - safe_serialize(output), - ) + _record_output(span, result) span.set_attribute(PydanticAIAttributes.IS_ERROR, False) span.set_status(Status(StatusCode.OK)) @@ -384,13 +399,7 @@ def sync_wrapper(self, *args, **kwargs): # Extract and record usage _record_usage(span, operation_usage.extract(result)) - output = getattr(result, "output", None) - if output is None: - output = getattr(result, "data", None) - if output is not None: - span.set_attribute( - PydanticAIAttributes.RUN_RESULT, safe_serialize(output) - ) + _record_output(span, result) span.set_attribute(PydanticAIAttributes.IS_ERROR, False) span.set_status(Status(StatusCode.OK)) diff --git a/python/frameworks/pydantic-ai/traceai_pydantic_ai/_attributes.py b/python/frameworks/pydantic-ai/traceai_pydantic_ai/_attributes.py index fae7162ea..e6a45c68f 100644 --- a/python/frameworks/pydantic-ai/traceai_pydantic_ai/_attributes.py +++ b/python/frameworks/pydantic-ai/traceai_pydantic_ai/_attributes.py @@ -28,6 +28,15 @@ class PydanticAIAttributes: # Span Kind # ========================================================================= SPAN_KIND = "pydantic_ai.span_kind" + GEN_AI_SPAN_KIND = "gen_ai.span.kind" + + # ========================================================================= + # Canonical Content Attributes + # ========================================================================= + INPUT_VALUE = "input.value" + INPUT_MIME_TYPE = "input.mime_type" + OUTPUT_VALUE = "output.value" + OUTPUT_MIME_TYPE = "output.mime_type" # ========================================================================= # Agent Attributes diff --git a/python/frameworks/pydantic-ai/traceai_pydantic_ai/_execution_wrappers.py b/python/frameworks/pydantic-ai/traceai_pydantic_ai/_execution_wrappers.py new file mode 100644 index 000000000..e39bdded9 --- /dev/null +++ b/python/frameworks/pydantic-ai/traceai_pydantic_ai/_execution_wrappers.py @@ -0,0 +1,214 @@ +"""Wrappers for SDK execution boundaries, independent of tool registration.""" + +import functools +import inspect +import time +from contextlib import asynccontextmanager, contextmanager +from contextvars import ContextVar + +from opentelemetry.trace import SpanKind, Status, StatusCode, use_span + +from ._agent_wrapper import ( + MODEL_REQUEST_SPAN_NAME, + TOOL_CALL_SPAN_NAME, + _ACTIVE_USAGE, + _parent_run_context, + _record_usage, + _usage_values, + safe_serialize, +) +from ._attributes import PydanticAIAttributes as A, PydanticAISpanKind + + +_ACTIVE_MODEL_REQUEST = ContextVar("pydantic_ai_model_request", default=None) + + +def _serialize(value): + # Observing arguments/results must not turn a successful call into a failure. + try: + return safe_serialize(value) + except Exception: + return "" + + +def _record_model_usage(span, response): + if span is None: + return + # Historical Model.request returns (ModelResponse, Usage); modern responses + # carry RequestUsage, and StreamedResponse exposes usage(). Never replace it. + try: + usage = response[1] if isinstance(response, tuple) else getattr(response, "usage", None) + if callable(usage): + usage = usage() + _record_usage(span, _usage_values(usage)) + except Exception: + # Usage is observation only; retain the SDK's original response. + pass + + +@contextmanager +def _model_request_span(model, tracer, current=True): + operation = _ACTIVE_USAGE.get() + active = _ACTIVE_MODEL_REQUEST.get() + # A subclass may delegate to super().request(). Do not double count that, + # but retain genuinely nested agent runs that happen to share a model. + if active is not None and active[0] is model and active[1] is operation: + yield None + return + + attributes = {A.SPAN_KIND: PydanticAISpanKind.MODEL_REQUEST.value} + for field, attribute in (("model_name", A.MODEL_NAME), ("system", A.MODEL_PROVIDER)): + try: + value = getattr(model, field, None) + except Exception: + continue + if isinstance(value, str): + attributes[attribute] = value + options = dict(kind=SpanKind.CLIENT, attributes=attributes, context=_parent_run_context()) + scope = ( + tracer.start_as_current_span( + MODEL_REQUEST_SPAN_NAME, **options, + record_exception=False, set_status_on_exception=False, + ) if current else tracer.start_span(MODEL_REQUEST_SPAN_NAME, **options) + ) + with scope as span: + token = _ACTIVE_MODEL_REQUEST.set((model, operation)) if current else None + start = time.perf_counter() + try: + yield span + except Exception as exc: + span.set_attribute(A.IS_ERROR, True) + span.set_attribute(A.ERROR_TYPE, type(exc).__name__) + span.set_attribute(A.ERROR_MESSAGE, str(exc)) + span.set_status(Status(StatusCode.ERROR, str(exc))) + span.record_exception(exc) + raise + else: + span.set_attribute(A.IS_ERROR, False) + span.set_status(Status(StatusCode.OK)) + finally: + if token is not None: + _ACTIVE_MODEL_REQUEST.reset(token) + span.set_attribute(A.DURATION_MS, (time.perf_counter() - start) * 1000) + + +class _ModelStreamContext: + """Scope context tokens to each awaited operation, never across a yield. + + Some SDK stream consumers enter and exit their manager in different tasks. + The span may outlive either task, but a ContextVar token may not cross them. + """ + + def __init__(self, manager, model, span): + self.manager = manager + self.model = model + self.span = span + + @contextmanager + def _current(self): + if self.span is None: + yield + return + token = _ACTIVE_MODEL_REQUEST.set((self.model, _ACTIVE_USAGE.get())) + try: + with use_span(self.span, end_on_exit=False, record_exception=False, + set_status_on_exception=False): + yield + finally: + _ACTIVE_MODEL_REQUEST.reset(token) + + async def __aenter__(self): + with self._current(): + return await self.manager.__aenter__() + + async def __aexit__(self, exc_type, exc_value, traceback): + with self._current(): + return await self.manager.__aexit__(exc_type, exc_value, traceback) + + +def wrap_model_request(original, tracer, streaming=False): + if streaming: + @functools.wraps(original) + @asynccontextmanager + async def stream_wrapper(self, *args, **kwargs): + with _model_request_span(self, tracer, current=False) as span: + manager = original(self, *args, **kwargs) + async with _ModelStreamContext(manager, self, span) as response: + yield response + _record_model_usage(span, response) + return stream_wrapper + + @functools.wraps(original) + async def wrapper(self, *args, **kwargs): + with _model_request_span(self, tracer) as span: + response = await original(self, *args, **kwargs) + _record_model_usage(span, response) + return response + return wrapper + + +def wrap_tool_call(original, tracer, legacy=False): + signature = inspect.signature(original) + + @functools.wraps(original) + async def wrapper(self, *args, **kwargs): + try: + bound = signature.bind(self, *args, **kwargs).arguments + except TypeError: + # Let the SDK, not the observer, reject invalid call arguments. + return await original(self, *args, **kwargs) + if legacy: + name = self.name + description = getattr(self, "description", None) + arguments = getattr(bound.get("message"), "args", None) + retry = getattr(self, "current_retry", None) + else: + name = bound.get("name", "unknown") + tool = bound.get("tool") + definition = getattr(tool, "tool_def", tool) + description = getattr(definition, "description", None) + arguments = bound.get("tool_args") + retry = getattr(bound.get("ctx"), "retry", None) + attributes = { + A.SPAN_KIND: PydanticAISpanKind.TOOL_CALL.value, + A.GEN_AI_TOOL_NAME: name, + A.TOOL_ARGS: _serialize(arguments), + } + if isinstance(description, str): + attributes[A.GEN_AI_TOOL_DESCRIPTION] = description[:500] + if isinstance(retry, int): + attributes[A.TOOL_RETRY_COUNT] = retry + with tracer.start_as_current_span( + f"{TOOL_CALL_SPAN_NAME}.{name}", kind=SpanKind.INTERNAL, + attributes=attributes, context=_parent_run_context(), + record_exception=False, set_status_on_exception=False, + ) as span: + start = time.perf_counter() + try: + result = await original(self, *args, **kwargs) + except Exception as exc: + # Modern toolsets let ModelRetry reach the SDK's caller. Observe + # the attempt, but do not swallow, replace or re-run it. + span.set_attribute(A.TOOL_IS_ERROR, True) + span.set_attribute(A.TOOL_ERROR_MESSAGE, str(exc)) + span.set_status(Status(StatusCode.ERROR, str(exc))) + span.record_exception(exc) + raise + else: + # Historical Tool.run catches ModelRetry itself and returns a + # RetryPromptPart. It is a failed attempt, not a failed run. + retry_result = legacy and getattr(result, "part_kind", None) == "retry-prompt" + content = getattr(result, "content", result) if legacy else result + span.set_attribute(A.TOOL_RESULT, _serialize(content)) + span.set_attribute(A.TOOL_IS_ERROR, retry_result) + if retry_result: + message = _serialize(content) + span.set_attribute(A.TOOL_ERROR_MESSAGE, message) + span.set_status(Status(StatusCode.ERROR, message)) + else: + span.set_status(Status(StatusCode.OK)) + return result + finally: + span.set_attribute(A.TOOL_DURATION_MS, (time.perf_counter() - start) * 1000) + + return wrapper diff --git a/python/frameworks/pydantic-ai/traceai_pydantic_ai/_instrumentor.py b/python/frameworks/pydantic-ai/traceai_pydantic_ai/_instrumentor.py index d3ecd3dff..dae75b3d3 100644 --- a/python/frameworks/pydantic-ai/traceai_pydantic_ai/_instrumentor.py +++ b/python/frameworks/pydantic-ai/traceai_pydantic_ai/_instrumentor.py @@ -3,15 +3,20 @@ Main instrumentor class that provides OpenTelemetry tracing for Pydantic AI. """ +import functools +import inspect import logging import sys from typing import Any, Collection, Optional +from fi_instrumentation import FITracer, TraceConfig from opentelemetry import trace as trace_api from opentelemetry.instrumentation.instrumentor import BaseInstrumentor -from ._agent_wrapper import wrap_agent_run, wrap_tool_function, wrap_legacy_usage_incr +from ._agent_wrapper import _record_output_value, wrap_agent_run, wrap_legacy_usage_incr from ._attributes import PydanticAIAttributes, PydanticAISpanKind +from ._privacy import _PydanticTraceConfig +from ._execution_wrappers import wrap_model_request, wrap_tool_call logger = logging.getLogger(__name__) logger.addHandler(logging.NullHandler()) @@ -61,6 +66,7 @@ def __init__(self): super().__init__() self._tracer: Optional[trace_api.Tracer] = None self._original_methods: dict = {} + self._execution_patches = [] self._legacy_usage_original = None self._use_builtin: bool = False @@ -74,7 +80,9 @@ def _instrument(self, **kwargs: Any) -> None: Args: **kwargs: Configuration options - tracer_provider: OpenTelemetry tracer provider - - use_builtin: If True, use Pydantic AI's built-in instrumentation + - config: TraceConfig; omitted fields resolve from the environment + - use_builtin: Deprecated compatibility flag; FI-aware wrappers + are used because native instrumentation bypasses TraceConfig """ if self._is_instrumented: logger.warning("Pydantic AI is already instrumented") @@ -85,14 +93,19 @@ def _instrument(self, **kwargs: Any) -> None: if not tracer_provider: tracer_provider = trace_api.get_tracer_provider() - # Create tracer - self._tracer = trace_api.get_tracer( - __name__, - "0.1.0", - tracer_provider, + # Resolve environment-backed defaults at instrumentation time, not import + # time. Apply policy before SDK samplers/processors can observe content. + config = kwargs.get("config") + if config is None: + config = TraceConfig() + if not isinstance(config, TraceConfig): + raise TypeError("config must be a TraceConfig") + self._tracer = FITracer( + trace_api.get_tracer(__name__, "0.1.0", tracer_provider), + config=_PydanticTraceConfig(config), ) - # Check if we should use built-in instrumentation + # Retain the option for compatibility, but never bypass FI's policy. self._use_builtin = kwargs.get("use_builtin", False) # Try to import and patch Pydantic AI @@ -117,9 +130,12 @@ def _patch_pydantic_ai(self) -> None: raise ImportError("Could not import pydantic_ai") if self._use_builtin: - # Use Pydantic AI's built-in instrumentation - self._enable_builtin_instrumentation() - return + # Native instrumentation creates its own raw tracers. It cannot + # enforce this instrumentor's content, error or suppression policy. + logger.warning( + "use_builtin is deprecated; using FI-aware Pydantic AI wrappers " + "to enforce TraceConfig" + ) # Before request-local usage lived on ModelResponse, the SDK updated # Usage in place. Observe its increments in the logical run's context. @@ -147,6 +163,8 @@ def _patch_pydantic_ai(self) -> None: if hasattr(Agent, "run_stream"): Agent.run_stream = self._wrap_run_stream(Agent.run_stream) + self._patch_execution_methods() + # Also patch in any modules that have already imported Agent for module in list(sys.modules.values()): try: @@ -157,19 +175,75 @@ def _patch_pydantic_ai(self) -> None: logger.debug("Patched Pydantic AI Agent") - def _enable_builtin_instrumentation(self) -> None: - """Enable Pydantic AI's built-in OpenTelemetry instrumentation.""" + def _patch_execution_methods(self) -> None: + """Observe execution, including tools registered before instrument().""" try: - from pydantic_ai import Agent + from pydantic_ai import models + except ImportError: + # Agent-only implementations can still use the run wrappers. + return - # Use Agent.instrument_all() if available - if hasattr(Agent, "instrument_all"): - Agent.instrument_all() - logger.debug("Enabled Pydantic AI built-in instrumentation via Agent.instrument_all()") + # Model.request is abstract: patching only the base implementation does + # not intercept concrete providers. Cover loaded classes and future + # provider imports without importing every optional provider ourselves. + Model = models.Model + if not isinstance(Model, type): + return + seen = set() + + def patch_model(cls): + if cls in seen: + return + seen.add(cls) + for name in ("request", "request_stream"): + original = cls.__dict__.get(name) + if callable(original) and not getattr(original, "__isabstractmethod__", False): + self._install_execution_patch( + cls, name, wrap_model_request( + original, self._tracer, streaming=name == "request_stream")) + for subclass in cls.__subclasses__(): + patch_model(subclass) + + original_init = Model.__dict__.get("__init_subclass__") + + def init_subclass(cls, **kwargs): + if original_init is None: + super(Model, cls).__init_subclass__(**kwargs) else: - logger.warning("Agent.instrument_all() not available") - except Exception as e: - logger.warning(f"Failed to enable built-in instrumentation: {e}") + original_init.__get__(None, cls)(**kwargs) + patch_model(cls) + + self._install_execution_patch(Model, "__init_subclass__", classmethod(init_subclass)) + patch_model(Model) + + # Toolsets dispatch both sync and async functions, preserving validation, + # context injection and executor offloading. Older SDKs use Tool.run, + # which may return RetryPromptPart rather than raising ModelRetry. + try: + from pydantic_ai.toolsets.function import FunctionToolset + except ImportError: + from pydantic_ai.tools import Tool + if isinstance(Tool, type) and hasattr(Tool, "run"): + self._install_execution_patch( + Tool, "run", wrap_tool_call(Tool.run, self._tracer, legacy=True)) + else: + if isinstance(FunctionToolset, type): + self._install_execution_patch( + FunctionToolset, "call_tool", wrap_tool_call(FunctionToolset.call_tool, self._tracer)) + + def _install_execution_patch(self, cls, name, replacement): + original = cls.__dict__.get(name) + setattr(cls, name, replacement) + self._execution_patches.append((cls, name, original, replacement)) + + def _restore_execution_methods(self): + for cls, name, original, replacement in reversed(self._execution_patches): + if cls.__dict__.get(name) is replacement: + if original is None: + delattr(cls, name) + else: + setattr(cls, name, original) + self._execution_patches.clear() def _wrap_run_stream(self, original_method): """Wrap run_stream method for async context manager. @@ -181,8 +255,10 @@ def _wrap_run_stream(self, original_method): Wrapped method """ import functools + import inspect import time import uuid + from contextlib import asynccontextmanager from ._agent_wrapper import ( extract_model_info, @@ -192,6 +268,7 @@ def _wrap_run_stream(self, original_method): tracer = self._tracer @functools.wraps(original_method) + @asynccontextmanager async def wrapper(self, *args, **kwargs): from ._attributes import PydanticAIAttributes, PydanticAISpanKind @@ -209,9 +286,12 @@ async def wrapper(self, *args, **kwargs): attributes = { PydanticAIAttributes.SPAN_KIND: PydanticAISpanKind.STREAM.value, + PydanticAIAttributes.GEN_AI_SPAN_KIND: "AGENT", PydanticAIAttributes.RUN_ID: run_id, PydanticAIAttributes.RUN_METHOD: "run_stream", PydanticAIAttributes.RUN_PROMPT: prompt, + PydanticAIAttributes.INPUT_VALUE: prompt, + PydanticAIAttributes.INPUT_MIME_TYPE: "text/plain", } if model_info.get("model_name"): @@ -221,31 +301,83 @@ async def wrapper(self, *args, **kwargs): start_time = time.time() - # Create span for the entire stream + # Keep the span current through entry, consumption, and cleanup. + traced_stream = None + with tracer.start_as_current_span( + "pydantic_ai.agent.run_stream", + kind=trace_api.SpanKind.CLIENT, + attributes=attributes, + record_exception=False, + set_status_on_exception=False, + ) as span: + try: + # Retain the original manager and its exit/suppression + # rules, including factories that return awaitable managers. + manager = original_method(self, *args, **kwargs) + if inspect.isawaitable(manager): + manager = await manager + async with manager as stream: + traced_stream = _TracedStream(stream) + yield traced_stream + except BaseException as e: + # Cancellation must also close the span without changing + # the exception delivered by the native context manager. + span.set_attribute(PydanticAIAttributes.IS_ERROR, True) + span.set_attribute(PydanticAIAttributes.ERROR_MESSAGE, str(e)) + span.set_status(trace_api.Status(trace_api.StatusCode.ERROR, str(e))) + span.record_exception(e) + raise + else: + span.set_attribute(PydanticAIAttributes.IS_ERROR, False) + span.set_status(trace_api.Status(trace_api.StatusCode.OK)) + finally: + duration_ms = (time.time() - start_time) * 1000 + span.set_attribute(PydanticAIAttributes.DURATION_MS, duration_ms) + span.set_attribute( + PydanticAIAttributes.STREAM_CHUNK_COUNT, + traced_stream._chunk_count if traced_stream is not None else 0, + ) + if traced_stream is not None: + traced_stream._record_final_output(span) + + if not inspect.iscoroutinefunction(original_method): + return wrapper + + # Retain the awaitable-manager protocol used by older/custom agents. + # Native factories above never become coroutines. + @functools.wraps(original_method) + async def coroutine_wrapper(self, *args, **kwargs): + prompt = safe_serialize(args[0]) if args else safe_serialize( + kwargs.get("user_prompt", "") + ) + attributes = { + PydanticAIAttributes.SPAN_KIND: PydanticAISpanKind.STREAM.value, + PydanticAIAttributes.GEN_AI_SPAN_KIND: "AGENT", + PydanticAIAttributes.RUN_ID: str(uuid.uuid4()), + PydanticAIAttributes.RUN_METHOD: "run_stream", + PydanticAIAttributes.RUN_PROMPT: prompt, + PydanticAIAttributes.INPUT_VALUE: prompt, + PydanticAIAttributes.INPUT_MIME_TYPE: "text/plain", + } + model_info = extract_model_info(self) + if model_info.get("model_name"): + attributes[PydanticAIAttributes.MODEL_NAME] = model_info["model_name"] + if model_info.get("model_provider"): + attributes[PydanticAIAttributes.MODEL_PROVIDER] = model_info["model_provider"] + start_time = time.time() span = tracer.start_span( "pydantic_ai.agent.run_stream", kind=trace_api.SpanKind.CLIENT, attributes=attributes, ) - try: - # Call original and return wrapped context manager stream = await original_method(self, *args, **kwargs) - - # Wrap the stream to track completion - return _TracedStream(stream, span, start_time, tracer) - - except Exception as e: - duration_ms = (time.time() - start_time) * 1000 - span.set_attribute(PydanticAIAttributes.DURATION_MS, duration_ms) - span.set_attribute(PydanticAIAttributes.IS_ERROR, True) - span.set_attribute(PydanticAIAttributes.ERROR_MESSAGE, str(e)) - span.set_status(trace_api.Status(trace_api.StatusCode.ERROR, str(e))) - span.record_exception(e) - span.end() + return _AwaitableTracedStream(stream, span, start_time) + except BaseException as error: + _AwaitableTracedStream(None, span, start_time)._finish(error) raise - return wrapper + return coroutine_wrapper def _uninstrument(self, **kwargs: Any) -> None: """Remove Pydantic AI instrumentation.""" @@ -269,6 +401,7 @@ def _uninstrument(self, **kwargs: Any) -> None: except ImportError: pass + self._restore_execution_methods() self._is_instrumented = False self._tracer = None @@ -280,73 +413,151 @@ def is_instrumented(self) -> bool: return self._is_instrumented +# Result methods that hand the run's final output to the caller. They are only +# observed: consuming or re-validating a stream here would repeat validators and +# message-history updates, so a stream the caller never finishes has no output. +_OUTPUT_STREAMS = ("stream_output", "stream_text", "stream") +_OUTPUT_GETTERS = ("get_output", "get_data") +_NO_OUTPUT = object() + + class _TracedStream: - """Wrapper for streaming responses that tracks completion.""" + """Proxy the entered result; its original manager owns stream cleanup.""" - def __init__(self, stream, span, start_time, tracer): + def __init__(self, stream): self._stream = stream + self._chunk_count = 0 + self._final_output = _NO_OUTPUT + + def __aiter__(self): + return self + + async def __anext__(self): + chunk = await self._stream.__anext__() + self._chunk_count += 1 + return chunk + + # Forward other attributes to underlying stream + def __getattr__(self, name): + value = getattr(self._stream, name) + if callable(value) and name in _OUTPUT_STREAMS: + return self._observe_stream(value) + if callable(value) and name in _OUTPUT_GETTERS: + return self._observe_getter(value) + return value + + def _observe_stream(self, method): + @functools.wraps(method) + def observed(*args, **kwargs): + items = method(*args, **kwargs) + if not hasattr(items, "__aiter__"): + return items + return self._observe_items(items, bool(kwargs.get("delta"))) + + return observed + + async def _observe_items(self, items, delta): + # A real async generator, so callers that close generators explicitly + # still close the native one. + parts = [] + try: + async for item in items: + if delta: + parts.append(item) + else: + parts = [item] + yield item + finally: + aclose = getattr(items, "aclose", None) + if aclose is not None: + await aclose() + if delta: + self._final_output = "".join(str(part) for part in parts) + elif parts: + self._final_output = parts[-1] + + def _observe_getter(self, method): + @functools.wraps(method) + def observed(*args, **kwargs): + output = method(*args, **kwargs) + if not inspect.isawaitable(output): + self._final_output = output + return output + return self._await_output(output) + + return observed + + async def _await_output(self, awaitable): + output = await awaitable + self._final_output = output + return output + + def _record_final_output(self, span): + if self._final_output is not _NO_OUTPUT: + _record_output_value(span, self._final_output) + + +class _AwaitableTracedStream(_TracedStream): + """Manage cleanup for coroutine factories without changing native streams.""" + + def __init__(self, stream, span, start_time): + super().__init__(stream) + self._context_manager = stream self._span = span self._start_time = start_time - self._tracer = tracer - self._chunk_count = 0 async def __aenter__(self): - if hasattr(self._stream, "__aenter__"): - self._stream = await self._stream.__aenter__() - return self + try: + if hasattr(self._context_manager, "__aenter__"): + self._stream = await self._context_manager.__aenter__() + return self + except BaseException as error: + self._finish(error) + raise async def __aexit__(self, exc_type, exc_val, exc_tb): - import time + error = exc_val + try: + # Exit the manager, not a potentially different entered result. + if hasattr(self._context_manager, "__aexit__"): + return await self._context_manager.__aexit__(exc_type, exc_val, exc_tb) + except BaseException as exit_error: + error = exit_error + raise + finally: + self._finish(error) - from ._attributes import PydanticAIAttributes + def _finish(self, error=None): + import time - duration_ms = (time.time() - self._start_time) * 1000 - self._span.set_attribute(PydanticAIAttributes.DURATION_MS, duration_ms) self._span.set_attribute( - PydanticAIAttributes.STREAM_CHUNK_COUNT, - self._chunk_count, + PydanticAIAttributes.DURATION_MS, (time.time() - self._start_time) * 1000 ) - - if exc_type: + self._span.set_attribute(PydanticAIAttributes.STREAM_CHUNK_COUNT, self._chunk_count) + if error is not None: self._span.set_attribute(PydanticAIAttributes.IS_ERROR, True) - self._span.set_attribute(PydanticAIAttributes.ERROR_MESSAGE, str(exc_val)) - self._span.set_status( - trace_api.Status(trace_api.StatusCode.ERROR, str(exc_val)) - ) - if exc_val: - self._span.record_exception(exc_val) + self._span.set_attribute(PydanticAIAttributes.ERROR_MESSAGE, str(error)) + self._span.set_status(trace_api.Status(trace_api.StatusCode.ERROR, str(error))) + self._span.record_exception(error) else: self._span.set_attribute(PydanticAIAttributes.IS_ERROR, False) self._span.set_status(trace_api.Status(trace_api.StatusCode.OK)) - + self._record_final_output(self._span) self._span.end() - if hasattr(self._stream, "__aexit__"): - await self._stream.__aexit__(exc_type, exc_val, exc_tb) - - def __aiter__(self): - return self - - async def __anext__(self): - chunk = await self._stream.__anext__() - self._chunk_count += 1 - return chunk - - # Forward other attributes to underlying stream - def __getattr__(self, name): - return getattr(self._stream, name) - # Convenience function for simple initialization def instrument_pydantic_ai( tracer_provider: Optional[Any] = None, use_builtin: bool = False, + config: Optional[TraceConfig] = None, ) -> PydanticAIInstrumentor: """Convenience function to instrument Pydantic AI. Args: tracer_provider: Optional OpenTelemetry tracer provider - use_builtin: If True, use Pydantic AI's built-in instrumentation + use_builtin: Deprecated compatibility flag; uses FI-aware wrappers + config: Content privacy policy; defaults resolve from the environment Returns: The instrumentor instance @@ -365,6 +576,8 @@ def instrument_pydantic_ai( kwargs["tracer_provider"] = tracer_provider if use_builtin: kwargs["use_builtin"] = use_builtin + if config is not None: + kwargs["config"] = config instrumentor.instrument(**kwargs) return instrumentor diff --git a/python/frameworks/pydantic-ai/traceai_pydantic_ai/_privacy.py b/python/frameworks/pydantic-ai/traceai_pydantic_ai/_privacy.py new file mode 100644 index 000000000..a60e8af78 --- /dev/null +++ b/python/frameworks/pydantic-ai/traceai_pydantic_ai/_privacy.py @@ -0,0 +1,50 @@ +"""Apply FI's content policy without changing Pydantic's exported field names.""" + +import wrapt +from fi_instrumentation.fi_types import SpanAttributes + +from ._attributes import PydanticAIAttributes as A + + +# These opaque values must use the same rules as input.value/output.value, +# including the narrower message/text flags. Renaming just the display fields +# or adding masked duplicates would leave the original aliases unprotected. +_CONTENT_ALIASES = { + A.RUN_PROMPT: SpanAttributes.INPUT_VALUE, + A.AGENT_INSTRUCTIONS: SpanAttributes.INPUT_VALUE, + A.GEN_AI_TOOL_DESCRIPTION: SpanAttributes.INPUT_VALUE, + A.TOOL_ARGS: SpanAttributes.INPUT_VALUE, + A.RUN_RESULT: SpanAttributes.OUTPUT_VALUE, + A.TOOL_RESULT: SpanAttributes.OUTPUT_VALUE, + A.ERROR_MESSAGE: "error.message", + A.TOOL_ERROR_MESSAGE: "error.message", + A.RETRY_REASON: "error.message", + A.VALIDATION_ERROR: "error.message", +} +_MIXED_CONTENT = (A.MESSAGE_CONTENT, A.MESSAGE_TOOL_CALLS) + + +def _matches(key, prefix): + return key == prefix or key.startswith(prefix + ".") + + +class _PydanticTraceConfig(wrapt.ObjectProxy): + """Delegate to the supplied config, adapting only content-key recognition. + + FITracer and FiSpan share this policy at creation, late updates, events and + links. All other config behavior, especially derived error details, PII and + image redaction, remains owned by the original TraceConfig. + """ + + def mask(self, key, value): + config = self.__wrapped__ + if any(_matches(key, prefix) for prefix in _MIXED_CONTENT): + # These aliases have no direction: either side's privacy rule wins. + value = config.mask(SpanAttributes.INPUT_VALUE, value) + if value is None: + return None + return config.mask(SpanAttributes.OUTPUT_VALUE, value) + for alias, canonical in _CONTENT_ALIASES.items(): + if _matches(key, alias): + return config.mask(canonical, value) + return config.mask(key, value) diff --git a/python/frameworks/smolagents/CHANGELOG.md b/python/frameworks/smolagents/CHANGELOG.md index 503633b2b..9efe984d2 100644 --- a/python/frameworks/smolagents/CHANGELOG.md +++ b/python/frameworks/smolagents/CHANGELOG.md @@ -1,5 +1,13 @@ +## [0.1.7] - pending release + +### Changed +- traceai-smolagents now requires fi-instrumentation-otel 1.0.0 or later. Action: Upgrade fi-instrumentation-otel if your environment pins an earlier version. + +### Fixed +- Streamed agent runs keep the run span open until the stream ends, so each step is nested under it, the run span records the final answer and token usage, and a stream closed before it finishes is not marked successful. + ## [0.1.6] - 2026-10-29 ### Fixed diff --git a/python/frameworks/smolagents/poetry.lock b/python/frameworks/smolagents/poetry.lock index 47a41be6d..36ffa0280 100644 --- a/python/frameworks/smolagents/poetry.lock +++ b/python/frameworks/smolagents/poetry.lock @@ -1897,4 +1897,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.1" python-versions = ">3.10, <3.14" -content-hash = "5204ac75b9648cb1426fe8e9ef5273f9046aa66e1f50f4e29661821752ef304b" +content-hash = "4ca9b3e36f5ee0a22901845cce4c39dcc3a7dafc12f6c14de389546661ee51d0" diff --git a/python/frameworks/smolagents/pyproject.toml b/python/frameworks/smolagents/pyproject.toml index 20eec5a78..0d294dded 100644 --- a/python/frameworks/smolagents/pyproject.toml +++ b/python/frameworks/smolagents/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "traceAI-smolagents" -version = "0.1.6" +version = "0.1.7" description = "OpenTelemetry instrumentation for smolagents" authors = ["Future AGI "] readme = "README.md" @@ -29,7 +29,7 @@ classifiers = [ [tool.poetry.dependencies] python = ">3.10, <3.14" smolagents = ">=1.12.0" -fi-instrumentation-otel = ">=0.1.11" +fi-instrumentation-otel = ">=1.0.0" [tool.poetry.urls] diff --git a/python/frameworks/smolagents/tests/test_run_stream.py b/python/frameworks/smolagents/tests/test_run_stream.py new file mode 100644 index 000000000..c270aee9a --- /dev/null +++ b/python/frameworks/smolagents/tests/test_run_stream.py @@ -0,0 +1,313 @@ +"""T5-PY-006: the agent run owns execution, not just iterator creation.""" + +import gc +from types import SimpleNamespace + +import pytest +from opentelemetry import context, trace +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +from traceai_smolagents._wrappers import ( + GEN_AI_USAGE_INPUT_TOKENS, + GEN_AI_USAGE_OUTPUT_TOKENS, + GEN_AI_USAGE_TOTAL_TOKENS, + OUTPUT_VALUE, + _RunWrapper, +) + + +class FakeAgent: + task = "offline task" + max_steps = 2 + tools = {} + managed_agents = {} + + def __init__(self): + self.monitor = SimpleNamespace( + total_input_token_count=0, total_output_token_count=0 + ) + + +@pytest.fixture +def tracing(): + exporter = InMemorySpanExporter() + provider = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + token = context.attach(context.get_current()) + try: + yield exporter, provider.get_tracer(__name__) + finally: + context.detach(token) + provider.shutdown() + + +def run_spans(exporter): + return [span for span in exporter.get_finished_spans() if span.name == "FakeAgent.run"] + + +@pytest.mark.parametrize("answer_shape", ["legacy", "final_step", "return"]) +def test_run_stream_finishes_with_actual_output_usage_and_parent(tracing, monkeypatch, answer_shape): + exporter, tracer = tracing + agent = FakeAgent() + calls = [] + + # Model the 1.20 final-answer event without executing any model or SDK run. + class FinalAnswerStep: + def __init__(self, output): + self.output = output + + import smolagents.memory + monkeypatch.setattr(smolagents.memory, "FinalAnswerStep", FinalAnswerStep, raising=False) + answer = FinalAnswerStep("done") if answer_shape == "final_step" else "done" + progress = object() + + def produce(): + with tracer.start_as_current_span("model"): + agent.monitor.total_input_token_count = 7 + agent.monitor.total_output_token_count = 3 + yield progress + if answer_shape == "return": + return "done" + yield answer + + def run(task, stream=False): + calls.append((task, stream)) + return produce() + + with tracer.start_as_current_span("caller") as caller: + iterator = _RunWrapper(tracer)(run, agent, ("offline task", True), {}) + assert calls == [("offline task", True)] # SDK setup remains eager. + assert not run_spans(exporter) + # Moving consumption must not move the run's original parent. + with tracer.start_as_current_span("consumer") as consumer: + assert next(iterator) is progress + assert trace.get_current_span() is consumer + assert not run_spans(exporter) + if answer_shape != "return": + assert iterator.send(None) is answer + assert not run_spans(exporter) # A final yield is not exhaustion. + with pytest.raises(StopIteration) as stopped: + next(iterator) + assert stopped.value.value == ("done" if answer_shape == "return" else None) + assert trace.get_current_span() is consumer + iterator.close() + assert trace.get_current_span() is caller + + (run_span,) = run_spans(exporter) + assert run_span.status.status_code.name == "OK" + assert run_span.attributes[OUTPUT_VALUE] == "done" + assert run_span.attributes[GEN_AI_USAGE_INPUT_TOKENS] == 7 + assert run_span.attributes[GEN_AI_USAGE_OUTPUT_TOKENS] == 3 + assert run_span.attributes[GEN_AI_USAGE_TOTAL_TOKENS] == 10 + assert run_span.parent.span_id == caller.get_span_context().span_id + (model,) = [span for span in exporter.get_finished_spans() if span.name == "model"] + assert model.parent.span_id == run_span.context.span_id + assert model.end_time <= run_span.end_time + + +@pytest.mark.parametrize("termination", ["close", "gc"]) +def test_unconsumed_run_stream_never_reports_success(tracing, termination): + exporter, tracer = tracing + calls = [] + + def produce(): + pytest.fail("an unconsumed run must not execute") + yield + + def run(task, stream=False): + calls.append(task) + return produce() + + iterator = _RunWrapper(tracer)(run, FakeAgent(), ("offline task",), {"stream": True}) + assert calls == ["offline task"] + assert not run_spans(exporter) + if termination == "close": + iterator.close() + iterator.close() + else: + del iterator + gc.collect() + spans = run_spans(exporter) + assert len(spans) <= 1 + assert all(span.status.status_code.name != "OK" for span in spans) + assert all(OUTPUT_VALUE not in span.attributes for span in spans) + assert all(GEN_AI_USAGE_TOTAL_TOKENS not in span.attributes for span in spans) + + +@pytest.mark.parametrize("termination", ["next", "send", "throw", "error", "close", "close_error"]) +def test_run_stream_protocol_keeps_producer_context_and_consumer_isolation(tracing, termination): + exporter, tracer = tracing + agent = FakeAgent() + marker = context.create_key("run-producer-marker") + failure = ValueError("offline producer failure") + cleanup_failure = ValueError("offline cleanup failure") + received = [] + cleanup = [] + + def produce(): + with tracer.start_as_current_span("producer-child") as child: + token = context.attach(context.set_value(marker, "producer-value")) + try: + try: + value = yield child.get_span_context().span_id + received.append(value) + except ValueError as exc: + assert exc is failure + received.append("recovered") + yield "recovered" + if termination == "error": + raise failure + agent.monitor.total_input_token_count = 7 + agent.monitor.total_output_token_count = 3 + return "done" + finally: + cleanup.append((trace.get_current_span().get_span_context().span_id, + context.get_value(marker))) + with tracer.start_as_current_span("cleanup"): + pass + context.detach(token) + if termination == "close_error": + raise cleanup_failure + + def run(task, stream=False): + return produce() + + iterator = None + try: + with tracer.start_as_current_span("caller") as caller: + iterator = _RunWrapper(tracer)(run, agent, ("offline task",), {"stream": True}) + child_id = next(iterator) + assert trace.get_current_span() is caller + assert context.get_value(marker) is None + assert not run_spans(exporter) + with tracer.start_as_current_span("consumer") as consumer: + if termination == "throw": + assert iterator.throw(failure) == "recovered" + assert trace.get_current_span() is consumer + assert context.get_value(marker) is None + assert not run_spans(exporter) + if termination in {"next", "send", "throw"}: + with pytest.raises(StopIteration) as stopped: + if termination == "send": + iterator.send("payload") + else: + next(iterator) + assert stopped.value.value == "done" + elif termination in {"error", "close_error"}: + with pytest.raises(ValueError) as caught: + if termination == "error": + next(iterator) + else: + iterator.close() + assert caught.value is (failure if termination == "error" else cleanup_failure) + else: + assert iterator.close() is None + assert trace.get_current_span() is consumer + assert context.get_value(marker) is None + iterator.close() + assert trace.get_current_span() is caller + assert cleanup == [(child_id, "producer-value")] + if termination == "send": + assert received == ["payload"] + elif termination == "throw": + assert received == ["recovered"] + spans = {span.name: span for span in exporter.get_finished_spans()} + (run_span,) = run_spans(exporter) + assert run_span.parent.span_id == caller.get_span_context().span_id + assert spans["producer-child"].parent.span_id == run_span.context.span_id + assert spans["cleanup"].parent.span_id == child_id + assert spans["cleanup"].end_time <= spans["producer-child"].end_time <= run_span.end_time + assert spans["consumer"].parent.span_id == caller.get_span_context().span_id + if termination in {"error", "close_error"}: + assert run_span.status.status_code.name == "ERROR" + assert len(run_span.events) == 1 + elif termination == "close": + assert run_span.status.status_code.name != "OK" + else: + assert run_span.status.status_code.name == "OK" + assert run_span.attributes[OUTPUT_VALUE] == "done" + assert run_span.attributes[GEN_AI_USAGE_TOTAL_TOKENS] == 10 + if termination in {"error", "close", "close_error"}: + assert OUTPUT_VALUE not in run_span.attributes + finally: + if iterator is not None: + iterator.close() + + +@pytest.mark.parametrize("termination", ["close", "gc"]) +def test_run_ordinary_iterator_cleanup_uses_producer_context(tracing, termination): + exporter, tracer = tracing + marker = context.create_key("run-cleanup-marker") + cleanup = [] + + def produce(): + with tracer.start_as_current_span("child") as child: + token = context.attach(context.set_value(marker, "producer-value")) + try: + yield child.get_span_context().span_id + finally: + cleanup.append((trace.get_current_span().get_span_context().span_id, + context.get_value(marker))) + context.detach(token) + + def run(task, stream=False): + return map(lambda value: value, produce()) + + with tracer.start_as_current_span("caller") as caller: + iterator = _RunWrapper(tracer)(run, FakeAgent(), ("offline task",), {"stream": True}) + child_id = next(iterator) + assert trace.get_current_span() is caller + if termination == "close": + iterator.close() + else: + del iterator + gc.collect() + assert trace.get_current_span() is caller + assert context.get_value(marker) is None + assert cleanup == [(child_id, "producer-value")] + (run_span,) = run_spans(exporter) + assert run_span.status.status_code.name != "OK" + (child,) = [span for span in exporter.get_finished_spans() if span.name == "child"] + assert child.parent.span_id == run_span.context.span_id + assert child.end_time <= run_span.end_time + + +@pytest.mark.parametrize("stream", [False, True]) +def test_run_setup_error_still_raises_eagerly(tracing, stream): + exporter, tracer = tracing + failure = ValueError("offline setup failure") + + def run(task, stream=False): + raise failure + + with pytest.raises(ValueError) as caught: + _RunWrapper(tracer)(run, FakeAgent(), ("offline task",), {"stream": stream}) + assert caught.value is failure + (span,) = run_spans(exporter) + assert span.status.status_code.name == "ERROR" + assert len(span.events) == 1 + + +def test_nonstreaming_and_suppressed_runs_are_unchanged(tracing): + exporter, tracer = tracing + agent = FakeAgent() + output = object() + + def run(task, stream=False): + agent.monitor.total_input_token_count = 7 + agent.monitor.total_output_token_count = 3 + return output + + assert _RunWrapper(tracer)(run, agent, ("offline task",), {}) is output + (span,) = run_spans(exporter) + assert span.attributes[OUTPUT_VALUE] == str(output) + assert span.attributes[GEN_AI_USAGE_TOTAL_TOKENS] == 10 + assert span.status.status_code.name == "OK" + exporter.clear() + token = context.attach(context.set_value(context._SUPPRESS_INSTRUMENTATION_KEY, True)) + try: + assert _RunWrapper(tracer)(run, agent, ("offline task",), {"stream": True}) is output + assert not exporter.get_finished_spans() + finally: + context.detach(token) diff --git a/python/frameworks/smolagents/tests/test_step_stream.py b/python/frameworks/smolagents/tests/test_step_stream.py index 805673b61..cea9e39cd 100644 --- a/python/frameworks/smolagents/tests/test_step_stream.py +++ b/python/frameworks/smolagents/tests/test_step_stream.py @@ -164,9 +164,15 @@ def respond(request): } (consumer,) = [span for span in spans if span.name == "unrelated-consumer-work"] assert consumer.parent.span_id == caller.get_span_context().span_id - assert all( - span.parent.span_id == caller.get_span_context().span_id for span in steps - ) + runs = [span for span in spans if span.name == "ToolCallingAgent.run"] + assert len(runs) == 2 + assert all(span.status.status_code.name == "OK" for span in runs) + assert all(span.parent.span_id == caller.get_span_context().span_id for span in runs) + assert {span.parent.span_id for span in steps} == {span.context.span_id for span in runs} + for run in runs: + children = [step for step in steps if step.parent.span_id == run.context.span_id] + assert {step.name for step in children} == {"Step 1", "Step 2"} + assert all(run.start_time <= step.start_time <= step.end_time <= run.end_time for step in children) @pytest.mark.parametrize("instrumented", [False, True]) diff --git a/python/frameworks/smolagents/traceai_smolagents/_wrappers.py b/python/frameworks/smolagents/traceai_smolagents/_wrappers.py index 0c3cd0cb1..c48469f8a 100644 --- a/python/frameworks/smolagents/traceai_smolagents/_wrappers.py +++ b/python/frameworks/smolagents/traceai_smolagents/_wrappers.py @@ -135,6 +135,98 @@ def run_in_context(*worker_args, **worker_kwargs): return decorate +class _RunIterator: + """Keep a run alive while isolating its producer context at every boundary.""" + + def __init__(self, iterator, span, producer_context, finish): + self._iterator = iterator + self._span = span + self._producer_context = producer_context + self._finish = finish + self._last = None + self._ended = False + + def __iter__(self): + return self + + def __next__(self): + return self._advance("__next__") + + def send(self, value): + # Ordinary iterators support next/send(None), but not send(value). + return self._advance("__next__") if value is None else self._advance("send", value) + + def throw(self, *args): + return self._advance("throw", *args) + + def close(self): + return self._advance("close") + + def _end(self): + if not self._ended: + self._ended = True + self._span.end() + + def _advance(self, method, *args): + token = context_api.attach(self._producer_context) + try: + try: + if method == "close": + try: + close = self._iterator.close + except AttributeError: + # Releasing map/other ordinary iterators may finalize an + # owned generator. Do so before restoring the consumer. + self._iterator = iter(()) + result = None + else: + result = close() + else: + result = getattr(self._iterator, method)(*args) + except StopIteration as stopped: + try: + if not self._ended and method != "close": + # Native runs yield their final answer; decorators may + # instead return it as the generator's terminal value. + self._finish(stopped.value if stopped.value is not None else self._last) + finally: + self._end() + raise + except GeneratorExit: + self._end() + raise + except BaseException as exc: + if not self._ended: + self._span.record_exception(exc) + self._span.set_status(trace_api.Status(trace_api.StatusCode.ERROR, str(exc))) + self._end() + raise + else: + if method == "close": + # Closing or abandoning a stream is not a successful run. + self._end() + else: + self._last = result + return result + finally: + self._producer_context = context_api.get_current() + context_api.detach(token) + + def __del__(self): + try: + if not self._ended: + self.close() + except BaseException: + # Explicit close propagates errors; finalization cannot do so. + pass + finally: + token = context_api.attach(self._producer_context) + try: + self._iterator = None + finally: + context_api.detach(token) + + class _RunWrapper: def __init__(self, tracer: trace_api.Tracer) -> None: self._tracer = tracer @@ -151,7 +243,7 @@ def __call__( span_name = f"{instance.__class__.__name__}.run" agent = instance arguments = _bind_arguments(wrapped, *args, **kwargs) - with self._tracer.start_as_current_span( + span = self._tracer.start_span( span_name, attributes=dict( _flatten( @@ -168,8 +260,18 @@ def __call__( } ) ), - ) as span: - agent_output = wrapped(*args, **kwargs) + ) + + def finish(agent_output): + if arguments.get("stream"): + # 1.20 yields FinalAnswerStep; older SDKs yield the answer itself. + try: + from smolagents.memory import FinalAnswerStep + except ImportError: + pass + else: + if isinstance(agent_output, FinalAnswerStep): + agent_output = agent_output.output span.set_attribute(OUTPUT_VALUE, _get_raw_output(agent_output)) span.set_attribute( GEN_AI_USAGE_INPUT_TOKENS, agent.monitor.total_input_token_count @@ -184,7 +286,26 @@ def __call__( ) span.set_status(trace_api.StatusCode.OK) span.set_attribute(OUTPUT_VALUE, str(agent_output)) - return agent_output + + token = context_api.attach(trace_api.set_span_in_context(span)) + streaming = False + try: + # Keep SDK initialization and validation eager, including errors. + agent_output = wrapped(*args, **kwargs) + if arguments.get("stream") and isinstance(agent_output, Iterator): + iterator = _RunIterator(agent_output, span, context_api.get_current(), finish) + streaming = True + return iterator + finish(agent_output) + return agent_output + except BaseException as exc: + span.record_exception(exc) + span.set_status(trace_api.Status(trace_api.StatusCode.ERROR, str(exc))) + raise + finally: + context_api.detach(token) + if not streaming: + span.end() class _StepWrapper: diff --git a/python/frameworks/smolagents/traceai_smolagents/version.py b/python/frameworks/smolagents/traceai_smolagents/version.py index 80efe0760..1b09c6e18 100644 --- a/python/frameworks/smolagents/traceai_smolagents/version.py +++ b/python/frameworks/smolagents/traceai_smolagents/version.py @@ -1 +1 @@ -__version__ = "0.1.6" # x-release-please-version +__version__ = "0.1.7" # x-release-please-version diff --git a/python/frameworks/strands/tests/test_real_core_names.py b/python/frameworks/strands/tests/test_real_core_names.py new file mode 100644 index 000000000..4ffb413c5 --- /dev/null +++ b/python/frameworks/strands/tests/test_real_core_names.py @@ -0,0 +1,45 @@ +"""Check the core names this package uses against the real installed core. + +tests/conftest.py replaces fi_instrumentation with mock modules, so the other tests here +cannot notice a core name that the declared fi-instrumentation-otel floor lacks. This test +reads the package source in a clean interpreter that imports the real core. +""" + +import pathlib +import subprocess +import sys + +PACKAGE = pathlib.Path(__file__).resolve().parents[1] / "traceai_strands" + +CHECK = r""" +import ast, importlib, pathlib, sys, types +missing = [] +for path in sorted(pathlib.Path(sys.argv[1]).rglob("*.py")): + tree = ast.parse(path.read_text(encoding="utf-8")) + names = {} + for node in ast.walk(tree): + if isinstance(node, ast.ImportFrom) and node.module and node.module.split(".")[0] == "fi_instrumentation": + module = importlib.import_module(node.module) + for alias in node.names: + if hasattr(module, alias.name): + names[alias.asname or alias.name] = getattr(module, alias.name) + else: + missing.append(f"{path.name}: {node.module}.{alias.name}") + for node in ast.walk(tree): + if isinstance(node, ast.Attribute) and isinstance(node.value, ast.Name) and node.value.id in names: + owner = names[node.value.id] + if isinstance(owner, (type, types.ModuleType)) and not hasattr(owner, node.attr): + missing.append(f"{path.name}: {node.value.id}.{node.attr}") +print("\n".join(sorted(set(missing)))) +sys.exit(1 if missing else 0) +""" + + +def test_every_core_name_the_package_uses_exists_in_the_installed_core(): + result = subprocess.run( + [sys.executable, "-I", "-c", CHECK, str(PACKAGE)], + capture_output=True, + text=True, + check=False, + ) + assert result.returncode == 0, "missing in the installed core:\n" + result.stdout + result.stderr diff --git a/python/frameworks/together/CHANGELOG.md b/python/frameworks/together/CHANGELOG.md index fa441c5eb..c3525aa7e 100644 --- a/python/frameworks/together/CHANGELOG.md +++ b/python/frameworks/together/CHANGELOG.md @@ -1,5 +1,10 @@ # Changelog +## [0.1.2] - pending release + +### Changed +- traceai-together now requires fi-instrumentation-otel 1.0.0 or later. Action: Upgrade fi-instrumentation-otel if your environment pins an earlier version. + ## [0.1.1] - 2026-10-29 ### Fixed diff --git a/python/frameworks/together/pyproject.toml b/python/frameworks/together/pyproject.toml index b1cde5195..9aa01fcc0 100644 --- a/python/frameworks/together/pyproject.toml +++ b/python/frameworks/together/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "traceAI-together" -version = "0.1.1" +version = "0.1.2" description = "OpenTelemetry instrumentation for Together AI - chat, completions, and embeddings" authors = ["Future AGI "] readme = "README.md" @@ -27,7 +27,7 @@ classifiers = [ [tool.poetry.dependencies] python = ">=3.10,<3.14" -fi-instrumentation-otel = ">=0.1.11" +fi-instrumentation-otel = ">=1.0.0" together = ">=1.0.0" [tool.poetry.urls] diff --git a/python/frameworks/together/traceai_together/version.py b/python/frameworks/together/traceai_together/version.py index 34d76ee9e..0d33eef3f 100644 --- a/python/frameworks/together/traceai_together/version.py +++ b/python/frameworks/together/traceai_together/version.py @@ -1 +1 @@ -__version__ = "0.1.1" # x-release-please-version +__version__ = "0.1.2" # x-release-please-version diff --git a/python/frameworks/vertexai/CHANGELOG.md b/python/frameworks/vertexai/CHANGELOG.md index ece1fb057..53d46cf59 100644 --- a/python/frameworks/vertexai/CHANGELOG.md +++ b/python/frameworks/vertexai/CHANGELOG.md @@ -1,5 +1,13 @@ +## [0.1.12] - pending release + +### Changed +- traceai-vertexai now requires fi-instrumentation-otel 1.0.0 or later. Action: Upgrade fi-instrumentation-otel if your environment pins an earlier version. + +### Fixed +- Tracing no longer fails on blocked responses that have no candidates or parts, and tool-call-only and image-only responses record their response data instead of empty text. + ## [0.1.11] - 2026-10-31 ### Fixed - Bound legacy `google-cloud-aiplatform` support to `>=1.61.0,<=1.101.0`, retaining the generative AI modules and allowing the protobuf version required by the tracing dependencies. diff --git a/python/frameworks/vertexai/poetry.lock b/python/frameworks/vertexai/poetry.lock index 6c69a0208..3aa49dbcf 100644 --- a/python/frameworks/vertexai/poetry.lock +++ b/python/frameworks/vertexai/poetry.lock @@ -1728,4 +1728,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.1" python-versions = ">=3.10,<3.14" -content-hash = "95680b3d0756180bc2d3625410dfb0a699cedf0b14b9a12c1e190570d040e55d" +content-hash = "386c645b730906e4b68f8e1f7776ff1969e7f95844a053f8e72bb1fbb5d20241" diff --git a/python/frameworks/vertexai/pyproject.toml b/python/frameworks/vertexai/pyproject.toml index 296cb6420..83e272d12 100644 --- a/python/frameworks/vertexai/pyproject.toml +++ b/python/frameworks/vertexai/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "traceAI-vertexai" -version = "0.1.11" +version = "0.1.12" description = "OpenTelemetry instrumentation for Vertex AI" authors = ["Future AGI "] readme = "README.md" @@ -28,7 +28,7 @@ classifiers = [ [tool.poetry.dependencies] python = ">=3.10,<3.14" -fi-instrumentation-otel = ">=0.1.11" +fi-instrumentation-otel = ">=1.0.0" google-cloud-aiplatform = ">=1.61.0,<=1.101.0" [tool.poetry.urls] diff --git a/python/frameworks/vertexai/tests/test_response_finalization.py b/python/frameworks/vertexai/tests/test_response_finalization.py new file mode 100644 index 000000000..dc450ed28 --- /dev/null +++ b/python/frameworks/vertexai/tests/test_response_finalization.py @@ -0,0 +1,154 @@ +"""valid non-text responses must not fail instrumentation.""" + +import json +from unittest.mock import Mock + +import pytest +from google.cloud import aiplatform_v1 as v1 +from google.cloud import aiplatform_v1beta1 as v1beta1 +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +from opentelemetry.trace import StatusCode + +from traceai_vertexai import _wrapper + + +@pytest.fixture +def tracing(monkeypatch): + monkeypatch.setattr(_wrapper._instrumentation_status, "_IS_INSTRUMENTED", True) + provider = TracerProvider() + exporter = InMemorySpanExporter() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + try: + yield provider.get_tracer(__name__), exporter + finally: + provider.shutdown() + + +def call_fake_rpc(tracer, request, response): + # Only the protobuf schema is real: no SDK client, credentials or network. + rpc = Mock(return_value=response) + result = _wrapper._Wrapper(tracer)(rpc)(request=request) + rpc.assert_called_once_with(request=request) + assert result is response + + +@pytest.mark.parametrize("sdk", [v1, v1beta1], ids=["v1", "v1beta1"]) +@pytest.mark.parametrize( + "response_data", + [ + pytest.param( + {"candidates": [], "prompt_feedback": {"block_reason": "SAFETY"}}, + id="no-candidates", + ), + pytest.param( + {"candidates": [{"content": {"role": "model", "parts": []}, + "finish_reason": "SAFETY"}]}, + id="empty-parts", + ), + pytest.param( + {"candidates": [{"content": {"role": "model", "parts": [ + {"function_call": {"name": "lookup", "args": {"city": "example"}}} + ]}}]}, + id="tool-call", + ), + pytest.param( + {"candidates": [{"content": {"role": "model", "parts": [ + {"inline_data": {"mime_type": "image/png", "data": b"fake-image"}} + ]}}]}, + id="image", + ), + ], +) +def test_non_text_response_is_returned_and_span_ends(tracing, sdk, response_data): + tracer, exporter = tracing + response = sdk.GenerateContentResponse( + **response_data, usage_metadata={"prompt_token_count": 2, "total_token_count": 2} + ) + request = sdk.GenerateContentRequest(model="test-model") + + call_fake_rpc(tracer, request, response) + + (span,) = exporter.get_finished_spans() + assert span.status.status_code is StatusCode.OK + assert json.loads(span.attributes[_wrapper.OUTPUT_VALUE]) == type(response).to_dict(response) + assert span.attributes[_wrapper.OUTPUT_MIME_TYPE] == _wrapper.JSON + assert span.attributes[_wrapper.GEN_AI_USAGE_INPUT_TOKENS] == 2 + assert span.attributes[_wrapper.GEN_AI_USAGE_TOTAL_TOKENS] == 2 + assert not span.events + + +def test_text_response_keeps_existing_output_and_ends_span(tracing): + tracer, exporter = tracing + response = v1.GenerateContentResponse( + candidates=[{"content": {"role": "model", "parts": [{"text": "Hello."}]}}] + ) + + call_fake_rpc(tracer, v1.GenerateContentRequest(model="test-model"), response) + + (span,) = exporter.get_finished_spans() + assert span.status.status_code is StatusCode.OK + assert span.attributes[_wrapper.OUTPUT_VALUE] == _wrapper.safe_json_dumps( + _wrapper.safe_json_dumps("Hello.") + ) + + +@pytest.mark.parametrize("failure_stage", ["serialization", "structured-attributes"]) +def test_response_extraction_failure_does_not_replace_result( + tracing, monkeypatch, failure_stage +): + tracer, exporter = tracing + response = v1.GenerateContentResponse( + candidates=[{"content": {"role": "model", "parts": [{"text": "Hello."}]}}] + ) + failure = Mock(side_effect=RuntimeError("synthetic extraction failure")) + if failure_stage == "serialization": + monkeypatch.setattr(v1.GenerateContentResponse, "to_dict", failure) + else: + original = _wrapper._update_span + + def update_span(obj, span): + if obj is response: + return failure(obj, span) + return original(obj, span) + + monkeypatch.setattr(_wrapper, "_update_span", update_span) + + call_fake_rpc(tracer, v1.GenerateContentRequest(model="test-model"), response) + + failure.assert_called_once() + (span,) = exporter.get_finished_spans() + assert span.status.status_code is StatusCode.OK + assert not span.events + + +def test_sdk_exception_is_not_replaced(tracing): + tracer, exporter = tracing + error = RuntimeError("synthetic SDK failure") + rpc = Mock(side_effect=error) + request = v1.GenerateContentRequest(model="test-model") + + with pytest.raises(RuntimeError) as caught: + _wrapper._Wrapper(tracer)(rpc)(request=request) + + assert caught.value is error + rpc.assert_called_once_with(request=request) + (span,) = exporter.get_finished_spans() + assert span.status.status_code is StatusCode.ERROR + assert len(span.events) == 1 + assert span.events[0].name == "exception" + + +def test_finish_records_exception_results_and_ends_span(tracing): + tracer, exporter = tracing + span = tracer.start_span("generate_content") + error = RuntimeError("synthetic callback failure") + + assert _wrapper._finish(span, error) is None + + assert not span.is_recording() + (finished,) = exporter.get_finished_spans() + assert finished.status.status_code is StatusCode.ERROR + assert len(finished.events) == 1 + assert finished.events[0].attributes["exception.message"] == str(error) diff --git a/python/frameworks/vertexai/traceai_vertexai/_wrapper.py b/python/frameworks/vertexai/traceai_vertexai/_wrapper.py index e0250970d..c4dfcc062 100644 --- a/python/frameworks/vertexai/traceai_vertexai/_wrapper.py +++ b/python/frameworks/vertexai/traceai_vertexai/_wrapper.py @@ -319,52 +319,55 @@ def _use_span(span: Span) -> Callable[[], ContextManager[Span]]: def _finish(span: Span, result: Any) -> None: - if isinstance(result, BaseException): - span.record_exception(result) - span.set_status(Status(StatusCode.ERROR, f"{type(result).__name__}: {result}")) - span.end() - return - - if isinstance(result, proto.Message): - raw_output = result.__class__.to_dict(result) - span.set_attribute(OUTPUT_VALUE, safe_json_dumps(raw_output)) - - if isinstance(result, v1.CountTokensResponse): - output = result.total_tokens - elif isinstance(result, v1.GenerateContentResponse): - output = result.candidates[0].content.parts[0].text - output = safe_json_dumps(output) - else: - output = result.__class__.to_dict(result) - - predictions = output.get("predictions", []) - for idx, prediction in enumerate(predictions): - if ( - isinstance(prediction, dict) - and "embeddings" in prediction - and isinstance(prediction["embeddings"], dict) - ): - output = prediction["embeddings"] - vector = output.get("values") - span.set_attribute(GEN_AI_SPAN_KIND, EMBEDDING) - span.set_attribute(EMBEDDING_EMBEDDINGS, safe_json_dumps(output)) - span.set_attribute( - f"{EMBEDDING_EMBEDDINGS}.{idx}.{EMBEDDING_VECTOR}", - safe_json_dumps(vector), - ) - - span.set_attribute(OUTPUT_VALUE, safe_json_dumps(output)) - span.set_attribute(OUTPUT_MIME_TYPE, JSON) - - elif result is not None: - span.set_attribute(OUTPUT_VALUE, safe_json_dumps(result)) - span.set_attribute(OUTPUT_VALUE, safe_json_dumps(result)) try: + if isinstance(result, BaseException): + span.record_exception(result) + span.set_status(Status(StatusCode.ERROR, f"{type(result).__name__}: {result}")) + return + + # The RPC succeeded even if response attribute extraction fails. + span.set_status(Status(StatusCode.OK)) + if isinstance(result, proto.Message): + raw_output = result.__class__.to_dict(result) + span.set_attribute(OUTPUT_VALUE, safe_json_dumps(raw_output)) + output = raw_output + + if isinstance(result, v1.CountTokensResponse): + output = result.total_tokens + elif isinstance(result, v1.GenerateContentResponse): + # Safety blocks may omit candidates or parts. Keep the raw + # response for these cases and for non-text (e.g. tool) parts. + if result.candidates: + parts = result.candidates[0].content.parts + if parts and parts[0].text: + output = safe_json_dumps(parts[0].text) + else: + predictions = output.get("predictions", []) + for idx, prediction in enumerate(predictions): + if ( + isinstance(prediction, dict) + and "embeddings" in prediction + and isinstance(prediction["embeddings"], dict) + ): + output = prediction["embeddings"] + vector = output.get("values") + span.set_attribute(GEN_AI_SPAN_KIND, EMBEDDING) + span.set_attribute(EMBEDDING_EMBEDDINGS, safe_json_dumps(output)) + span.set_attribute( + f"{EMBEDDING_EMBEDDINGS}.{idx}.{EMBEDDING_VECTOR}", + safe_json_dumps(vector), + ) + + span.set_attribute(OUTPUT_VALUE, safe_json_dumps(output)) + span.set_attribute(OUTPUT_MIME_TYPE, JSON) + + elif result is not None: + span.set_attribute(OUTPUT_VALUE, safe_json_dumps(result)) _update_span(result, span) except BaseException as exc: logger.exception(str(exc)) - span.set_status(Status(StatusCode.OK)) - span.end() + finally: + span.end() def _get_prediction_service_request(args: Iterable[Any]) -> Optional[proto.Message]: diff --git a/python/frameworks/vertexai/traceai_vertexai/version.py b/python/frameworks/vertexai/traceai_vertexai/version.py index 8c39d1009..d22c42508 100644 --- a/python/frameworks/vertexai/traceai_vertexai/version.py +++ b/python/frameworks/vertexai/traceai_vertexai/version.py @@ -1 +1 @@ -__version__ = "0.1.11" # x-release-please-version +__version__ = "0.1.12" # x-release-please-version diff --git a/python/frameworks/xai/CHANGELOG.md b/python/frameworks/xai/CHANGELOG.md index fa441c5eb..694889c70 100644 --- a/python/frameworks/xai/CHANGELOG.md +++ b/python/frameworks/xai/CHANGELOG.md @@ -1,5 +1,10 @@ # Changelog +## [0.1.2] - pending release + +### Changed +- traceai-xai now requires fi-instrumentation-otel 1.0.0 or later. Action: Upgrade fi-instrumentation-otel if your environment pins an earlier version. + ## [0.1.1] - 2026-10-29 ### Fixed diff --git a/python/frameworks/xai/pyproject.toml b/python/frameworks/xai/pyproject.toml index 1f0ca1d30..f445f4c7b 100644 --- a/python/frameworks/xai/pyproject.toml +++ b/python/frameworks/xai/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "traceAI-xai" -version = "0.1.1" +version = "0.1.2" description = "OpenTelemetry instrumentation for xAI (Grok)" authors = ["Future AGI "] readme = "README.md" @@ -27,7 +27,7 @@ classifiers = [ [tool.poetry.dependencies] python = ">=3.10,<3.14" -fi-instrumentation-otel = ">=0.1.11" +fi-instrumentation-otel = ">=1.0.0" openai = ">=1.0.0" [tool.poetry.group.dev.dependencies] diff --git a/python/frameworks/xai/traceai_xai/version.py b/python/frameworks/xai/traceai_xai/version.py index 34d76ee9e..0d33eef3f 100644 --- a/python/frameworks/xai/traceai_xai/version.py +++ b/python/frameworks/xai/traceai_xai/version.py @@ -1 +1 @@ -__version__ = "0.1.1" # x-release-please-version +__version__ = "0.1.2" # x-release-please-version diff --git a/python/pyproject.toml b/python/pyproject.toml index 29b68e589..5b1088d75 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "fi-instrumentation-otel" -version = "1.3.1" +version = "1.3.2" description = "OpenTelemetry tracing for AI applications: tracer setup, span helpers, exporters, and semantic conventions for Future AGI." authors = ["Future AGI "] readme = "README.md" diff --git a/python/tests/test_context_helper_decorators.py b/python/tests/test_context_helper_decorators.py new file mode 100644 index 000000000..abb33649f --- /dev/null +++ b/python/tests/test_context_helper_decorators.py @@ -0,0 +1,584 @@ +"""Context-helper decorators preserve attribution for the executed body.""" + +import asyncio +import inspect + +import pytest +from opentelemetry.context import Context, attach, detach, get_current +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter + +from fi_instrumentation import FITracer +from fi_instrumentation.fi_types import SpanAttributes +from fi_instrumentation.instrumentation.context_attributes import ( + get_attributes_from_context, + using_attributes, + using_metadata, + using_prompt_template, + using_session, + using_tags, + using_user, +) + + +SESSION = { + SpanAttributes.SESSION_ID: "decorated-session", + SpanAttributes.GEN_AI_CONVERSATION_ID: "decorated-session", +} +USER = {SpanAttributes.USER_ID: "decorated-user"} +METADATA = {SpanAttributes.METADATA: '{"source": "test"}'} +TAGS = {SpanAttributes.TAG_TAGS: ("first", "second")} +PROMPT = { + SpanAttributes.GEN_AI_PROMPT_TEMPLATE_NAME: "Hello {name}", + SpanAttributes.GEN_AI_PROMPT_TEMPLATE_LABEL: "test", + SpanAttributes.GEN_AI_PROMPT_TEMPLATE_VERSION: "v1", + SpanAttributes.GEN_AI_PROMPT_TEMPLATE_VARIABLES: '{"name": "reader"}', +} +HELPERS = [ + pytest.param(lambda: using_session("decorated-session"), SESSION, id="session"), + pytest.param(lambda: using_user("decorated-user"), USER, id="user"), + pytest.param(lambda: using_metadata({"source": "test"}), METADATA, id="metadata"), + pytest.param(lambda: using_tags(["first", "second"]), TAGS, id="tags"), + pytest.param( + lambda: using_prompt_template( + template="Hello {name}", label="test", version="v1", + variables={"name": "reader"}, + ), + PROMPT, + id="prompt-template", + ), + pytest.param( + lambda: using_attributes( + session_id="decorated-session", user_id="decorated-user", + metadata={"source": "test"}, tags=["first", "second"], + prompt_template="Hello {name}", prompt_template_label="test", + prompt_template_version="v1", prompt_template_variables={"name": "reader"}, + ), + {**SESSION, **USER, **METADATA, **TAGS, **PROMPT}, + id="attributes", + ), +] + + +@pytest.fixture +def tracing(): + token = attach(Context()) + exporter = InMemorySpanExporter() + provider = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + tracer = FITracer(provider.get_tracer("context-helper-tests")) + try: + yield tracer, exporter + finally: + provider.shutdown() + detach(token) + + +def span_attributes(exporter): + return {span.name: dict(span.attributes) for span in exporter.get_finished_spans()} + + +def assert_attributes(actual, expected): + for key, value in expected.items(): + assert actual.get(key) == value, key + + +@pytest.mark.parametrize("helper,expected", HELPERS) +def test_async_decorator_attributes_after_await(tracing, helper, expected): + tracer, exporter = tracing + + @helper() + async def decorated(value, *, suffix): + """Return a value after recording the attributed span.""" + await asyncio.sleep(0) + with tracer.start_as_current_span("body"): + return value + suffix + + async def run(): + prior = get_current() + pending = decorated("result", suffix="!") + assert get_current() is prior + assert await pending == "result!" + assert get_current() is prior + + asyncio.run(run()) + assert_attributes(span_attributes(exporter)["body"], expected) + assert inspect.iscoroutinefunction(decorated) + assert decorated.__name__ == "decorated" + assert decorated.__doc__ == "Return a value after recording the attributed span." + assert tuple(inspect.signature(decorated).parameters) == ("value", "suffix") + + +def test_async_decorator_restores_context_after_exception(tracing): + tracer, exporter = tracing + error = ValueError("expected failure") + + @using_session("inner-session") + async def decorated(): + await asyncio.sleep(0) + with tracer.start_as_current_span("exception"): + raise error + + async def run(): + with using_session("outer-session"): + prior = get_current() + with pytest.raises(ValueError) as caught: + await decorated() + assert caught.value is error + assert get_current() is prior + assert dict(get_attributes_from_context()) == {} + + asyncio.run(run()) + assert span_attributes(exporter)["exception"][SpanAttributes.SESSION_ID] == "inner-session" + + +def test_concurrent_decorators_keep_distinct_sessions(tracing): + tracer, exporter = tracing + + async def run(): + ready = [asyncio.Event(), asyncio.Event()] + + def task(index, session): + @using_session(session) + async def decorated(): + ready[index].set() + await ready[1 - index].wait() + await asyncio.sleep(0) + with tracer.start_as_current_span(session): + pass + return decorated() + + prior = get_current() + await asyncio.gather(task(0, "first-session"), task(1, "second-session")) + assert get_current() is prior + + asyncio.run(run()) + spans = span_attributes(exporter) + for session in ("first-session", "second-session"): + assert spans[session][SpanAttributes.SESSION_ID] == session + assert spans[session][SpanAttributes.GEN_AI_CONVERSATION_ID] == session + + +def test_shared_async_decorator_uses_call_local_tokens(tracing): + tracer, exporter = tracing + + async def run(): + ready = [asyncio.Event(), asyncio.Event()] + first_finished = asyncio.Event() + + @using_user("shared-user") + async def decorated(index): + ready[index].set() + await ready[1 - index].wait() + if index == 1: + await first_finished.wait() + with tracer.start_as_current_span(str(index)): + pass + + async def task(index): + with using_attributes(session_id=str(index), user_id="outer-user"): + prior = get_current() + try: + await decorated(index) + assert get_current() is prior + finally: + if index == 0: + first_finished.set() + + await asyncio.gather(task(0), task(1)) + assert dict(get_attributes_from_context()) == {} + + asyncio.run(run()) + spans = span_attributes(exporter) + for index in ("0", "1"): + assert spans[index][SpanAttributes.SESSION_ID] == index + assert spans[index][SpanAttributes.USER_ID] == "shared-user" + + +def test_recursive_async_decorator_restores_each_call(tracing): + tracer, exporter = tracing + + @using_session("recursive-session") + async def decorated(depth): + await asyncio.sleep(0) + if depth: + prior = get_current() + await decorated(depth - 1) + assert get_current() is prior + with tracer.start_as_current_span(str(depth)): + pass + + async def run(): + with using_session("outer-session"): + prior = get_current() + await decorated(2) + assert get_current() is prior + + asyncio.run(run()) + for attributes in span_attributes(exporter).values(): + assert attributes[SpanAttributes.SESSION_ID] == "recursive-session" + + +def test_recursive_sync_decorator_restores_each_call(tracing): + tracer, exporter = tracing + + @using_session("recursive-session") + def decorated(depth): + if depth: + prior = get_current() + decorated(depth - 1) + assert get_current() is prior + with tracer.start_as_current_span(str(depth)): + pass + + with using_session("outer-session"): + prior = get_current() + decorated(2) + assert get_current() is prior + for attributes in span_attributes(exporter).values(): + assert attributes[SpanAttributes.SESSION_ID] == "recursive-session" + + +def test_async_decorator_restores_context_after_cancellation(tracing): + tracer, exporter = tracing + + async def run(): + started = asyncio.Event() + pending = asyncio.Event() + + @using_session("cancelled-session") + async def decorated(): + await asyncio.sleep(0) + with tracer.start_as_current_span("cancelled"): + started.set() + await pending.wait() + + async def caller(): + with using_session("outer-session"): + prior = get_current() + try: + await decorated() + finally: + assert get_current() is prior + + task = asyncio.create_task(caller()) + await started.wait() + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + assert dict(get_attributes_from_context()) == {} + + asyncio.run(run()) + assert span_attributes(exporter)["cancelled"][SpanAttributes.SESSION_ID] == "cancelled-session" + + +@pytest.mark.parametrize("ending", ["exhaust", "close", "exception"]) +def test_async_generator_keeps_attribution_across_yields(tracing, ending): + tracer, exporter = tracing + cleaned_up = [] + + @using_session("generator-session") + async def decorated(): + """Yield attributed results.""" + try: + for index in range(2): + await asyncio.sleep(0) + with tracer.start_as_current_span(str(index)): + pass + yield index + if ending == "exception": + raise ValueError("generator failure") + finally: + cleaned_up.append(True) + + async def run(): + with using_session("outer-session"): + prior = get_current() + iterator = decorated() + assert get_current() is prior + try: + assert await iterator.__anext__() == 0 + await asyncio.sleep(0) + assert await iterator.__anext__() == 1 + if ending == "close": + await iterator.aclose() + elif ending == "exception": + with pytest.raises(ValueError, match="generator failure"): + await iterator.__anext__() + else: + with pytest.raises(StopAsyncIteration): + await iterator.__anext__() + finally: + await iterator.aclose() + assert get_current() is prior + assert cleaned_up == [True] + + asyncio.run(run()) + assert len(span_attributes(exporter)) == 2 + for attributes in span_attributes(exporter).values(): + assert attributes[SpanAttributes.SESSION_ID] == "generator-session" + assert inspect.isasyncgenfunction(decorated) + assert decorated.__name__ == "decorated" + assert decorated.__doc__ == "Yield attributed results." + + +@pytest.mark.parametrize("ending", ["exhaust", "close-out-of-order"]) +@pytest.mark.parametrize("stacked", [False, True], ids=["combined", "stacked"]) +def test_interleaved_async_generators_isolate_each_resume(tracing, ending, stacked): + tracer, exporter = tracing + + def stream(name): + async def body(): + with using_tags([name]): + try: + for index in range(3): + await asyncio.sleep(0) + with tracer.start_as_current_span(f"{name}-{index}"): + pass + yield index + finally: + await asyncio.sleep(0) + with tracer.start_as_current_span(f"{name}-cleanup"): + pass + + if stacked: + return using_session(name)(using_user(name + "-user")(body))() + return using_attributes(session_id=name, user_id=name + "-user")(body)() + + async def run(): + with using_attributes(session_id="outer-session", user_id="outer-user"): + prior = get_current() + contexts = [] + first, second = stream("first"), stream("second") + + async def advance(iterator, expected): + assert await iterator.__anext__() == expected + contexts.append(get_current()) + with tracer.start_as_current_span(f"caller-{len(contexts)}"): + pass + + try: + await advance(first, 0) + await advance(second, 0) + await advance(first, 1) + if ending == "close-out-of-order": + await first.aclose() + contexts.append(get_current()) + await advance(second, 1) + await second.aclose() + contexts.append(get_current()) + else: + await advance(second, 1) + await advance(first, 2) + await advance(second, 2) + for iterator in (first, second): + with pytest.raises(StopAsyncIteration): + await iterator.__anext__() + contexts.append(get_current()) + finally: + await first.aclose() + await second.aclose() + contexts.append(get_current()) + return prior, contexts + + prior, contexts = asyncio.run(run()) + spans = span_attributes(exporter) + count = 3 if ending == "exhaust" else 2 + for name in ("first", "second"): + for suffix in [*range(count), "cleanup"]: + assert_attributes(spans[f"{name}-{suffix}"], { + SpanAttributes.SESSION_ID: name, + SpanAttributes.GEN_AI_CONVERSATION_ID: name, + SpanAttributes.USER_ID: name + "-user", + SpanAttributes.TAG_TAGS: (name,), + }) + for name, attributes in spans.items(): + if name.startswith("caller-"): + assert_attributes(attributes, { + SpanAttributes.SESSION_ID: "outer-session", + SpanAttributes.USER_ID: "outer-user", + }) + assert SpanAttributes.TAG_TAGS not in attributes + assert all(context is prior for context in contexts) + + +def test_async_generator_asend_preserves_body_context_and_restores_each_caller(tracing): + tracer, exporter = tracing + received = [] + + @using_session("generator-session") + async def decorated(): + with using_user("body-user"): + try: + received.append((yield "ready")) + await asyncio.sleep(0) + with tracer.start_as_current_span("sent"): + pass + received.append((yield "sent")) + with tracer.start_as_current_span("finished"): + pass + finally: + with tracer.start_as_current_span("cleanup"): + pass + + async def run(): + with using_attributes(session_id="outer-session", user_id="outer-user"): + original = get_current() + iterator = decorated() + contexts = [] + try: + assert await iterator.__anext__() == "ready" + contexts.append((get_current(), original)) + with using_attributes(session_id="changed-session", user_id="changed-user"): + changed = get_current() + assert await iterator.asend("payload") == "sent" + contexts.append((get_current(), changed)) + with pytest.raises(StopAsyncIteration): + await iterator.asend("last") + contexts.append((get_current(), original)) + finally: + await iterator.aclose() + assert received == ["payload", "last"] + assert all(actual is expected for actual, expected in contexts) + + asyncio.run(run()) + spans = span_attributes(exporter) + assert set(spans) == {"sent", "finished", "cleanup"} + for attributes in spans.values(): + assert_attributes(attributes, { + SpanAttributes.SESSION_ID: "generator-session", + SpanAttributes.USER_ID: "body-user", + }) + + +@pytest.mark.parametrize("handled", [False, True], ids=["propagated", "handled"]) +def test_async_generator_athrow_runs_body_and_cleanup_in_its_context(tracing, handled): + tracer, exporter = tracing + error = ValueError("injected failure") + caught = [] + + @using_attributes(session_id="generator-session", user_id="generator-user") + async def decorated(): + try: + try: + yield "ready" + except ValueError as exc: + caught.append(exc) + await asyncio.sleep(0) + with tracer.start_as_current_span("caught"): + pass + if not handled: + raise + yield "recovered" + finally: + await asyncio.sleep(0) + with tracer.start_as_current_span("cleanup"): + pass + + async def run(): + with using_attributes(session_id="outer-session", user_id="outer-user"): + prior = get_current() + contexts = [] + iterator = decorated() + try: + assert await iterator.__anext__() == "ready" + contexts.append(get_current()) + if handled: + assert await iterator.athrow(error) == "recovered" + contexts.append(get_current()) + await iterator.aclose() + else: + with pytest.raises(ValueError) as raised: + await iterator.athrow(error) + assert raised.value is error + contexts.append(get_current()) + finally: + await iterator.aclose() + assert caught == [error] + assert all(context is prior for context in contexts) + + asyncio.run(run()) + spans = span_attributes(exporter) + assert set(spans) == {"caught", "cleanup"} + for attributes in spans.values(): + assert_attributes(attributes, { + SpanAttributes.SESSION_ID: "generator-session", + SpanAttributes.USER_ID: "generator-user", + }) + + +def test_async_generator_cancelled_resume_restores_caller_context(tracing): + tracer, exporter = tracing + + async def run(): + started = asyncio.Event() + pending = asyncio.Event() + + @using_session("generator-session") + async def decorated(): + try: + yield "ready" + started.set() + await pending.wait() + finally: + with tracer.start_as_current_span("cleanup"): + pass + + async def caller(): + with using_session("outer-session"): + prior = get_current() + iterator = decorated() + after_yield = prior + try: + assert await iterator.__anext__() == "ready" + after_yield = get_current() + await iterator.__anext__() + finally: + await iterator.aclose() + assert get_current() is prior + assert after_yield is prior + + task = asyncio.create_task(caller()) + try: + await asyncio.wait_for(started.wait(), timeout=2) + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + finally: + if not task.done(): + task.cancel() + await asyncio.gather(task, return_exceptions=True) + + asyncio.run(run()) + assert span_attributes(exporter)["cleanup"][SpanAttributes.SESSION_ID] == "generator-session" + + +@pytest.mark.parametrize("helper,expected", HELPERS) +@pytest.mark.parametrize("mode", ["sync-decorator", "with-block", "async-with-block"]) +def test_control_existing_context_forms(tracing, helper, expected, mode): + tracer, exporter = tracing + prior = get_current() + + def body(): + with tracer.start_as_current_span("control"): + return "result" + + if mode == "sync-decorator": + decorated = helper()(body) + assert decorated() == "result" + assert decorated.__name__ == body.__name__ + elif mode == "with-block": + with helper(): + assert body() == "result" + else: + async def run(): + before = get_current() + async with helper(): + await asyncio.sleep(0) + assert body() == "result" + assert get_current() is before + asyncio.run(run()) + + assert get_current() is prior + assert_attributes(span_attributes(exporter)["control"], expected) diff --git a/python/tests/test_custom_eval_model.py b/python/tests/test_custom_eval_model.py new file mode 100644 index 000000000..11e04aa97 --- /dev/null +++ b/python/tests/test_custom_eval_model.py @@ -0,0 +1,116 @@ +"""Regression coverage for nullable custom-evaluation models (T1-PY-CORE-005).""" + +import json +from unittest.mock import patch + +import pytest + +from fi_instrumentation import register +from fi_instrumentation.fi_types import ( + EvalName, + EvalSpanKind, + EvalTag, + EvalTagType, + ModelChoices, + ProjectType, + prepare_eval_tags, +) +from fi_instrumentation.otel import EVAL_TAGS + + +@pytest.fixture +def custom_eval_template(): + with patch( + "fi_instrumentation.fi_types.get_custom_eval_template", + return_value={ + "isUserEvalTemplate": True, + "evalTemplate": {"config": {"requiredKeys": ["output"]}}, + }, + ) as lookup: + yield lookup + + +def test_custom_eval_without_model_serializes(custom_eval_template): + tag = EvalTag( + type=EvalTagType.OBSERVATION_SPAN, + value=EvalSpanKind.LLM, + eval_name="probe_custom_eval", + mapping={"output": "output.value"}, + ) + + assert tag.model is None + custom_eval_template.assert_called_once_with("probe_custom_eval") + serialized = tag.to_dict() + assert serialized["model"] is None + assert prepare_eval_tags([tag]) == [serialized] + + +@pytest.mark.parametrize( + ("model_kwargs", "expected_model"), + [ + ({}, None), + ({"model": None}, None), + ({"model": "custom-model"}, "custom-model"), + ({"model": ModelChoices.TURING_LARGE}, ModelChoices.TURING_LARGE.value), + ], + ids=["omitted", "explicit-none", "string", "enum"], +) +def test_register_custom_eval_model( + custom_eval_template, model_kwargs, expected_model +): + tag = EvalTag( + type=EvalTagType.OBSERVATION_SPAN, + value=EvalSpanKind.LLM, + eval_name="probe_custom_eval", + mapping={"output": "output.value"}, + **model_kwargs, + ) + expected_tags = [{ + "type": EvalTagType.OBSERVATION_SPAN.value, + "value": EvalSpanKind.LLM.value, + "eval_name": "probe_custom_eval", + "config": {}, + "mapping": {"output": "output.value"}, + "custom_eval_name": "probe_custom_eval", + "model": expected_model, + }] + + with patch( + "fi_instrumentation.otel.check_custom_eval_config_exists", return_value=False + ) as check_config, patch( + "fi_instrumentation.otel.TracerProvider" + ) as provider_class, patch( + "fi_instrumentation.otel.BatchSpanProcessor" + ) as processor_class: + provider = register( + project_name="probe", + project_version_name="test-version", + eval_tags=[tag], + verbose=False, + ) + + custom_eval_template.assert_called_once_with("probe_custom_eval") + check_config.assert_called_once_with( + project_name="probe", + project_type=ProjectType.EXPERIMENT.value, + eval_tags=expected_tags, + ) + provider_class.assert_called_once() + assert provider is provider_class.return_value + resource = provider_class.call_args.kwargs["resource"] + assert json.loads(resource.attributes[EVAL_TAGS]) == expected_tags + processor_class.assert_called_once() + provider.add_span_processor.assert_called_once_with(processor_class.return_value) + + +def test_system_eval_still_requires_model(): + with patch( + "fi_instrumentation.fi_types.get_custom_eval_template", + return_value={"isUserEvalTemplate": False, "evalTemplate": {}}, + ), pytest.raises(ValueError, match="model must be a of type ModelChoices"): + EvalTag( + type=EvalTagType.OBSERVATION_SPAN, + value=EvalSpanKind.LLM, + eval_name=EvalName.TOXICITY, + mapping={"input": "input.value"}, + ) diff --git a/python/tests/test_session_attributes.py b/python/tests/test_session_attributes.py new file mode 100644 index 000000000..06d165e5a --- /dev/null +++ b/python/tests/test_session_attributes.py @@ -0,0 +1,78 @@ +"""Regression coverage for T1-PY-CORE-006 manual session attributes.""" + +import json +from unittest.mock import Mock + +import pytest +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import ( + SimpleSpanProcessor, + SpanExporter, + SpanExportResult, +) +from opentelemetry.sdk.trace.sampling import ALWAYS_ON + +from fi_instrumentation.instrumentation import ( + get_context_attributes, + get_session_attributes, +) + + +@pytest.mark.parametrize( + "helper", + [get_session_attributes, get_context_attributes], + ids=["session", "context"], +) +@pytest.mark.parametrize("session_id", ["session-A", ""], ids=["nonempty", "empty"]) +def test_manual_session_helpers_emit_both_keys(helper, session_id): + assert helper(session_id=session_id) == { + "session.id": session_id, + "gen_ai.conversation.id": session_id, + } + + +def test_context_session_preserves_other_attributes(): + attributes = get_context_attributes( + session_id="session-A", + user_id="user-A", + metadata={"source": "unit-test"}, + tags=["manual"], + ) + + assert attributes.get("session.id") == "session-A" + assert attributes["gen_ai.conversation.id"] == "session-A" + assert attributes["user.id"] == "user-A" + assert json.loads(attributes["metadata"]) == {"source": "unit-test"} + assert attributes["tag.tags"] == ["manual"] + + +@pytest.mark.parametrize("kwargs", [{}, {"session_id": None}], ids=["omitted", "none"]) +def test_context_without_session_omits_both_keys(kwargs): + assert get_context_attributes(user_id="user-A", **kwargs) == {"user.id": "user-A"} + + +@pytest.mark.parametrize( + "helper", + [get_session_attributes, get_context_attributes], + ids=["session", "context"], +) +def test_manual_session_helpers_export_native_session_in_normal_scope(helper): + # Exercise ordinary OTel spans without a collector or network exporter. + exporter = Mock(spec=SpanExporter) + exporter.export.return_value = SpanExportResult.SUCCESS + provider = TracerProvider(sampler=ALWAYS_ON, shutdown_on_exit=False) + provider.add_span_processor(SimpleSpanProcessor(exporter)) + try: + with provider.get_tracer("my-app").start_as_current_span( + "manual-operation", attributes=helper(session_id="session-A") + ): + pass + finally: + provider.shutdown() + + exporter.export.assert_called_once() + (span,) = exporter.export.call_args.args[0] + assert span.instrumentation_scope.name == "my-app" + assert span.attributes["gen_ai.conversation.id"] == "session-A" + # Native FutureAGI session identity uses session.id, not the conversation key. + assert span.attributes.get("session.id") == "session-A" diff --git a/python/tests/test_tracer_generators.py b/python/tests/test_tracer_generators.py new file mode 100644 index 000000000..8fbc3dd46 --- /dev/null +++ b/python/tests/test_tracer_generators.py @@ -0,0 +1,282 @@ +"""T1-PY-CORE-002: offline decorator regressions using fake stream functions.""" + +import asyncio +from contextvars import ContextVar + +import pytest +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +from opentelemetry.trace import StatusCode, get_current_span + +from fi_instrumentation.fi_types import SpanAttributes +from fi_instrumentation.instrumentation._tracers import FITracer +from fi_instrumentation.instrumentation.config import TraceConfig + + +@pytest.fixture(params=["chain", "agent", "tool"]) +def tracing(request): + exporter = InMemorySpanExporter() + provider = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + tracer = FITracer(provider.get_tracer("generator-regression"), TraceConfig()) + yield tracer, exporter, request.param + provider.shutdown() + + +@pytest.fixture(params=[False, True], ids=["bare", "named"]) +def decorate(tracing, request): + tracer, _, kind = tracing + decorator = getattr(tracer, kind) + return decorator(name="stream") if request.param else decorator + + +def finished(exporter, name="stream"): + return [span for span in exporter.get_finished_spans() if span.name == name] + + +def assert_error(exporter): + spans = finished(exporter) + assert len(spans) == 1 + assert spans[0].status.status_code is StatusCode.ERROR + assert any(event.name == "exception" for event in spans[0].events) + + +def test_sync_generator_lifetime_output_parent_and_return(tracing, decorate): + tracer, exporter, kind = tracing + application_state = ContextVar("application_state", default="before") + result = object() + + @decorate + def stream(): + application_state.set("inside") + with tracer.start_as_current_span("work", fi_span_kind="chain"): + yield "first" + yield "second" + return result + + iterator = stream() + assert exporter.get_finished_spans() == () + with tracer.start_as_current_span("caller", fi_span_kind="chain") as caller: + assert next(iterator) == "first" + assert application_state.get() == "inside" + assert get_current_span() is caller + assert not finished(exporter) + with tracer.start_as_current_span("consumer", fi_span_kind="chain"): + pass + assert next(iterator) == "second" + assert get_current_span() is caller + assert not finished(exporter) + with pytest.raises(StopIteration) as stopped: + next(iterator) + assert stopped.value.value is result + assert get_current_span() is caller + + span, = finished(exporter) + child, = finished(exporter, "work") + consumer, = finished(exporter, "consumer") + assert span.attributes[SpanAttributes.GEN_AI_SPAN_KIND] == kind.upper() + assert span.attributes[SpanAttributes.OUTPUT_VALUE] == "second" + assert span.parent.span_id == caller.get_span_context().span_id + assert child.parent.span_id == span.context.span_id + assert consumer.parent.span_id == caller.get_span_context().span_id + assert span.start_time <= child.start_time <= child.end_time <= span.end_time + + +def test_sync_generator_late_exception(tracing, decorate): + tracer, exporter, _ = tracing + failure = ValueError("fake stream failure") + + @decorate + def stream(): + with tracer.start_as_current_span("work", fi_span_kind="chain"): + yield "partial" + raise failure + + iterator = stream() + assert not finished(exporter) + assert next(iterator) == "partial" + with pytest.raises(ValueError) as raised: + next(iterator) + assert raised.value is failure + assert_error(exporter) + assert finished(exporter)[0].attributes[SpanAttributes.OUTPUT_VALUE] == "partial" + + +def test_sync_generator_send_throw_and_close(tracing, decorate): + tracer, exporter, _ = tracing + cleanup_parents = [] + + @decorate + def stream(): + try: + value = yield "ready" + try: + yield value + except LookupError: + yield "recovered" + finally: + cleanup_parents.append(get_current_span().get_span_context().span_id) + + iterator = stream() + assert not finished(exporter) + with pytest.raises(TypeError): + iterator.send("not started") + assert not finished(exporter) + assert next(iterator) == "ready" + assert iterator.send("sent") == "sent" + assert iterator.throw(LookupError("fake recoverable failure")) == "recovered" + assert not finished(exporter) + iterator.close() + iterator.close() + span, = finished(exporter) + assert cleanup_parents == [span.context.span_id] + assert span.attributes[SpanAttributes.OUTPUT_VALUE] == "recovered" + assert span.status.status_code is not StatusCode.ERROR + + +def test_unstarted_generators_and_argument_errors(tracing, decorate): + _, exporter, _ = tracing + entered = [] + + @decorate + def stream(required): + entered.append(True) + yield required + + with pytest.raises(TypeError): + stream() + iterator = stream("unused") + iterator.close() + assert not entered + assert not finished(exporter) + + +def test_async_generator_lifetime_output_and_parent(tracing, decorate): + tracer, exporter, _ = tracing + application_state = ContextVar("async_application_state", default="before") + + @decorate + async def stream(): + application_state.set("inside") + with tracer.start_as_current_span("work", fi_span_kind="chain"): + await asyncio.sleep(0) + yield "first" + await asyncio.sleep(0) + yield "second" + + async def consume(): + iterator = stream() + assert not finished(exporter) + with tracer.start_as_current_span("caller", fi_span_kind="chain") as caller: + assert await iterator.__anext__() == "first" + assert application_state.get() == "inside" + assert get_current_span() is caller + assert not finished(exporter) + assert await iterator.__anext__() == "second" + assert get_current_span() is caller + assert not finished(exporter) + with pytest.raises(StopAsyncIteration): + await iterator.__anext__() + assert get_current_span() is caller + + asyncio.run(consume()) + span, = finished(exporter) + child, = finished(exporter, "work") + caller, = finished(exporter, "caller") + assert span.attributes[SpanAttributes.OUTPUT_VALUE] == "second" + assert span.parent.span_id == caller.context.span_id + assert child.parent.span_id == span.context.span_id + assert span.start_time <= child.start_time <= child.end_time <= span.end_time + + +def test_async_generator_asend_athrow_and_aclose(tracing, decorate): + _, exporter, _ = tracing + cleanup_parents = [] + + @decorate + async def stream(): + try: + value = yield "ready" + try: + yield value + except LookupError: + yield "recovered" + finally: + await asyncio.sleep(0) + cleanup_parents.append(get_current_span().get_span_context().span_id) + + async def consume(): + iterator = stream() + assert not finished(exporter) + with pytest.raises(TypeError): + await iterator.asend("not started") + assert not finished(exporter) + assert await iterator.__anext__() == "ready" + assert await iterator.asend("sent") == "sent" + assert await iterator.athrow(LookupError("fake recoverable failure")) == "recovered" + assert not finished(exporter) + await iterator.aclose() + await iterator.aclose() + + asyncio.run(consume()) + span, = finished(exporter) + assert cleanup_parents == [span.context.span_id] + assert span.attributes[SpanAttributes.OUTPUT_VALUE] == "recovered" + assert span.status.status_code is not StatusCode.ERROR + + +def test_async_generator_late_exception(tracing, decorate): + _, exporter, _ = tracing + failure = ValueError("fake async stream failure") + + @decorate + async def stream(): + yield "partial" + raise failure + + async def consume(): + iterator = stream() + assert not finished(exporter) + assert await iterator.__anext__() == "partial" + with pytest.raises(ValueError) as raised: + await iterator.__anext__() + assert raised.value is failure + + asyncio.run(consume()) + assert_error(exporter) + assert finished(exporter)[0].attributes[SpanAttributes.OUTPUT_VALUE] == "partial" + + +def test_async_generator_cancellation(tracing, decorate): + _, exporter, _ = tracing + + @decorate + async def stream(): + yield "ready" + raise asyncio.CancelledError() + + async def consume(): + iterator = stream() + assert not finished(exporter) + assert await iterator.__anext__() == "ready" + with pytest.raises(asyncio.CancelledError): + await iterator.__anext__() + + asyncio.run(consume()) + assert_error(exporter) + + +def test_explicit_stream_output_is_preserved(tracing, decorate): + _, exporter, _ = tracing + + @decorate + def stream(): + get_current_span().set_output("explicit output") + yield "automatic output" + + iterator = stream() + assert not finished(exporter) + assert list(iterator) == ["automatic output"] + span, = finished(exporter) + assert span.attributes[SpanAttributes.OUTPUT_VALUE] == "explicit output" diff --git a/typescript/packages/fi-core/CHANGELOG.md b/typescript/packages/fi-core/CHANGELOG.md index 7d5e5debf..647340a82 100644 --- a/typescript/packages/fi-core/CHANGELOG.md +++ b/typescript/packages/fi-core/CHANGELOG.md @@ -1,3 +1,8 @@ +## [1.2.4] - pending release + +### Fixed +- A full collector URL set in the environment is used as given, so traces are delivered instead of failing with 404. + ## [1.2.3] - pending release ### Fix - **Upgrade.** 1.2.0, 1.2.1 and 1.2.2 could export short base64 media payloads that carry no recognized file signature: type- or modality-declared video (`type: video`, `type: input_video`, `modality: video`) at any cap, including the default 0; type-declared images (for example `{"type":"image","data":""}`) when a positive `base64ImageMaxLength` (or `FI_BASE64_IMAGE_MAX_LENGTH`) was set; and images whose declaration is a separate attribute from the payload (for example `parts.0.type` = `image` beside `parts.0.data`), at any cap. 1.2.3 omits all of these at any cap. Setting the cap to 0 is not a workaround. diff --git a/typescript/packages/fi-core/package.json b/typescript/packages/fi-core/package.json index a1dcb3102..81fef9bdf 100644 --- a/typescript/packages/fi-core/package.json +++ b/typescript/packages/fi-core/package.json @@ -1,6 +1,6 @@ { "name": "@traceai/fi-core", - "version": "1.2.3", + "version": "1.2.4", "description": "OpenTelemetry tracing for AI applications: tracer setup, span helpers, and exporters for Future AGI.", "main": "./dist/src/index.js", "module": "./dist/esm/index.js", diff --git a/typescript/packages/fi-core/src/otel.ts b/typescript/packages/fi-core/src/otel.ts index 4ced29544..73c380343 100644 --- a/typescript/packages/fi-core/src/otel.ts +++ b/typescript/packages/fi-core/src/otel.ts @@ -393,39 +393,32 @@ function getEnvGrpcCollectorEndpoint(): string { return endpoint; } -// This function now constructs the FULL endpoint URL -// It prioritizes the endpoint passed directly to `register` (if any), -// then checks FI_BASE_URL (or a more specific FI_COLLECTOR_ENDPOINT env var), -// and finally falls back to a default base URL. +// Explicit endpoints retain their base-or-full URL behavior. Environment collector +// endpoints are full URLs; only FI_BASE_URL and the default are base origins. function constructFullEndpoint(customEndpoint?: string): string { - let baseUrlToUse: string; - if (customEndpoint) { try { const parsedCustom = new URL(customEndpoint); if (!parsedCustom.protocol || !parsedCustom.host) { diag.warn(`Custom endpoint '${customEndpoint}' is missing protocol or host. Falling back to environment or default.`); - baseUrlToUse = getEnv("FI_BASE_URL") ?? getEnv("FI_COLLECTOR_ENDPOINT") ?? DEFAULT_FI_COLLECTOR_BASE_URL; } else if (parsedCustom.pathname !== "/" && parsedCustom.pathname !== FI_COLLECTOR_PATH) { diag.warn(`Using custom endpoint as full URL: ${customEndpoint}`); return customEndpoint; } else { - baseUrlToUse = `${parsedCustom.protocol}//${parsedCustom.host}`; + return `${parsedCustom.protocol}//${parsedCustom.host}${FI_COLLECTOR_PATH}`; } } catch (e) { diag.warn(`Custom endpoint '${customEndpoint}' is not a valid URL. Falling back to environment or default.`); - baseUrlToUse = getEnv("FI_BASE_URL") ?? getEnv("FI_COLLECTOR_ENDPOINT") ?? DEFAULT_FI_COLLECTOR_BASE_URL; } - } else { - baseUrlToUse = getEnv("FI_BASE_URL") ?? getEnv("FI_COLLECTOR_ENDPOINT") ?? DEFAULT_FI_COLLECTOR_BASE_URL; - } - - // Ensure no trailing slash from baseUrl before appending path - if (baseUrlToUse.endsWith('/')) { - baseUrlToUse = baseUrlToUse.slice(0, -1); } - return `${baseUrlToUse}${FI_COLLECTOR_PATH}`; + const baseUrl = getEnv("FI_BASE_URL"); + const collectorEndpoint = getEnv("FI_COLLECTOR_ENDPOINT"); + if (baseUrl === undefined && collectorEndpoint !== undefined) { + return collectorEndpoint; + } + const baseUrlToUse = baseUrl ?? DEFAULT_FI_COLLECTOR_BASE_URL; + return `${baseUrlToUse.replace(/\/$/, "")}${FI_COLLECTOR_PATH}`; } @@ -501,7 +494,8 @@ class FITracerProvider extends BasicTracerProvider { // Explicit lists, including [], never construct a hidden default exporter. let defaultProcessor: DefaultBatchSpanProcessor | DefaultSimpleSpanProcessor | undefined; if (config.spanProcessors === undefined) { - const exporter = createSpanExporter({ ...config, transport, endpoint, headers }); + // Resolve the original options, not an already-resolved full environment URL. + const exporter = createSpanExporter({ ...config, transport, headers }); defaultProcessor = config.batch ? new DefaultBatchSpanProcessor(exporter) : new DefaultSimpleSpanProcessor(exporter); diff --git a/typescript/packages/fi-core/test/provider-lifecycle.test.cjs b/typescript/packages/fi-core/test/provider-lifecycle.test.cjs index 104dea0b3..19ca1c03a 100644 --- a/typescript/packages/fi-core/test/provider-lifecycle.test.cjs +++ b/typescript/packages/fi-core/test/provider-lifecycle.test.cjs @@ -26,14 +26,15 @@ afterEach(async () => { propagation.disable(); }); -async function collector() { +async function collector(acceptedPath) { const requests = []; const server = http.createServer((request, response) => { const chunks = []; request.on('data', chunk => chunks.push(chunk)); request.on('end', () => { - requests.push({ path: request.url, body: JSON.parse(Buffer.concat(chunks).toString()) }); - response.writeHead(200, { 'Content-Type': 'application/json' }); + const status = !acceptedPath || request.url === acceptedPath ? 200 : 404; + requests.push({ path: request.url, status, body: JSON.parse(Buffer.concat(chunks).toString()) }); + response.writeHead(status, { 'Content-Type': 'application/json' }); response.end('{}'); }); }); @@ -55,6 +56,53 @@ function memoryProcessor() { return { exporter, processor: new SimpleSpanProcessor(exporter) }; } +// Each collector accepts only the configured route, so an extra suffix loses the span. +for (const route of ['/v1/traces', '/tracer/v1/traces']) { + test(`collector endpoint regression: full environment URL ${route}`, async () => { + const receiver = await collector(route); + process.env.FI_COLLECTOR_ENDPOINT = `${receiver.endpoint}${route}`; + try { + const provider = setup({ endpoint: undefined }); + provider.getTracer('endpoint').startSpan('environment-span').end(); + let flushError; + try { await provider.forceFlush(); } catch (error) { flushError = error; } + assert.deepEqual(receiver.requests.map(({ path, status }) => ({ path, status })), + [{ path: route, status: 200 }]); + assert.equal(flushError, undefined); + assert.deepEqual(receiver.spans().map(span => span.name), ['environment-span']); + } finally { + delete process.env.FI_COLLECTOR_ENDPOINT; + } + }); +} + +for (const item of [ + { name: 'base origin', base: '', path: '/tracer/v1/traces' }, + { name: 'base origin with trailing slash', base: '/', path: '/tracer/v1/traces' }, + { name: 'base origin retains precedence', base: '', env: '/unused', path: '/tracer/v1/traces' }, + { name: 'explicit full URL', endpoint: '/v1/traces', path: '/v1/traces' }, + { name: 'explicit collector URL', endpoint: '/tracer/v1/traces', path: '/tracer/v1/traces' }, + { name: 'explicit base origin', endpoint: '', path: '/tracer/v1/traces' }, + { name: 'explicit URL overrides environment', endpoint: '/v1/traces', base: '/unused', env: '/unused', path: '/v1/traces' }, +]) { + test(`collector endpoint control: ${item.name}`, async () => { + const receiver = await collector(item.path); + if (item.base !== undefined) process.env.FI_BASE_URL = `${receiver.endpoint}${item.base}`; + if (item.env !== undefined) process.env.FI_COLLECTOR_ENDPOINT = `${receiver.endpoint}${item.env}`; + try { + const provider = setup({ endpoint: item.endpoint === undefined ? undefined : `${receiver.endpoint}${item.endpoint}` }); + provider.getTracer('endpoint').startSpan('control-span').end(); + await provider.forceFlush(); + assert.deepEqual(receiver.requests.map(({ path, status }) => ({ path, status })), + [{ path: item.path, status: 200 }]); + assert.deepEqual(receiver.spans().map(span => span.name), ['control-span']); + } finally { + delete process.env.FI_BASE_URL; + delete process.env.FI_COLLECTOR_ENDPOINT; + } + }); +} + for (const batch of [false, true]) { test(`register preserves async parents on the OTLP wire (batch=${batch})`, async () => { const receiver = await collector(); diff --git a/typescript/packages/fi-semantic-conventions/CHANGELOG.md b/typescript/packages/fi-semantic-conventions/CHANGELOG.md index ea8ee718b..6fcf837ae 100644 --- a/typescript/packages/fi-semantic-conventions/CHANGELOG.md +++ b/typescript/packages/fi-semantic-conventions/CHANGELOG.md @@ -1,3 +1,8 @@ +## [1.3.0] - pending release + +### Changed +- ATTR_PROJECT_NAME now emits project_name instead of fi.project.name, and ATTR_SERVICE_NAME now emits service.name instead of SERVICE_NAME. Action: Update any resource lookups, processors or queries that read the old keys; traces already recorded keep the old keys. + ## [1.2.0] - 2026-11-10 ### Added - Align GenAI constant names with Python's `SpanAttributes`: expose `GEN_AI_*` names as named exports and on `SemanticConventions`. diff --git a/typescript/packages/fi-semantic-conventions/README.md b/typescript/packages/fi-semantic-conventions/README.md index e2daeb604..655bfae91 100644 --- a/typescript/packages/fi-semantic-conventions/README.md +++ b/typescript/packages/fi-semantic-conventions/README.md @@ -237,17 +237,30 @@ span.setAttribute('fi.span_kind', 'LLM'); ## Resource Attributes +Use the public constants when constructing a resource for a custom OpenTelemetry provider: + ```typescript -import { ResourceAttributes } from '@traceai/fi-semantic-conventions'; +import { resourceFromAttributes } from '@opentelemetry/resources'; +import { + ATTR_PROJECT_NAME, + ATTR_SERVICE_NAME, +} from '@traceai/fi-semantic-conventions'; + +const resource = resourceFromAttributes({ + [ATTR_PROJECT_NAME]: 'demo-project', + [ATTR_SERVICE_NAME]: 'demo-service', + project_type: 'observe', +}); ``` -Standard OpenTelemetry resource attributes for service identification: +| Export | Resource attribute | Description | +|--------|--------------------|-------------| +| `ATTR_PROJECT_NAME` | `project_name` | Collector project name; required when no supported project header is supplied | +| `ATTR_SERVICE_NAME` | `service.name` | Standard OpenTelemetry service name | -| Attribute | Description | -|-----------|-------------| -| `service.name` | Service name | -| `service.version` | Service version | -| `deployment.environment` | Deployment environment | +Other resource attributes, such as `service.version` and `deployment.environment`, can be supplied as literal keys. + +**Compatibility:** These constants previously emitted `fi.project.name` and `SERVICE_NAME`, respectively. They now emit the canonical collector and OpenTelemetry keys. Update resource lookups, processors, and queries that depended on the old spellings. Existing resources and historical traces using the old keys are not automatically rewritten. The project resource attribute set by `@traceai/fi-core`'s `register()` already uses `project_name` and is unchanged. ## TypeScript Configuration diff --git a/typescript/packages/fi-semantic-conventions/package.json b/typescript/packages/fi-semantic-conventions/package.json index 63c0dcfc8..30a7a4047 100644 --- a/typescript/packages/fi-semantic-conventions/package.json +++ b/typescript/packages/fi-semantic-conventions/package.json @@ -1,6 +1,6 @@ { "name": "@traceai/fi-semantic-conventions", - "version": "1.2.0", + "version": "1.3.0", "description": "Semantic conventions and span attributes for traceAI OpenTelemetry instrumentation.", "main": "./dist/src/index.js", "module": "./dist/esm/index.js", diff --git a/typescript/packages/fi-semantic-conventions/src/__tests__/resource.test.ts b/typescript/packages/fi-semantic-conventions/src/__tests__/resource.test.ts index a1759c224..60718e4ed 100644 --- a/typescript/packages/fi-semantic-conventions/src/__tests__/resource.test.ts +++ b/typescript/packages/fi-semantic-conventions/src/__tests__/resource.test.ts @@ -1,10 +1,24 @@ import { describe, it, expect } from '@jest/globals'; -import { ATTR_PROJECT_NAME, ATTR_SERVICE_NAME } from '../resource/ResourceAttributes'; +import { ATTR_PROJECT_NAME, ATTR_SERVICE_NAME } from '../index'; + +// Offline fake of OTel resourceFromAttributes: preserve the caller's keys and values. +// No provider registration, resource overlay, or project header supplies missing keys. +function fakeResourceFromAttributes(attributes: Record) { + return { attributes: { ...attributes } }; +} + +function customProviderResource() { + return fakeResourceFromAttributes({ + [ATTR_PROJECT_NAME]: 'demo-project', + [ATTR_SERVICE_NAME]: 'demo-service', + project_type: 'observe', + }); +} describe('Resource Attributes', () => { describe('Project Attributes', () => { it('should have correct project name attribute', () => { - expect(ATTR_PROJECT_NAME).toBe('fi.project.name'); + expect(ATTR_PROJECT_NAME).toBe('project_name'); expect(typeof ATTR_PROJECT_NAME).toBe('string'); }); @@ -12,14 +26,19 @@ describe('Resource Attributes', () => { expect(ATTR_PROJECT_NAME.length).toBeGreaterThan(0); }); - it('should follow semantic naming convention', () => { - expect(ATTR_PROJECT_NAME).toMatch(/^fi\./); + it('T6-TS-CORE-AUTO-003: supplies the collector project key without a project header', () => { + const resource = customProviderResource(); + + // The collector reads the literal project_name key, not fi.project.name. + expect(resource.attributes['project_name']).toBe('demo-project'); + expect(resource.attributes['project_type']).toBe('observe'); + expect(Object.keys(resource.attributes)).not.toContain('fi.project.name'); }); }); describe('Service Attributes', () => { it('should have correct service name attribute', () => { - expect(ATTR_SERVICE_NAME).toBe('SERVICE_NAME'); + expect(ATTR_SERVICE_NAME).toBe('service.name'); expect(typeof ATTR_SERVICE_NAME).toBe('string'); }); @@ -27,8 +46,11 @@ describe('Resource Attributes', () => { expect(ATTR_SERVICE_NAME.length).toBeGreaterThan(0); }); - it('should be uppercase', () => { - expect(ATTR_SERVICE_NAME).toBe(ATTR_SERVICE_NAME.toUpperCase()); + it('T6-TS-CORE-AUTO-003: supplies the standard OpenTelemetry service key', () => { + const resource = customProviderResource(); + + expect(resource.attributes['service.name']).toBe('demo-service'); + expect(Object.keys(resource.attributes)).not.toContain('SERVICE_NAME'); }); }); diff --git a/typescript/packages/fi-semantic-conventions/src/resource/ResourceAttributes.ts b/typescript/packages/fi-semantic-conventions/src/resource/ResourceAttributes.ts index cdc0dfd08..c48318a8b 100644 --- a/typescript/packages/fi-semantic-conventions/src/resource/ResourceAttributes.ts +++ b/typescript/packages/fi-semantic-conventions/src/resource/ResourceAttributes.ts @@ -1,8 +1,11 @@ -const PROJECT_NAME = "fi.project.name"; +const PROJECT_NAME = "project_name"; /** - * The project name to group traces under for traceai compatible services + * The collector resource attribute for the project name to group traces under. */ export const ATTR_PROJECT_NAME = PROJECT_NAME; -export const ATTR_SERVICE_NAME = "SERVICE_NAME"; \ No newline at end of file +/** + * The standard OpenTelemetry resource attribute for the service name. + */ +export const ATTR_SERVICE_NAME = "service.name"; diff --git a/typescript/packages/traceai_anthropic/CHANGELOG.md b/typescript/packages/traceai_anthropic/CHANGELOG.md index eeba1c795..15f0c232c 100644 --- a/typescript/packages/traceai_anthropic/CHANGELOG.md +++ b/typescript/packages/traceai_anthropic/CHANGELOG.md @@ -1,3 +1,8 @@ +## [0.1.6] - pending release + +### Fixed +- Manually configured tracing stops when disabled and resumes when re-enabled, separate copies of the Anthropic SDK loaded in one process can each be traced, and non-streamed output messages keep the text around tool calls. + ## [0.1.3] - 2026-10-18 ### Fix - Content hiding now applies to the prompt passed when the span is created (`gen_ai.input.messages`, `raw.input`), via `@traceai/fi-core` 1.1.0, which this release depends on. 0.1.2 was published against `@traceai/fi-core` `^0.1.13`, which exported those attributes unmasked. diff --git a/typescript/packages/traceai_anthropic/package.json b/typescript/packages/traceai_anthropic/package.json index 1fc0810ca..5ef196da9 100644 --- a/typescript/packages/traceai_anthropic/package.json +++ b/typescript/packages/traceai_anthropic/package.json @@ -1,6 +1,6 @@ { "name": "@traceai/anthropic", - "version": "0.1.5", + "version": "0.1.6", "description": "OpenTelemetry instrumentation for Anthropic", "main": "./dist/src/index.js", "module": "./dist/esm/index.js", diff --git a/typescript/packages/traceai_anthropic/src/__tests__/instrumentation.comprehensive.test.ts b/typescript/packages/traceai_anthropic/src/__tests__/instrumentation.comprehensive.test.ts index 8923cbd53..b6cc40051 100644 --- a/typescript/packages/traceai_anthropic/src/__tests__/instrumentation.comprehensive.test.ts +++ b/typescript/packages/traceai_anthropic/src/__tests__/instrumentation.comprehensive.test.ts @@ -72,13 +72,9 @@ describe('Anthropic Instrumentation Comprehensive Tests', () => { instrumentation.manuallyInstrument(MockAnthropic); expect(isPatched()).toBe(true); - // Note: disable() calls super.disable() which only unpatches modules - // registered through the module definition system. manuallyInstrument - // bypasses that, so the global _isFIPatched flag remains true. - // The unpatch for manual instrumentation would require calling unpatch - // directly on the module exports. instrumentation.disable(); - expect(isPatched()).toBe(true); + expect(isPatched()).toBe(false); + expect(MockAnthropic.Messages.prototype.create).toBe(mockCreate); }); it('should handle multiple enable/disable cycles', () => { @@ -87,9 +83,9 @@ describe('Anthropic Instrumentation Comprehensive Tests', () => { instrumentation.manuallyInstrument(MockAnthropic); expect(isPatched()).toBe(true); - // disable() does not reset _isFIPatched for manually instrumented modules instrumentation.disable(); - expect(isPatched()).toBe(true); + expect(isPatched()).toBe(false); + expect(MockAnthropic.Messages.prototype.create).toBe(mockCreate); }); it('should prevent double patching', () => { @@ -462,9 +458,7 @@ describe('Anthropic Instrumentation Comprehensive Tests', () => { } as any; instrumentation.enable(); - // patch() checks for Messages?.prototype and returns early if not found, - // but since _isFIPatched may already be true from prior tests in this - // suite sharing the same module-level global, we just verify no throw. + // A missing prototype must leave instrumentation state unchanged. expect(() => instrumentation.manuallyInstrument(IncompleteAnthropic)).not.toThrow(); }); @@ -539,9 +533,9 @@ describe('Anthropic Instrumentation Comprehensive Tests', () => { instrumentation.manuallyInstrument(MockAnthropic); expect(isPatched()).toBe(true); - // disable() does not reset _isFIPatched for manually instrumented modules instrumentation.disable(); - expect(isPatched()).toBe(true); + expect(isPatched()).toBe(false); + expect(MockAnthropic.Messages.prototype.create).toBe(mockCreate); }); }); }); \ No newline at end of file diff --git a/typescript/packages/traceai_anthropic/src/__tests__/manualLifecycle.test.ts b/typescript/packages/traceai_anthropic/src/__tests__/manualLifecycle.test.ts new file mode 100644 index 000000000..2a3d30135 --- /dev/null +++ b/typescript/packages/traceai_anthropic/src/__tests__/manualLifecycle.test.ts @@ -0,0 +1,302 @@ +import { afterEach, describe, expect, it, jest } from "@jest/globals"; +import Anthropic from "@anthropic-ai/sdk"; +import { + BasicTracerProvider, + InMemorySpanExporter, + SimpleSpanProcessor, +} from "@opentelemetry/sdk-trace-base"; +import { AnthropicInstrumentation, isPatched } from "../instrumentation"; + +const params = { + model: "synthetic-model", + max_tokens: 8, + messages: [{ role: "user" as const, content: "MANUAL_INPUT_CANARY" }], +}; +const response = { + id: "msg_manual_fixture", + type: "message", + role: "assistant", + model: params.model, + content: [{ type: "text", text: "MANUAL_OUTPUT_CANARY" }], + stop_reason: "end_turn", + stop_sequence: null, + usage: { input_tokens: 3, output_tokens: 2 }, +}; + +// These concrete targets never enter the CommonJS loader registry, just like +// the documented native-ESM manual setup. No SDK clients or network are used. +function manualSDK() { + class Messages { + calls: unknown[][] = []; + create(...args: unknown[]) { + this.calls.push(args); + return Promise.resolve(response); + } + } + return class LegacySDK { + static Messages = Messages; + }; +} + +function sdkModule(module: unknown): typeof Anthropic { + return module as typeof Anthropic; +} + +class TestInstrumentation extends AnthropicInstrumentation { + // Exercise the same callback a cached CommonJS loader entry would invoke. + loaderUnpatch(module: typeof Anthropic) { + this.init().unpatch?.(module); + } +} + +const instruments: AnthropicInstrumentation[] = []; +const providers: BasicTracerProvider[] = []; +function tracerProvider() { + const memory = new InMemorySpanExporter(); + const provider = new BasicTracerProvider({ spanProcessors: [new SimpleSpanProcessor(memory)] }); + providers.push(provider); + return { memory, provider }; +} +function instrument(hideContent = false) { + const { memory, provider } = tracerProvider(); + const instrumentation = new TestInstrumentation({ + instrumentationConfig: { enabled: false }, + traceConfig: { hideInputs: hideContent, hideOutputs: hideContent }, + }); + instruments.push(instrumentation); + instrumentation.setTracerProvider(provider); + instrumentation.enable(); + return { instrumentation, memory }; +} + +afterEach(async () => { + for (const instrumentation of instruments.splice(0)) instrumentation.disable(); + for (const provider of providers.splice(0)) await provider.shutdown(); +}); + +describe("real SDK patch generations", () => { + it.each([false, true])("records one SDK call once after re-enable (outer wrapper=%s)", async (outerWrapper) => { + const sdkCreate = jest.spyOn(Anthropic.Messages.prototype, "create"); + const syntheticFetch = jest.fn(async () => new Response(JSON.stringify(response), { + status: 200, + headers: { "content-type": "application/json", "request-id": "req_lifecycle_fixture" }, + })); + const client = new Anthropic({ + apiKey: "synthetic", + baseURL: "https://offline.invalid", + maxRetries: 0, + fetch: syntheticFetch as any, + }); + const { instrumentation, memory } = instrument(); + try { + instrumentation.manuallyInstrument(Anthropic); + const retained = Anthropic.Messages.prototype.create; + const delegate = function (this: Anthropic.Messages, ...args: Parameters) { + return retained.apply(this, args); + } as Anthropic.Messages["create"]; + if (outerWrapper) Anthropic.Messages.prototype.create = delegate; + + await expect(client.messages.create(params)).resolves.toEqual(response); + expect(sdkCreate).toHaveBeenCalledTimes(1); + expect(syntheticFetch).toHaveBeenCalledTimes(1); + expect(memory.getFinishedSpans()).toHaveLength(1); + + for (let cycle = 0; cycle < 2; cycle++) { + instrumentation.disable(); + expect(Anthropic.Messages.prototype.create).toBe(outerWrapper ? delegate : sdkCreate); + expect(isPatched()).toBe(false); + sdkCreate.mockClear(); + syntheticFetch.mockClear(); + memory.reset(); + await expect(client.messages.create(params)).resolves.toEqual(response); + expect(sdkCreate).toHaveBeenCalledTimes(1); + expect(syntheticFetch).toHaveBeenCalledTimes(1); + expect(memory.getFinishedSpans()).toHaveLength(0); + + instrumentation.enable(); + const active = Anthropic.Messages.prototype.create; + instrumentation.enable(); + instrumentation.manuallyInstrument(Anthropic); + expect(Anthropic.Messages.prototype.create).toBe(active); + sdkCreate.mockClear(); + syntheticFetch.mockClear(); + const promise = client.messages.create(params); + expect(typeof promise.withResponse).toBe("function"); + const result = await promise.withResponse(); + expect(result.data).toEqual(response); + expect(result.response.status).toBe(200); + expect(sdkCreate).toHaveBeenCalledTimes(1); + expect(syntheticFetch).toHaveBeenCalledTimes(1); + expect(memory.getFinishedSpans()).toHaveLength(1); + } + } finally { + instrumentation.disable(); + sdkCreate.mockRestore(); + } + }); + + it("keeps a directly retained generation inactive after re-enable", async () => { + const sdkCreate = jest.spyOn(Anthropic.Messages.prototype, "create"); + const syntheticFetch = jest.fn(async () => new Response(JSON.stringify(response), { + status: 200, + headers: { "content-type": "application/json" }, + })); + const client = new Anthropic({ apiKey: "synthetic", maxRetries: 0, fetch: syntheticFetch as any }); + const { instrumentation, memory } = instrument(); + try { + instrumentation.manuallyInstrument(Anthropic); + const retained = Anthropic.Messages.prototype.create; + instrumentation.disable(); + instrumentation.enable(); + await expect(retained.call(client.messages, params)).resolves.toEqual(response); + expect(sdkCreate).toHaveBeenCalledTimes(1); + expect(syntheticFetch).toHaveBeenCalledTimes(1); + expect(memory.getFinishedSpans()).toHaveLength(0); + await expect(client.messages.create(params)).resolves.toEqual(response); + expect(sdkCreate).toHaveBeenCalledTimes(2); + expect(syntheticFetch).toHaveBeenCalledTimes(2); + expect(memory.getFinishedSpans()).toHaveLength(1); + } finally { + instrumentation.disable(); + sdkCreate.mockRestore(); + } + }); +}); + +describe("manual ESM lifecycle", () => { + it("restores create and stops spans after disabling a manual-only target", async () => { + const SDK = manualSDK(); + const original = SDK.Messages.prototype.create; + const { instrumentation, memory } = instrument(); + const messages = new SDK.Messages(); + const options = { timeout: 123 }; + + instrumentation.manuallyInstrument(sdkModule(SDK)); + const retainedWrapper = SDK.Messages.prototype.create; + expect(retainedWrapper).not.toBe(original); + await expect(messages.create(params, options)).resolves.toBe(response); + expect(memory.getFinishedSpans()).toHaveLength(1); + + instrumentation.disable(); + expect(SDK.Messages.prototype.create).toBe(original); + expect(isPatched()).toBe(false); + expect(SDK).toHaveProperty("fiPatched", false); + await expect(messages.create(params, options)).resolves.toBe(response); + await expect(retainedWrapper.call(messages, params, options)).resolves.toBe(response); + expect(messages.calls).toEqual([[params, options], [params, options], [params, options]]); + expect(memory.getFinishedSpans()).toHaveLength(1); + + instrumentation.disable(); + expect(SDK.Messages.prototype.create).toBe(original); + }); + + it("re-enables the manual target once with a replacement tracer provider", async () => { + const SDK = manualSDK(); + const original = SDK.Messages.prototype.create; + const { instrumentation, memory } = instrument(); + const messages = new SDK.Messages(); + instrumentation.manuallyInstrument(sdkModule(SDK)); + await messages.create(params); + instrumentation.disable(); + expect(SDK.Messages.prototype.create).toBe(original); + + const replacement = tracerProvider(); + instrumentation.setTracerProvider(replacement.provider); + instrumentation.enable(); + expect(SDK.Messages.prototype.create).not.toBe(original); + const wrapped = SDK.Messages.prototype.create; + instrumentation.manuallyInstrument(sdkModule(SDK)); + instrumentation.enable(); + expect(SDK.Messages.prototype.create).toBe(wrapped); + await expect(messages.create(params)).resolves.toBe(response); + expect(memory.getFinishedSpans()).toHaveLength(1); + expect(replacement.memory.getFinishedSpans()).toHaveLength(1); + instrumentation.disable(); + expect(SDK.Messages.prototype.create).toBe(original); + expect(isPatched()).toBe(false); + }); + + it("hands off privacy configuration without letting a non-owner remove the patch", async () => { + const SDK = manualSDK(); + const original = SDK.Messages.prototype.create; + const first = instrument(); + const replacement = instrument(true); + const alias = { default: SDK }; + const messages = new SDK.Messages(); + first.instrumentation.manuallyInstrument(sdkModule(SDK)); + const firstWrapper = SDK.Messages.prototype.create; + replacement.instrumentation.manuallyInstrument(sdkModule(alias)); + replacement.instrumentation.loaderUnpatch(sdkModule(SDK)); + replacement.instrumentation.disable(); + expect(SDK.Messages.prototype.create).toBe(firstWrapper); + expect(isPatched()).toBe(true); + await messages.create(params); + expect(first.memory.getFinishedSpans()).toHaveLength(1); + expect(replacement.memory.getFinishedSpans()).toHaveLength(0); + expect(JSON.stringify(first.memory.getFinishedSpans()[0].attributes)).toContain("MANUAL_INPUT_CANARY"); + + first.instrumentation.disable(); + expect(SDK.Messages.prototype.create).toBe(original); + replacement.instrumentation.enable(); + const replacementWrapper = SDK.Messages.prototype.create; + expect(replacementWrapper).not.toBe(original); + first.instrumentation.loaderUnpatch(sdkModule(SDK)); + first.instrumentation.disable(); + expect(SDK.Messages.prototype.create).toBe(replacementWrapper); + expect(isPatched()).toBe(true); + await expect(messages.create(params)).resolves.toBe(response); + expect(first.memory.getFinishedSpans()).toHaveLength(1); + expect(replacement.memory.getFinishedSpans()).toHaveLength(1); + const attributes = JSON.stringify(replacement.memory.getFinishedSpans()[0].attributes); + expect(attributes).not.toContain("MANUAL_INPUT_CANARY"); + expect(attributes).not.toContain("MANUAL_OUTPUT_CANARY"); + replacement.instrumentation.disable(); + expect(SDK.Messages.prototype.create).toBe(original); + expect(isPatched()).toBe(false); + }); + + it("cleans up frozen namespace targets and deduplicates aliases by prototype", async () => { + const SDK = manualSDK(); + const original = SDK.Messages.prototype.create; + const namespace = Object.freeze({ default: SDK }); + const { instrumentation, memory } = instrument(); + instrumentation.manuallyInstrument(sdkModule(namespace)); + const wrapped = SDK.Messages.prototype.create; + expect(wrapped).not.toBe(original); + instrumentation.manuallyInstrument(sdkModule(SDK)); + expect(SDK.Messages.prototype.create).toBe(wrapped); + await new SDK.Messages().create(params); + expect(memory.getFinishedSpans()).toHaveLength(1); + instrumentation.disable(); + expect(SDK.Messages.prototype.create).toBe(original); + expect(isPatched()).toBe(false); + }); + + it("cleans up every owned target while preserving another owner's target", async () => { + const firstSDK = manualSDK(); + const secondSDK = manualSDK(); + const otherSDK = manualSDK(); + const originals = [firstSDK, secondSDK, otherSDK].map(SDK => SDK.Messages.prototype.create); + const first = instrument(); + const other = instrument(); + first.instrumentation.manuallyInstrument(sdkModule(firstSDK)); + first.instrumentation.manuallyInstrument(sdkModule(secondSDK)); + other.instrumentation.manuallyInstrument(sdkModule(otherSDK)); + for (const [index, SDK] of [firstSDK, secondSDK, otherSDK].entries()) { + expect(SDK.Messages.prototype.create).not.toBe(originals[index]); + await new SDK.Messages().create(params); + } + expect(first.memory.getFinishedSpans()).toHaveLength(2); + expect(other.memory.getFinishedSpans()).toHaveLength(1); + first.instrumentation.disable(); + expect(firstSDK.Messages.prototype.create).toBe(originals[0]); + expect(secondSDK.Messages.prototype.create).toBe(originals[1]); + expect(otherSDK.Messages.prototype.create).not.toBe(originals[2]); + expect(isPatched()).toBe(true); + await new otherSDK.Messages().create(params); + expect(other.memory.getFinishedSpans()).toHaveLength(2); + other.instrumentation.disable(); + expect(otherSDK.Messages.prototype.create).toBe(originals[2]); + expect(isPatched()).toBe(false); + }); +}); diff --git a/typescript/packages/traceai_anthropic/src/__tests__/plainPromise.test.ts b/typescript/packages/traceai_anthropic/src/__tests__/plainPromise.test.ts index c3fcd524f..27d437888 100644 --- a/typescript/packages/traceai_anthropic/src/__tests__/plainPromise.test.ts +++ b/typescript/packages/traceai_anthropic/src/__tests__/plainPromise.test.ts @@ -11,6 +11,7 @@ import { SimpleSpanProcessor, } from "@opentelemetry/sdk-trace-base"; import { AnthropicInstrumentation } from "../instrumentation"; +import { SemanticConventions } from "@traceai/fi-semantic-conventions"; let nextResult: unknown = null; class PlainSDK { @@ -34,6 +35,71 @@ describe("plain-promise create()", () => { await provider.shutdown(); }); + it.each([ + { + label: "adjacent text blocks", + content: [ + { type: "text", text: "first " }, + { type: "text", text: "second" }, + ], + toolCalls: undefined, + }, + { + label: "text-tool-text blocks", + content: [ + { type: "text", text: "first " }, + { + type: "tool_use", + id: "toolu_fake_weather", + name: "get_weather", + input: { location: "New York" }, + }, + { type: "text", text: "second" }, + ], + toolCalls: [{ function: { name: "get_weather", arguments: '{"location":"New York"}' } }], + }, + { + label: "a trailing empty text block", + content: [ + { type: "text", text: "first " }, + { type: "text", text: "second" }, + { type: "text", text: "" }, + ], + toolCalls: undefined, + }, + ])("preserves $label in canonical output", async ({ content, toolCalls }) => { + memory.reset(); + const response: Anthropic.Message = { + id: "msg_fake_multiblock", + type: "message", + role: "assistant", + model: "synthetic-model", + content: content as Anthropic.ContentBlock[], + stop_reason: toolCalls ? "tool_use" : "end_turn", + stop_sequence: null, + usage: { input_tokens: 3, output_tokens: 8 }, + }; + const originalResponse = JSON.stringify(response); + nextResult = response; + + const result = await messages.create(params); + await provider.forceFlush(); + + expect(result).toBe(response); + expect(JSON.stringify(result)).toBe(originalResponse); + const spans = memory.getFinishedSpans(); + expect(spans).toHaveLength(1); + expect(spans[0].status.code).toBe(SpanStatusCode.OK); + const attributes = spans[0].attributes; + expect(attributes[SemanticConventions.RAW_OUTPUT]).toBe(originalResponse); + expect(attributes[SemanticConventions.OUTPUT_VALUE]).toBe(originalResponse); + const outputMessages = JSON.parse(attributes[SemanticConventions.LLM_OUTPUT_MESSAGES] as string); + expect(outputMessages).toHaveLength(1); + expect(outputMessages[0].role).toBe("assistant"); + expect(outputMessages[0].content).toBe("first second"); + expect(outputMessages[0].tool_calls).toEqual(toolCalls); + }); + it.each([ ["null", null], ["a non-object", "unexpected"], diff --git a/typescript/packages/traceai_anthropic/src/__tests__/responseAttributes.test.ts b/typescript/packages/traceai_anthropic/src/__tests__/responseAttributes.test.ts index c60c48a6d..7d8b9e50c 100644 --- a/typescript/packages/traceai_anthropic/src/__tests__/responseAttributes.test.ts +++ b/typescript/packages/traceai_anthropic/src/__tests__/responseAttributes.test.ts @@ -267,8 +267,8 @@ describe('Anthropic Response Attributes', () => { const parsed = JSON.parse(attributes[SemanticConventions.LLM_OUTPUT_MESSAGES] as string); expect(parsed).toHaveLength(1); expect(parsed[0].role).toBe('assistant'); - // The implementation keeps the *last* text content - expect(parsed[0].content).toBe('Let me get that information.'); + // retain text on both sides of the tool call. + expect(parsed[0].content).toBe('I\'ll check the weather for you.Let me get that information.'); expect(parsed[0].tool_calls).toHaveLength(1); expect(parsed[0].tool_calls[0].function.name).toBe('get_weather'); expect(parsed[0].tool_calls[0].function.arguments).toBe('{"location":"New York"}'); diff --git a/typescript/packages/traceai_anthropic/src/__tests__/subclassLifecycle.test.ts b/typescript/packages/traceai_anthropic/src/__tests__/subclassLifecycle.test.ts new file mode 100644 index 000000000..647965e2e --- /dev/null +++ b/typescript/packages/traceai_anthropic/src/__tests__/subclassLifecycle.test.ts @@ -0,0 +1,72 @@ +import { expect, it, jest } from "@jest/globals"; +import Anthropic from "@anthropic-ai/sdk"; +import { + BasicTracerProvider, + InMemorySpanExporter, + SimpleSpanProcessor, +} from "@opentelemetry/sdk-trace-base"; +import { AnthropicInstrumentation } from "../instrumentation"; + +it("keeps the original Messages instrumented after instrumenting a Messages subclass", async () => { + class DerivedSDK extends Anthropic { + static Messages = class DerivedMessages extends Anthropic.Messages {}; + } + const params = { + model: "synthetic-model", + max_tokens: 8, + messages: [{ role: "user" as const, content: "subclass fixture" }], + }; + const response = { + id: "msg_subclass_fixture", + type: "message", + role: "assistant", + model: params.model, + content: [{ type: "text", text: "subclass response" }], + stop_reason: "end_turn", + stop_sequence: null, + usage: { input_tokens: 3, output_tokens: 2 }, + }; + const sdkCreate = jest.spyOn(Anthropic.Messages.prototype, "create"); + const syntheticFetch = jest.fn(async () => new Response(JSON.stringify(response), { + status: 200, + headers: { "content-type": "application/json" }, + })); + const client = new Anthropic({ + apiKey: "synthetic", + baseURL: "https://offline.invalid", + maxRetries: 0, + fetch: syntheticFetch as any, + }); + const memory = new InMemorySpanExporter(); + const provider = new BasicTracerProvider({ spanProcessors: [new SimpleSpanProcessor(memory)] }); + const instrumentation = new AnthropicInstrumentation({ instrumentationConfig: { enabled: false } }); + instrumentation.setTracerProvider(provider); + instrumentation.enable(); + + async function expectOneSpan(messages: Anthropic.Messages) { + memory.reset(); + sdkCreate.mockClear(); + syntheticFetch.mockClear(); + await expect(messages.create(params)).resolves.toEqual(response); + expect(sdkCreate).toHaveBeenCalledTimes(1); + expect(syntheticFetch).toHaveBeenCalledTimes(1); + expect(memory.getFinishedSpans()).toHaveLength(1); + } + + try { + instrumentation.manuallyInstrument(Anthropic); + await expectOneSpan(client.messages); + + // The subclass inherits create (and its wrapper), rather than defining its own. + instrumentation.manuallyInstrument(DerivedSDK); + await expectOneSpan(client.messages); + await expectOneSpan(new DerivedSDK.Messages(client)); + + instrumentation.manuallyInstrument(Anthropic); + await expectOneSpan(client.messages); + } finally { + instrumentation.disable(); + sdkCreate.mockRestore(); + await provider.shutdown(); + } +}); diff --git a/typescript/packages/traceai_anthropic/src/instrumentation.ts b/typescript/packages/traceai_anthropic/src/instrumentation.ts index 77d132430..2058a6464 100644 --- a/typescript/packages/traceai_anthropic/src/instrumentation.ts +++ b/typescript/packages/traceai_anthropic/src/instrumentation.ts @@ -30,10 +30,17 @@ import { type MessageStreamEvent } from "@anthropic-ai/sdk/resources/messages"; import { Stream } from "@anthropic-ai/sdk/streaming"; const MODULE_NAME = "@anthropic-ai/sdk"; -let _isFIPatched = false; +// Key ownership by the concrete target, not the module namespace: ESM aliases +// can share Messages, and frozen namespaces cannot carry a fiPatched flag. +const patchedMessages = new Map(); export function isPatched() { - return _isFIPatched; + return patchedMessages.size > 0; } function getExecContext(span: Span) { @@ -334,6 +341,7 @@ function finishInstrumentationStream( export class AnthropicInstrumentation extends InstrumentationBase{ private fiTracer!: FITracer; private _traceConfig?: TraceConfigOptions; + private _manualModules = new Set(); constructor({ instrumentationConfig, @@ -349,6 +357,16 @@ export class AnthropicInstrumentation extends InstrumentationBase{ public override enable(): void { super.enable(); this.fiTracer = new FITracer({ tracer: this.tracer, traceConfig: this._traceConfig }); + // InstrumentationBase can call enable() before subclass fields initialize. + for (const module of this._manualModules ?? []) this.patch(module); + } + + public override disable(): void { + super.disable(); + // The automatic loader never sees native-ESM manual targets. + for (const [messagesPrototype, patch] of patchedMessages) { + if (patch.owner === this) this.unpatchMessages(messagesPrototype); + } } protected init(){ @@ -362,7 +380,8 @@ export class AnthropicInstrumentation extends InstrumentationBase{ } manuallyInstrument(module: typeof Anthropic) { - this.patch(module); + this._manualModules.add(module); + if (this.isEnabled()) this.patch(module); } private patch(moduleExports: typeof Anthropic, moduleVersion?: string) { @@ -380,22 +399,23 @@ export class AnthropicInstrumentation extends InstrumentationBase{ return moduleExports; } - if ((moduleExports as any).fiPatched || _isFIPatched) { + const messagesPrototype = getMessagesPrototype(moduleExports); + if (!messagesPrototype) { + diag.error("Anthropic.Messages.prototype not found, cannot patch messages.create."); + return moduleExports; + } + if (patchedMessages.has(messagesPrototype) || (moduleExports as any).fiPatched) { return moduleExports; } // eslint-disable-next-line @typescript-eslint/no-this-alias const instrumentation: AnthropicInstrumentation = this; - if (!(instrumentation.fiTracer as any).tracer || (instrumentation.fiTracer as any).tracer !== instrumentation.tracer) { + if (!instrumentation.fiTracer || (instrumentation.fiTracer as any).tracer !== instrumentation.tracer) { instrumentation.fiTracer = new FITracer({ tracer: instrumentation.tracer, traceConfig: instrumentation._traceConfig }); } - const messagesPrototype = getMessagesPrototype(moduleExports); - if (!messagesPrototype) { - diag.error("Anthropic.Messages.prototype not found, cannot patch messages.create."); - return moduleExports; - } - + const previousCreate = messagesPrototype.create; + const generation = Symbol("messages.create"); // Patch messages.create this._wrap( messagesPrototype, @@ -404,6 +424,11 @@ export class AnthropicInstrumentation extends InstrumentationBase{ // Not async: the SDK returns an APIPromise whose withResponse(), asResponse() // and lazy body parsing callers (including messages.stream()) depend on. return function patchedMessagesCreate(this: Anthropic.Messages, ...args: Parameters) { + // Retained wrappers stay inactive even after this instance is re-enabled. + // Unpatch removes the registration; a later patch gets a new generation. + if (!instrumentation.isEnabled() || patchedMessages.get(messagesPrototype)?.generation !== generation) { + return originalCreate.apply(this, args); + } const params = args[0] as Anthropic.MessageCreateParams; const { messages: _messages, system: _system, tools: _tools, ...invocationParameters } = params; @@ -504,7 +529,13 @@ export class AnthropicInstrumentation extends InstrumentationBase{ } ); - _isFIPatched = true; + if (messagesPrototype.create === previousCreate) return moduleExports; + patchedMessages.set(messagesPrototype, { + owner: this, + moduleExports, + create: messagesPrototype.create, + generation, + }); try { (moduleExports as any).fiPatched = true; } catch (e) { @@ -516,13 +547,20 @@ export class AnthropicInstrumentation extends InstrumentationBase{ private unpatch(moduleExports: typeof Anthropic & { fiPatched?: boolean }) { const messagesPrototype = getMessagesPrototype(moduleExports); if (messagesPrototype) { - this._unwrap(messagesPrototype, "create"); + this.unpatchMessages(messagesPrototype); } else { diag.warn("Anthropic.Messages.prototype not found during unpatch, could not unwrap messages.create."); } - _isFIPatched = false; + } + + private unpatchMessages(messagesPrototype: Anthropic.Messages) { + const patch = patchedMessages.get(messagesPrototype); + // Loader callbacks from a different instance must not remove our wrapper. + if (!patch || patch.owner !== this) return; + if (messagesPrototype.create === patch.create) this._unwrap(messagesPrototype, "create"); + patchedMessages.delete(messagesPrototype); try { - (moduleExports as any).fiPatched = false; + (patch.moduleExports as any).fiPatched = false; } catch (e) { diag.warn(`Failed to unset fiPatched flag on module '${MODULE_NAME}'. This is usually not an issue. Error: ${e}`); } diff --git a/typescript/packages/traceai_anthropic/src/responseAttributes.ts b/typescript/packages/traceai_anthropic/src/responseAttributes.ts index f6c08a0e3..ca85e3546 100644 --- a/typescript/packages/traceai_anthropic/src/responseAttributes.ts +++ b/typescript/packages/traceai_anthropic/src/responseAttributes.ts @@ -52,13 +52,13 @@ export function getAnthropicOutputMessagesAttributes( response: Anthropic.Message, ): Attributes { const outputMessage: Record = { role: response.role }; - let lastTextContent = ""; + let textContent = ""; const toolCalls: Record[] = []; if (response.content && Array.isArray(response.content)) { response.content.forEach((contentBlock: Anthropic.ContentBlock) => { if (contentBlock.type === "text") { - lastTextContent = (contentBlock as Anthropic.TextBlock).text; + textContent += (contentBlock as Anthropic.TextBlock).text; } else if (contentBlock.type === "tool_use") { const toolUseBlock = contentBlock as Anthropic.ToolUseBlock; toolCalls.push({ @@ -68,8 +68,8 @@ export function getAnthropicOutputMessagesAttributes( }); } - if (lastTextContent) { - outputMessage.content = lastTextContent; + if (textContent) { + outputMessage.content = textContent; } if (toolCalls.length > 0) { outputMessage.tool_calls = toolCalls; diff --git a/typescript/packages/traceai_anthropic/src/version.ts b/typescript/packages/traceai_anthropic/src/version.ts index 90d2111f1..95b82c785 100644 --- a/typescript/packages/traceai_anthropic/src/version.ts +++ b/typescript/packages/traceai_anthropic/src/version.ts @@ -1 +1 @@ -export const VERSION = '0.1.5'; // x-release-please-version \ No newline at end of file +export const VERSION = '0.1.6'; // x-release-please-version \ No newline at end of file diff --git a/typescript/packages/traceai_auto/CHANGELOG.md b/typescript/packages/traceai_auto/CHANGELOG.md index 21789ab21..51798d665 100644 --- a/typescript/packages/traceai_auto/CHANGELOG.md +++ b/typescript/packages/traceai_auto/CHANGELOG.md @@ -1,8 +1,12 @@ -## [Unreleased] +## [0.1.4] - pending release ### Added - Registry entries for `@traceai/n8n` (import hook on `n8n-core`), `@traceai/openrouter` and `@traceai/vitest-evals` (both manual setup), so `traceai-auto` reports them and the coverage table lists them. +### Fixed +- OpenAI-compatible Chat Completions and Responses calls captured from the global fetch, including streamed response.failed events, mark the span as an error when the provider returns a failure inside a successful HTTP response, with the provider's message when one is given. Calls made with the OpenAI SDK are traced by the OpenAI instrumentation instead. +- Gemini capture records every generated candidate, not only the first. + ## [0.1.3] - pending release ### Added diff --git a/typescript/packages/traceai_auto/docs/ARCHITECTURE.md b/typescript/packages/traceai_auto/docs/ARCHITECTURE.md index 0ad7e2786..c08a328e8 100644 --- a/typescript/packages/traceai_auto/docs/ARCHITECTURE.md +++ b/typescript/packages/traceai_auto/docs/ARCHITECTURE.md @@ -407,10 +407,12 @@ record delimiter, and partial JSON is never promoted to a completed frame. even when the native cancellation promise rejects. Application bytes, promises and errors remain unchanged, but the one span can be incomplete, omit later output/ usage and end too early. Later consumption can still return the complete answer and usage; branch-aware cancellation finalization is not provided. -- A provider failure carried inside an HTTP 200 stream (for example OpenAI Responses - `response.failed`) can retain HTTP-derived OK status without the provider error - description, even when `gen_ai.response.finish_reasons` records `failed`. HTTP - success is not currently overridden by that in-stream provider failure. +- Captured OpenAI-compatible chat/completions and Responses JSON/SSE errors + (including Responses `status: "failed"` and `response.failed`) override HTTP + success in telemetry, retaining partial output and usage. Provider error codes + and descriptions pass through the existing privacy scrubbing. Other provider-specific + failure envelopes, or failures hidden by unavailable/truncated bodies, can still + retain HTTP-derived status; application response bytes and errors remain unchanged. - Native `pipeTo`/`pipeThrough` algorithms bypass instance reader methods. These paths retain metadata-only completion (`traceai.wire.body_unavailable=true`), without replacing the stream/sink or fabricating output bytes. Their completion diff --git a/typescript/packages/traceai_auto/package.json b/typescript/packages/traceai_auto/package.json index b6a1dbcbd..50c350d57 100644 --- a/typescript/packages/traceai_auto/package.json +++ b/typescript/packages/traceai_auto/package.json @@ -1,6 +1,6 @@ { "name": "@traceai/auto", - "version": "0.1.3", + "version": "0.1.4", "description": "Fail-open, zero-code AI auto-instrumentation for Node.js", "main": "./dist/src/index.js", "module": "./dist/esm/index.js", diff --git a/typescript/packages/traceai_auto/src/index.ts b/typescript/packages/traceai_auto/src/index.ts index fdea046a7..7eea28f79 100644 --- a/typescript/packages/traceai_auto/src/index.ts +++ b/typescript/packages/traceai_auto/src/index.ts @@ -13,7 +13,7 @@ import { captureSecrets, SpanResult } from "./wire/core"; import { withDeadline } from "./deadline"; import { privacyFilter } from "./export-privacy"; -const VERSION = "0.1.3"; // x-release-please-version +const VERSION = "0.1.4"; // x-release-please-version import { chainSigterm } from "./signals"; import { observeTransports } from "./wire/observe-node"; import { borrowConvertedSpanDestination, ConvertedSpanDestination } from "./converted-spans"; diff --git a/typescript/packages/traceai_auto/src/wire/core.ts b/typescript/packages/traceai_auto/src/wire/core.ts index 488d63c6f..12ea70fea 100644 --- a/typescript/packages/traceai_auto/src/wire/core.ts +++ b/typescript/packages/traceai_auto/src/wire/core.ts @@ -90,12 +90,17 @@ function json(data?: Uint8Array | string): any { export function parseSSE(data: Uint8Array): [string | undefined, any][] { return streamFrames(data, true).map(frame => [frame.event, frame.payload]); } -interface Body { kind: string; data?: any; text?: string; payloads: any[] } +interface Body { kind: string; data?: any; text?: string; payloads: any[]; eventNames?: (string | undefined)[] } function decode(resp: ResponseData): Body { const type = header(resp.headers, "content-type"); if (isBinary(type)) return { kind: "binary", payloads: [] }; if (!resp.body || resp.truncated) return { kind: "empty", payloads: [] }; - if (type === "text/event-stream") return { kind: "sse", payloads: parseSSE(resp.body).filter(([, payload]) => object(payload)).map(([event, payload]) => event && payload.type === undefined ? { ...payload, type: event } : payload) }; + if (type === "text/event-stream") { + const frames = parseSSE(resp.body).filter(([, payload]) => object(payload)); + // Keep event names independently: an error payload can use type for its error code. + return { kind: "sse", eventNames: frames.map(([event]) => event), + payloads: frames.map(([event, payload]) => event && payload.type === undefined ? { ...payload, type: event } : payload) }; + } const text = decodeBytes(resp.body); if (["application/x-ndjson", "application/jsonl", "application/ndjson"].includes(type)) { return { kind: "ndjson", payloads: streamFrames(resp.body, false, Infinity, !resp.error && !resp.incomplete && !resp.truncated).map(frame => frame.payload).filter(object) }; @@ -175,6 +180,7 @@ interface Parsed { usage: Record; usageFamily?: "anthropic" | "gemini" | "converse"; inputValue?: string; inputMime?: string; outputValue?: string; outputMime?: string; extra: Attributes; skip?: boolean; + providerError?: { type: string; message?: string }; } function parsed(): Parsed { return { finish: [], inputs: [], outputs: [], tools: [], params: {}, usage: {}, extra: {} }; } function usage(p: Parsed, values: Record): void { @@ -363,7 +369,16 @@ function openaiUsage(p: Parsed, u: any): void { function transcript(p: Parsed, text: any): void { if (typeof text === "string") { p.extra["gen_ai.audio.transcript"] = text; p.outputValue = text; p.outputMime = "text/plain"; } } +function recordProviderError(p: Parsed, error: any, failed = false): void { + if (!failed && !object(error) && !(typeof error === "string" && error)) return; + const code = typeof error?.code === "string" && error.code ? error.code : num(error?.code); + const type = typeof error?.type === "string" && error.type !== "error" ? error.type : undefined; + // Failure is sticky: subsequent output/usage or completion records cannot undo it. + p.providerError ??= { type: String(code ?? (type || "provider_error")), + message: typeof error === "string" ? error : typeof error?.message === "string" ? error.message : undefined }; +} function responseOutput(p: Parsed, r: any): void { + recordProviderError(p, r.error, r.status === "failed"); p.responseId = r.id || p.responseId; p.responseModel = r.model || p.responseModel; const out = message("assistant"), texts: string[] = []; for (const item of list(r.output).filter(object)) { @@ -380,6 +395,7 @@ function responseParser(m: Match, body: Body, p: Parsed): void { switch (f) { case "openai.chat": case "openai.completions": if (body.kind === "json") { + recordProviderError(p, d.error); p.responseId = d.id; p.responseModel = d.model; for (const c of list(d.choices)) { if (c.message && Object.keys(c.message).length) p.outputs.push(openaiMessage(c.message)); @@ -389,7 +405,9 @@ function responseParser(m: Match, body: Body, p: Parsed): void { openaiUsage(p, d.usage); } else if (body.kind === "sse") { const slots = new Map }>(), finish = new Map(); - for (const chunk of events) { + for (const [index, chunk] of events.entries()) { + const errorEvent = body.eventNames?.[index] === "error" || chunk.type === "error"; + recordProviderError(p, chunk.error ?? (errorEvent ? chunk : undefined), errorEvent); p.responseId ||= chunk.id; p.responseModel ||= chunk.model; openaiUsage(p, chunk.usage); for (const c of list(chunk.choices)) { const i = c.index ?? 0, delta = c.delta || {}; @@ -416,7 +434,10 @@ function responseParser(m: Match, body: Body, p: Parsed): void { if (body.kind === "json") responseOutput(p, d); else if (body.kind === "sse") { let final: any; const deltas: string[] = [], calls = new Map(); - for (const e of events) { + for (const [index, e] of events.entries()) { + const event = body.eventNames?.[index], errorEvent = event === "error" || e.type === "error"; + recordProviderError(p, e.response?.error ?? e.error ?? (errorEvent ? e : undefined), + event === "response.failed" || e.type === "response.failed" || errorEvent || e.response?.status === "failed"); if (object(e.response)) openaiUsage(p, e.response.usage); if (["response.completed", "response.incomplete", "response.failed"].includes(e.type) && object(e.response)) final = e.response; else if (e.type === "response.created" && object(e.response)) { p.responseId = e.response.id; p.responseModel = e.response.model; } @@ -495,18 +516,27 @@ function responseParser(m: Match, body: Body, p: Parsed): void { } case "gemini.generate_content": { p.usageFamily = "gemini"; - const items = body.kind === "sse" ? events : Array.isArray(body.data) ? body.data : [d], out = message("assistant"), texts: string[] = []; + const items = body.kind === "sse" ? events : Array.isArray(body.data) ? body.data : [d]; + const slots = new Map(); for (const item of items.filter(object)) { p.responseModel = item.modelVersion || p.responseModel; p.responseId = item.responseId || p.responseId; - const candidates = list(item.candidates); - if (object(candidates[0])) { - const msg = geminiMessage(candidates[0].content || {}); if (msg.content) texts.push(msg.content); out.calls.push(...msg.calls); - const reasons = candidates.filter(x => x?.finishReason).map(x => String(x.finishReason)); if (reasons.length) p.finish = reasons; + // Positions can change between chunks; candidate indices own content and finish reasons. + for (const [position, candidate] of list(item.candidates).entries()) { + if (!object(candidate)) continue; + const index = int(candidate.index) ?? position; + let slot = slots.get(index); + if (!slot) { slot = { out: message("assistant"), texts: [] }; slots.set(index, slot); } + const msg = geminiMessage(candidate.content || {}); + if (msg.content) slot.texts.push(msg.content); + slot.out.calls.push(...msg.calls); + if (candidate.finishReason) slot.finish = String(candidate.finishReason); } const u = item.usageMetadata; usage(p, { input: u?.promptTokenCount, tool_input: u?.toolUsePromptTokenCount, output: u?.candidatesTokenCount, total: u?.totalTokenCount, cache_read: u?.cachedContentTokenCount, reasoning: u?.thoughtsTokenCount }); } - out.content = texts.join("") || undefined; if (out.content !== undefined || out.calls.length) p.outputs = [out]; return; + const ordered = [...slots].sort(([a], [b]) => a - b).map(([, slot]) => slot); + p.outputs = ordered.map(slot => ({ ...slot.out, content: slot.texts.join("") || undefined })); + p.finish = ordered.flatMap(slot => slot.finish ? [slot.finish] : []); return; } case "gemini.embed_content": { const first = d.embedding || d.embeddings?.[0]; @@ -611,9 +641,10 @@ function errorMessage(body: Body): string | undefined { export function build(m: Match, req: RequestData, resp: ResponseData, initial?: Parsed): SpanResult | undefined { try { const p = initial ?? checkRequest(m, req); if (!p) return undefined; - const e = m.endpoint, body = decode(resp), ok = !resp.error && (resp.status ?? 0) < 400; + const e = m.endpoint, body = decode(resp), transportOk = !resp.error && (resp.status ?? 0) < 400; if (e.only_when === "completed" && !resp.error && !["completed", "error"].includes(body.data?.status)) return undefined; - if (ok || resp.error) { try { responseParser(m, body, p); } catch { /* Malformed provider data is nonfatal. */ } if (p.skip) return undefined; } + if (transportOk || resp.error) { try { responseParser(m, body, p); } catch { /* Malformed provider data is nonfatal. */ } if (p.skip) return undefined; } + const ok = transportOk && !p.providerError; const a: Attributes = { "gen_ai.span.kind": e.span_kind, "gen_ai.operation.name": e.operation, "gen_ai.provider.name": m.provider, "traceai.capture.layer": "wire", "traceai.wire.endpoint": e.id, "server.address": m.host, "server.port": m.port, "http.request.method": req.method.toUpperCase() }; @@ -664,7 +695,13 @@ export function build(m: Match, req: RequestData, resp: ResponseData, initial?: if (resp.undecodedEncoding) a["traceai.wire.undecoded_encoding"] = resp.undecodedEncoding; const model = p.requestModel || p.responseModel; let description: string | undefined; - if (!ok) { a["error.type"] = resp.error ? resp.error.name : String(resp.status); description = resp.error ? resp.error.message || resp.error.name : errorMessage(body) || `HTTP ${resp.status}`; } + if (resp.error) { + a["error.type"] = resp.error.name; description = resp.error.message || resp.error.name; + } else if (!transportOk) { + a["error.type"] = String(resp.status); description = errorMessage(body) || `HTTP ${resp.status}`; + } else if (p.providerError) { + a["error.type"] = p.providerError.type; description = p.providerError.message || "Provider error"; + } let secrets = credentialSecrets(req.headers, req.secrets), privacyFailed = false; try { // Preserve raw request credentials privately before best-effort scrubbing erases diff --git a/typescript/packages/traceai_auto/test/gemini-candidates.test.ts b/typescript/packages/traceai_auto/test/gemini-candidates.test.ts new file mode 100644 index 000000000..3b81f92b3 --- /dev/null +++ b/typescript/packages/traceai_auto/test/gemini-candidates.test.ts @@ -0,0 +1,118 @@ +import { SpanResult } from "../src/wire/core"; +import { patchFetch } from "../src/wire/fetch"; + +const usageMetadata = { promptTokenCount: 3, candidatesTokenCount: 2, totalTokenCount: 5 }; + +async function capture(raw: string, type = "application/json", operation = "generateContent", candidateCount = 2) { + const url = `https://generativelanguage.googleapis.com/v1beta/models/gemini-2.0-flash:${operation}`; + const init = { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ + contents: [{ role: "user", parts: [{ text: "hello" }] }], generationConfig: { candidateCount }, + }) }; + const original = globalThis.fetch, spans: SpanResult[] = []; + const response = new Response(raw, { headers: { "content-type": type } }); + const mock = jest.fn(async () => response); + globalThis.fetch = mock; + const hook = patchFetch(4194304, span => spans.push(span)); + try { + const received = await fetch(url, init); + expect(received).toBe(response); + expect(received.status).toBe(200); + expect(await received.text()).toBe(raw); + await Promise.all([...hook.pending]); + expect(mock).toHaveBeenCalledTimes(1); + expect(mock).toHaveBeenCalledWith(url, init); + expect(spans).toHaveLength(1); + expect(spans[0].ok).toBe(true); + expect(spans[0].attributes["traceai.wire.truncated"]).toBeUndefined(); + expect(spans[0].attributes["traceai.wire.incomplete"]).toBeUndefined(); + return spans[0].attributes; + } finally { + hook.remove(); + globalThis.fetch = original; + } +} + +test("T6-TS-CORE-AUTO-005/json-multiple: retain both answers without changing application bytes", async () => { + const a = await capture(JSON.stringify({ candidates: [ + { index: 0, content: { role: "model", parts: [{ text: "First" }] }, finishReason: "STOP" }, + { index: 1, content: { role: "model", parts: [{ text: "Second" }] }, finishReason: "STOP" }, + ], usageMetadata })); + expect(a["output.value"]).toBe("First"); + expect(a["gen_ai.output.messages.0.message.role"]).toBe("assistant"); + expect(a["gen_ai.output.messages.0.message.content"]).toBe("First"); + expect(a["gen_ai.output.messages.1.message.role"]).toBe("assistant"); + expect(a["gen_ai.output.messages.1.message.content"]).toBe("Second"); + expect(a["gen_ai.output.messages.2.message.role"]).toBeUndefined(); + expect(a["gen_ai.response.finish_reasons"]).toEqual(["STOP", "STOP"]); + expect(a["gen_ai.usage.input_tokens"]).toBe(3); + expect(a["gen_ai.usage.output_tokens"]).toBe(2); + expect(a["gen_ai.usage.total_tokens"]).toBe(5); +}); + +test("T6-TS-CORE-AUTO-005/json-order-tools: sort sparse indices and retain a tool-only alternative", async () => { + const a = await capture(JSON.stringify({ candidates: [ + { index: 4, content: { role: "model", parts: [{ functionCall: { id: "alternative-call", name: "lookup", args: { city: "Paris" } } }] }, finishReason: "MAX_TOKENS" }, + { index: 0, content: { role: "model", parts: [{ text: "First" }] }, finishReason: "STOP" }, + ] })); + expect(a["output.value"]).toBe("First"); + expect(a["gen_ai.output.messages.0.message.content"]).toBe("First"); + expect(a["gen_ai.output.messages.0.message.tool_calls.0.tool_call.function.name"]).toBeUndefined(); + expect(a["gen_ai.output.messages.1.message.role"]).toBe("assistant"); + expect(a["gen_ai.output.messages.1.message.content"]).toBeUndefined(); + expect(a["gen_ai.output.messages.1.message.tool_calls.0.tool_call.id"]).toBe("alternative-call"); + expect(a["gen_ai.output.messages.1.message.tool_calls.0.tool_call.function.name"]).toBe("lookup"); + expect(a["gen_ai.output.messages.1.message.tool_calls.0.tool_call.function.arguments"]).toBe('{"city":"Paris"}'); + expect(a["gen_ai.response.finish_reasons"]).toEqual(["STOP", "MAX_TOKENS"]); +}); + +test("T6-TS-CORE-AUTO-005/json-unindexed: fall back to array positions for omitted indices", async () => { + const a = await capture(JSON.stringify({ candidates: [ + { content: { role: "model", parts: [{ text: "First" }] } }, + { content: { role: "model", parts: [{ text: "Second" }] } }, + ] })); + expect(a["output.value"]).toBe("First"); + expect(a["gen_ai.output.messages.0.message.content"]).toBe("First"); + expect(a["gen_ai.output.messages.1.message.content"]).toBe("Second"); +}); + +test.each(["sse", "json"])("T6-TS-CORE-AUTO-005/stream-%s: accumulate text, calls and terminal reasons by candidate index", async encoding => { + const chunks = [ + { candidates: [ + { index: 1, content: { role: "model", parts: [{ text: "Sec" }] } }, + { index: 0, content: { role: "model", parts: [{ text: "Fir" }] } }, + ] }, + { candidates: [{ index: 1, content: { parts: [{ text: "ond" }, { functionCall: { id: "second-call", name: "second_lookup", args: { city: "Paris" } } }] } }] }, + { candidates: [{ index: 0, content: { parts: [{ text: "st" }, { functionCall: { id: "first-call", name: "first_lookup", args: { city: "Rome" } } }] } }] }, + { candidates: [{ index: 1, finishReason: "MAX_TOKENS" }] }, + { candidates: [{ index: 0, finishReason: "STOP" }] }, + { usageMetadata }, + ]; + const raw = encoding === "sse" ? chunks.map(chunk => `data: ${JSON.stringify(chunk)}\n\n`).join("") : JSON.stringify(chunks); + const a = await capture(raw, encoding === "sse" ? "text/event-stream" : "application/json", "streamGenerateContent"); + expect(a["gen_ai.request.stream"]).toBe(true); + expect(a["output.value"]).toBe("First"); + expect(a["gen_ai.output.messages.0.message.content"]).toBe("First"); + expect(a["gen_ai.output.messages.1.message.content"]).toBe("Second"); + for (const [index, id, name, city] of [[0, "first-call", "first_lookup", "Rome"], [1, "second-call", "second_lookup", "Paris"]]) { + const base = `gen_ai.output.messages.${index}.message`; + expect(a[`${base}.role`]).toBe("assistant"); + expect(a[`${base}.tool_calls.0.tool_call.id`]).toBe(id); + expect(a[`${base}.tool_calls.0.tool_call.function.name`]).toBe(name); + expect(a[`${base}.tool_calls.0.tool_call.function.arguments`]).toBe(JSON.stringify({ city })); + expect(a[`${base}.tool_calls.1.tool_call.function.name`]).toBeUndefined(); + } + expect(a["gen_ai.output.messages.2.message.role"]).toBeUndefined(); + expect(a["gen_ai.response.finish_reasons"]).toEqual(["STOP", "MAX_TOKENS"]); + expect(a["gen_ai.usage.total_tokens"]).toBe(5); +}); + +test("T6-TS-CORE-AUTO-005/single-control: preserve an unindexed single candidate", async () => { + const a = await capture(JSON.stringify({ candidates: [ + { content: { role: "model", parts: [{ text: "First" }] }, finishReason: "STOP" }, + ] }), "application/json", "generateContent", 1); + expect(a["output.value"]).toBe("First"); + expect(a["output.mime_type"]).toBe("text/plain"); + expect(a["gen_ai.output.messages.0.message.content"]).toBe("First"); + expect(a["gen_ai.output.messages.1.message.role"]).toBeUndefined(); + expect(a["gen_ai.response.finish_reasons"]).toEqual(["STOP"]); +}); diff --git a/typescript/packages/traceai_auto/test/provider-failure.test.ts b/typescript/packages/traceai_auto/test/provider-failure.test.ts new file mode 100644 index 000000000..aff479dde --- /dev/null +++ b/typescript/packages/traceai_auto/test/provider-failure.test.ts @@ -0,0 +1,170 @@ +import { build, match, SpanResult } from "../src/wire/core"; +import { createFetchHook, FetchHook } from "../src/wire/fetch-runtime"; + +const encoder = new TextEncoder(); +const failure = { code: "server_error", message: "Synthetic provider failure" }; +const usage = { input_tokens: 7, output_tokens: 2, total_tokens: 9 }; +const output = [{ type: "message", role: "assistant", content: [{ type: "output_text", text: "Partial answer" }] }]; +const frame = (payload: unknown, event?: string) => `${event ? `event: ${event}\n` : ""}data: ${JSON.stringify(payload)}\n\n`; +let original: typeof fetch; +let hook: FetchHook | undefined; + +beforeEach(() => { original = globalThis.fetch; }); +afterEach(() => { hook?.remove(); hook = undefined; globalThis.fetch = original; }); + +function request(path: string, authorization?: string): RequestInit { + const input = path === "responses" ? { input: "hello" } : path === "completions" ? { prompt: "hello" } : { messages: [{ role: "user", content: "hello" }] }; + return { method: "POST", headers: { "content-type": "application/json", ...(authorization ? { authorization } : {}) }, + body: JSON.stringify({ model: "fixture-model", ...input }) }; +} + +async function observe(path: string, body: string, type = "text/event-stream", status = 200, authorization?: string): Promise { + const spans: SpanResult[] = [], bytes = encoder.encode(body); + const response = new Response(bytes, { status, headers: { "content-type": type } }); + const promise = Promise.resolve(response); + const fake = jest.fn(() => promise); + globalThis.fetch = fake; + hook = createFetchHook(4194304, span => spans.push(span), { + active: () => undefined, skip: () => false, run: (_parent, fn) => fn(), + }); + const init = request(path, authorization), url = `https://api.openai.com/v1/${path}`; + const actualPromise = fetch(url, init); + expect(actualPromise).toBe(promise); + const actual = await actualPromise; + expect(actual).toBe(response); + expect(actual.status).toBe(status); + const reader = actual.body!.getReader(), received: number[] = []; + while (true) { + const item = await reader.read(); + if (item.done) break; + received.push(...item.value); + } + await Promise.all([...hook.pending]); + expect(new Uint8Array(received)).toEqual(bytes); + expect(fake).toHaveBeenCalledTimes(1); + expect(fake).toHaveBeenCalledWith(url, init); + expect(spans).toHaveLength(1); + return spans[0]; +} + +describe("T6-TS-CORE-AUTO-001", () => { + test("Responses response.failed overrides HTTP success and retains partial output and usage", async () => { + const body = frame({ type: "response.created", response: { id: "resp_fixture", model: "fixture-model", status: "in_progress" } }) + + frame({ type: "response.output_text.delta", delta: "Partial answer" }) + + frame({ type: "response.failed", response: { id: "resp_fixture", status: "failed", error: failure, output: [], usage } }, "response.failed"); + const result = await observe("responses", body); + expect(result).toMatchObject({ ok: false, description: failure.message, attributes: { + "error.type": "server_error", "http.response.status_code": 200, + "gen_ai.response.id": "resp_fixture", "gen_ai.response.finish_reasons": ["failed"], + "output.value": "Partial answer", "gen_ai.usage.input_tokens": 7, + "gen_ai.usage.output_tokens": 2, "gen_ai.usage.total_tokens": 9, + } }); + }); + + test("Responses failed JSON overrides HTTP success without dropping output or usage", async () => { + const result = await observe("responses", JSON.stringify({ id: "resp_fixture", status: "failed", error: failure, output, usage }), "application/json"); + expect(result).toMatchObject({ ok: false, description: failure.message, attributes: { + "error.type": "server_error", "gen_ai.response.finish_reasons": ["failed"], + "output.value": "Partial answer", "gen_ai.usage.total_tokens": 9, + } }); + }); + + test("compatible chat SSE errors retain preceding output and usage", async () => { + const body = frame({ choices: [{ index: 0, delta: { role: "assistant", content: "Partial answer" } }], + usage: { prompt_tokens: 7, completion_tokens: 2, total_tokens: 9 } }) + + frame({ error: { type: "server_error", message: failure.message } }, "error"); + const result = await observe("chat/completions", body); + expect(result).toMatchObject({ ok: false, description: failure.message, attributes: { + "error.type": "server_error", "output.value": "Partial answer", "gen_ai.usage.total_tokens": 9, + } }); + }); + + test.each([ + { label: "Responses standalone error", path: "responses", body: frame({ type: "error", ...failure }), type: "text/event-stream" }, + { label: "Responses event-name-only error", path: "responses", body: frame(failure, "error"), type: "text/event-stream" }, + { label: "Responses error event with a provider type", path: "responses", body: frame({ type: "server_error", message: failure.message }, "error"), type: "text/event-stream" }, + { label: "chat error event with a provider type", path: "chat/completions", body: frame({ type: "server_error", message: failure.message }, "error"), type: "text/event-stream" }, + { label: "Responses event-name-only failure", path: "responses", body: frame({ response: { error: failure } }, "response.failed"), type: "text/event-stream" }, + { label: "completions SSE error", path: "completions", body: frame({ error: failure }), type: "text/event-stream" }, + { label: "chat JSON error", path: "chat/completions", body: JSON.stringify({ error: failure }), type: "application/json" }, + ])("$label records the provider code and message", async ({ path, body, type }) => { + expect(await observe(path, body, type)).toMatchObject({ ok: false, description: failure.message, + attributes: { "error.type": "server_error" } }); + }); + + test.each([ + { label: "failed JSON without details", body: JSON.stringify({ status: "failed" }), type: "application/json" }, + { label: "failed SSE without a response object", body: frame({ type: "response.failed" }), type: "text/event-stream" }, + { label: "error SSE without details", body: frame({}, "error"), type: "text/event-stream" }, + ])("$label still has a non-HTTP error classification", async ({ body, type }) => { + expect(await observe("responses", body, type)).toMatchObject({ ok: false, description: "Provider error", + attributes: { "error.type": "provider_error" } }); + }); + + test("a later completed event cannot erase an observed provider failure", async () => { + const body = frame({ type: "response.failed", response: { status: "failed", error: failure } }) + + frame({ type: "response.completed", response: { status: "completed", error: null, output, usage } }); + expect(await observe("responses", body)).toMatchObject({ ok: false, description: failure.message, + attributes: { "error.type": "server_error", "output.value": "Partial answer", "gen_ai.usage.total_tokens": 9 } }); + }); + + test.each(["application/json", "text/event-stream"])("provider errors use existing credential privacy for %s", async type => { + const credential = "SYNTHETIC_PROVIDER_CREDENTIAL_NOT_REAL"; + const response = { status: "failed", error: { code: credential, message: `Synthetic failure mentioning ${credential}` } }; + const body = type === "application/json" ? JSON.stringify(response) : frame({ type: "response.failed", response }); + const result = await observe("responses", body, type, 200, `Bearer ${credential}`); + expect(result.ok).toBe(false); + expect(result.description).toContain("Synthetic failure mentioning"); + expect(result.description).not.toContain(credential); + expect(result.attributes["error.type"]).toBeDefined(); + expect(JSON.stringify(result)).not.toContain(credential); + }); + + test.each([ + { label: "completed JSON", body: JSON.stringify({ status: "completed", error: null, output, usage }), type: "application/json", finish: "completed" }, + { label: "completed SSE", body: frame({ type: "response.completed", response: { status: "completed", error: null, output, usage } }), type: "text/event-stream", finish: "completed" }, + { label: "incomplete SSE", body: frame({ type: "response.incomplete", response: { status: "incomplete", error: null, output, usage, incomplete_details: { reason: "max_output_tokens" } } }), type: "text/event-stream", finish: "incomplete" }, + ])("$label remains successful", async ({ body, type, finish }) => { + const result = await observe("responses", body, type); + expect(result).toMatchObject({ ok: true, attributes: { "output.value": "Partial answer", "gen_ai.usage.total_tokens": 9, + "gen_ai.response.finish_reasons": [finish] } }); + expect(result.attributes["error.type"]).toBeUndefined(); + expect(result.description).toBeUndefined(); + }); + + test("chat finish reasons and error-like output are not provider failures", async () => { + const content = JSON.stringify({ error: failure, status: "failed" }); + const result = await observe("chat/completions", frame({ choices: [{ delta: { content }, finish_reason: "content_filter" }] })); + expect(result).toMatchObject({ ok: true, attributes: { "output.value": content, "gen_ai.response.finish_reasons": ["content_filter"] } }); + expect(result.attributes["error.type"]).toBeUndefined(); + }); + + test("HTTP failures keep their existing status classification", async () => { + expect(await observe("responses", JSON.stringify({ error: failure }), "application/json", 500)).toMatchObject({ + ok: false, description: failure.message, attributes: { "error.type": "500", "http.response.status_code": 500 }, + }); + }); + + test("transport errors take precedence over captured provider errors", () => { + const url = "https://api.openai.com/v1/responses", init = request("responses"); + const body = encoder.encode(frame({ type: "response.failed", response: { status: "failed", error: failure } })); + const result = build(match("POST", url)!, { method: "POST", url, headers: { "content-type": "application/json" }, body: encoder.encode(init.body as string) }, + { status: 200, headers: { "content-type": "text/event-stream" }, body, size: body.length, error: new TypeError("Synthetic transport failure") }); + expect(result).toMatchObject({ ok: false, description: "Synthetic transport failure", attributes: { "error.type": "TypeError" } }); + }); + + test("fetch rejection identity and behavior remain unchanged", async () => { + const failure = new TypeError("Synthetic transport failure"), spans: SpanResult[] = []; + const promise = Promise.reject(failure), fake = jest.fn(() => promise); + globalThis.fetch = fake; + hook = createFetchHook(4194304, span => spans.push(span), { + active: () => undefined, skip: () => false, run: (_parent, fn) => fn(), + }); + const actual = fetch("https://api.openai.com/v1/responses", request("responses")); + expect(actual).toBe(promise); + await expect(actual).rejects.toBe(failure); + expect(fake).toHaveBeenCalledTimes(1); + expect(spans).toHaveLength(1); + expect(spans[0]).toMatchObject({ ok: false, description: failure.message, attributes: { "error.type": "TypeError" } }); + }); +}); diff --git a/typescript/packages/traceai_auto/test/round5.test.ts b/typescript/packages/traceai_auto/test/round5.test.ts index 98ee668b8..588b2dd19 100644 --- a/typescript/packages/traceai_auto/test/round5.test.ts +++ b/typescript/packages/traceai_auto/test/round5.test.ts @@ -146,14 +146,14 @@ test.each(["clone", "released-reader"])("documented OTHER %s cancellation keeps expect(actual.spans[0].attributes["traceai.wire.incomplete"]).toBe(true); expect(actual.spans[0].attributes["output.value"]).toBeUndefined(); } finally { globalThis.fetch = original; } }); -test("documented OTHER HTTP-200 provider-failure stream preserves baseline bytes but retains HTTP-derived successful status", async () => { +test("documented OTHER HTTP-200 provider-failure stream preserves baseline bytes and reports the provider failure", async () => { const original = globalThis.fetch, text = 'data: {"type":"response.failed","response":{"status":"failed","error":{"message":"provider failed"}}}\n\n'; const baseline = await new Response(text).text(), spans: any[] = []; globalThis.fetch = () => Promise.resolve(new Response(text, { headers: { "content-type": "text/event-stream" } })); const hook = createFetchHook(4096, span => spans.push(span), { active: () => null, skip: () => false, run: (_p, work) => work() }); try { expect(await (await fetch("https://api.openai.com/v1/responses", { method: "POST", body: '{"model":"fixture"}' })).text()).toBe(baseline); - expect(spans).toHaveLength(1); expect(spans[0].ok).toBe(true); expect(spans[0].description).toBeUndefined(); + expect(spans).toHaveLength(1); expect(spans[0].ok).toBe(false); expect(spans[0].description).toBe("provider failed"); expect(spans[0].attributes["gen_ai.response.finish_reasons"]).toEqual(["failed"]); } finally { hook.remove(); globalThis.fetch = original; } }); diff --git a/typescript/packages/traceai_bedrock/CHANGELOG.md b/typescript/packages/traceai_bedrock/CHANGELOG.md index e62761548..69b534961 100644 --- a/typescript/packages/traceai_bedrock/CHANGELOG.md +++ b/typescript/packages/traceai_bedrock/CHANGELOG.md @@ -1,3 +1,8 @@ +## [0.1.4] - pending release + +### Fixed +- Traced send() keeps its options and callbacks, response streams are read only as your code reads them, the requested model id is recorded in full, and streamed tool arguments from Anthropic models are recorded, with a placeholder when the argument JSON is incomplete or malformed. + ## [0.1.1] - 2026-10-18 ### Fix - Declare `@aws-sdk/client-bedrock-runtime` `^3.693.0` as a peer dependency, like the other instrumentors declare their SDK. The package loads it at runtime, so a clean install without it failed with "Cannot find module". diff --git a/typescript/packages/traceai_bedrock/package.json b/typescript/packages/traceai_bedrock/package.json index cca0c9460..b00d64fb6 100644 --- a/typescript/packages/traceai_bedrock/package.json +++ b/typescript/packages/traceai_bedrock/package.json @@ -1,6 +1,6 @@ { "name": "@traceai/bedrock", - "version": "0.1.3", + "version": "0.1.4", "description": "OpenTelemetry instrumentation for AWS Bedrock", "main": "./dist/src/index.js", "module": "./dist/esm/index.js", diff --git a/typescript/packages/traceai_bedrock/src/__tests__/model-id.test.ts b/typescript/packages/traceai_bedrock/src/__tests__/model-id.test.ts new file mode 100644 index 000000000..e34545d66 --- /dev/null +++ b/typescript/packages/traceai_bedrock/src/__tests__/model-id.test.ts @@ -0,0 +1,106 @@ +import { afterAll, beforeEach, describe, expect, it, jest } from "@jest/globals"; +import { ConverseCommand, InvokeModelCommand } from "@aws-sdk/client-bedrock-runtime"; +import { SpanStatusCode } from "@opentelemetry/api"; +import { + BasicTracerProvider, + InMemorySpanExporter, + SimpleSpanProcessor, +} from "@opentelemetry/sdk-trace-base"; +import { BedrockInstrumentation } from "../instrumentation"; + +const modelIds = [ + "us.anthropic.claude-3-5-sonnet-20241022-v2:0", + "eu.anthropic.claude-3-5-sonnet-20241022-v2:0", + "apac.anthropic.claude-3-5-sonnet-20241022-v2:0", + "us.anthropic.claude-3-5-haiku-20241022-v1:0", + "anthropic.claude-3-5-sonnet-20241022-v2:0", + "arn:aws:bedrock:us-east-1:000000000000:inference-profile/us.anthropic.claude-3-5-sonnet-20241022-v2:0", + "arn:aws:bedrock:us-east-1:000000000000:application-inference-profile/000000000000", +]; + +const operations: [ + string, + (modelId: string) => ConverseCommand | InvokeModelCommand, + unknown, +][] = [ + [ + "bedrock.converse", + (modelId) => + new ConverseCommand({ + modelId, + messages: [{ role: "user", content: [{ text: "hello" }] }], + }), + { + output: { message: { role: "assistant", content: [{ text: "hello" }] } }, + stopReason: "end_turn", + usage: { inputTokens: 1, outputTokens: 1, totalTokens: 2 }, + metrics: { latencyMs: 1 }, + }, + ], + [ + "bedrock.invoke_model", + (modelId) => + new InvokeModelCommand({ + modelId, + contentType: "application/json", + body: JSON.stringify({ + anthropic_version: "bedrock-2023-05-31", + max_tokens: 10, + messages: [{ role: "user", content: [{ type: "text", text: "hello" }] }], + }), + }), + { + contentType: "application/json", + body: new TextEncoder().encode( + JSON.stringify({ + content: [{ type: "text", text: "hello" }], + usage: { input_tokens: 1, output_tokens: 1 }, + }), + ), + }, + ], +]; + +describe("T8-TS-CLOUD-004 Bedrock request model identity", () => { + const exporter = new InMemorySpanExporter(); + const provider = new BasicTracerProvider({ + spanProcessors: [new SimpleSpanProcessor(exporter)], + }); + + beforeEach(() => exporter.reset()); + afterAll(() => provider.shutdown()); + + describe.each(operations)("%s", (spanName, makeCommand, response) => { + it.each(modelIds)("retains the complete requested modelId: %s", async (modelId) => { + const originalSend = jest.fn(async (_command: unknown) => response); + // Patch only a fresh fake client; no AWS client, credentials, or network is used. + class BedrockRuntimeClient { + send(command: unknown): Promise { + return originalSend(command); + } + } + const instrumentation = new BedrockInstrumentation({ + tracerProvider: provider, + instrumentationConfig: { enabled: false }, + }); + instrumentation.manuallyInstrument({ + BedrockRuntimeClient, + } as unknown as Parameters[0]); + + const command = makeCommand(modelId); + const inputBefore = JSON.stringify(command.input); + const result = await new BedrockRuntimeClient().send(command); + + expect(originalSend).toHaveBeenCalledTimes(1); + expect(originalSend).toHaveBeenCalledWith(command); + expect(result).toBe(response); + expect(JSON.stringify(command.input)).toBe(inputBefore); + + const spans = exporter.getFinishedSpans(); + expect(spans).toHaveLength(1); + expect(spans[0].name).toBe(spanName); + expect(spans[0].status.code).toBe(SpanStatusCode.OK); + expect(spans[0].attributes["gen_ai.request.model"]).toBe(modelId); + }); + }); +}); diff --git a/typescript/packages/traceai_bedrock/src/__tests__/send-overloads.test.ts b/typescript/packages/traceai_bedrock/src/__tests__/send-overloads.test.ts new file mode 100644 index 000000000..00d8e17d4 --- /dev/null +++ b/typescript/packages/traceai_bedrock/src/__tests__/send-overloads.test.ts @@ -0,0 +1,236 @@ +import { describe, it, expect, jest } from "@jest/globals"; +import { + ConverseCommand, + InvokeModelCommand, + InvokeModelWithResponseStreamCommand, +} from "@aws-sdk/client-bedrock-runtime"; +import { + BasicTracerProvider, + InMemorySpanExporter, + SimpleSpanProcessor, +} from "@opentelemetry/sdk-trace-base"; +import { SpanStatusCode } from "@opentelemetry/api"; +import { BedrockInstrumentation } from "../instrumentation"; + +const modelId = "amazon.nova-pro-v1:0"; +const messages = [{ role: "user" as const, content: [{ text: "hello" }] }]; + +class UnrecognizedCommand {} + +const commands: [string, () => unknown][] = [ + ["InvokeModel", () => new InvokeModelCommand({ modelId, body: JSON.stringify({ messages }) })], + ["InvokeModelWithResponseStream", () => new InvokeModelWithResponseStreamCommand({ modelId, body: JSON.stringify({ messages }) })], + ["Converse", () => new ConverseCommand({ modelId, messages })], + ["unrecognized", () => new UnrecognizedCommand()], +]; + +type Callback = (this: unknown, error: unknown, response: unknown) => void; + +/** SDK-shaped double: callback sends return void; promise sends return a Promise. */ +function recordingClient({ + response = {}, + error, + synchronousCallback = false, + synchronousError, +}: { + response?: unknown; + error?: Error; + synchronousCallback?: boolean; + synchronousError?: Error; +} = {}) { + const calls: { receiver: unknown; args: unknown[] }[] = []; + const callbackReceiver = {}; + let pendingCallback: Callback | undefined; + const complete = () => pendingCallback?.call(callbackReceiver, error, response); + + class BedrockRuntimeClient { + send(...args: unknown[]): Promise | void { + calls.push({ receiver: this, args }); + if (synchronousError) throw synchronousError; + const callback = typeof args[1] === "function" ? args[1] : args[2]; + if (typeof callback === "function") { + pendingCallback = callback as Callback; + if (synchronousCallback) complete(); + return; + } + return error ? Promise.reject(error) : Promise.resolve(response); + } + } + + const exporter = new InMemorySpanExporter(); + const provider = new BasicTracerProvider({ + spanProcessors: [new SimpleSpanProcessor(exporter)], + }); + const instrumentation = new BedrockInstrumentation({ + instrumentationConfig: { enabled: false }, + tracerProvider: provider, + }); + instrumentation.manuallyInstrument({ + BedrockRuntimeClient, + } as unknown as Parameters[0]); + + return { client: new BedrockRuntimeClient(), calls, complete, callbackReceiver, exporter }; +} + +function abortedOptions() { + const controller = new AbortController(); + controller.abort(); + return { abortSignal: controller.signal, requestTimeout: 1 }; +} + +describe("T8-TS-CLOUD-001: Bedrock send overloads", () => { + describe.each(commands)("%s", (name, makeCommand) => { + it("forwards promise options and the original receiver", async () => { + const response = {}; + const { client, calls } = recordingClient({ response }); + const command = makeCommand(); + const options = abortedOptions(); + + const result = client.send(command, options); + expect(result).toBeInstanceOf(Promise); + await expect(result).resolves.toBe(response); + expect(calls).toHaveLength(1); + expect(calls[0].receiver).toBe(client); + expect(calls[0].args).toHaveLength(2); + expect(calls[0].args[0]).toBe(command); + expect(calls[0].args[1]).toBe(options); + }); + + it("preserves promise rejections", async () => { + const error = new Error("offline failure"); + const { client, calls } = recordingClient({ error }); + const options = abortedOptions(); + await expect(client.send(makeCommand(), options)).rejects.toBe(error); + expect(calls[0].args[1]).toBe(options); + }); + + it.each(["callback", "options + callback", "undefined + callback"])( + "preserves the %s overload, return value and callback receiver", + async (overload) => { + const response = {}; + const { client, calls, complete, callbackReceiver, exporter } = recordingClient({ response }); + const command = makeCommand(); + const options = abortedOptions(); + let receiver: unknown; + const callback = jest.fn(function (this: unknown, _error: unknown, _response: unknown) { + receiver = this; + }); + const args = overload === "callback" + ? [command, callback] + : [command, overload === "options + callback" ? options : undefined, callback]; + + const result = client.send(...args); + // Drain the incorrect promise on the red revision without leaving work behind. + if (result instanceof Promise) await result; + expect(callback).not.toHaveBeenCalled(); + complete(); + + expect(result).toBeUndefined(); + expect(callback).toHaveBeenCalledTimes(1); + expect(callback).toHaveBeenCalledWith(undefined, response); + expect(receiver).toBe(callbackReceiver); + expect(calls).toHaveLength(1); + expect(calls[0].receiver).toBe(client); + expect(calls[0].args).toHaveLength(args.length); + expect(calls[0].args[0]).toBe(command); + if (args.length === 3) expect(calls[0].args[1]).toBe(args[1]); + expect(exporter.getFinishedSpans()).toHaveLength(name === "unrecognized" ? 0 : 1); + }, + ); + + it.each([false, true])("preserves callback errors (options: %s)", async (withOptions) => { + const error = new Error("offline callback failure"); + const response = {}; + const { client, complete, exporter } = recordingClient({ error, response }); + const callback = jest.fn(); + const command = makeCommand(); + const result = withOptions + ? client.send(command, abortedOptions(), callback) + : client.send(command, callback); + if (result instanceof Promise) await result.catch(() => undefined); + complete(); + + expect(result).toBeUndefined(); + expect(callback).toHaveBeenCalledTimes(1); + expect(callback).toHaveBeenCalledWith(error, response); + if (name !== "unrecognized") { + const spans = exporter.getFinishedSpans(); + expect(spans).toHaveLength(1); + expect(spans[0].status).toEqual({ code: SpanStatusCode.ERROR, message: error.message }); + } + }); + + it("preserves synchronous send errors without invoking the callback", () => { + const error = new Error("invalid offline options"); + const { client, calls } = recordingClient({ synchronousError: error }); + const callback = jest.fn(); + const options = abortedOptions(); + expect(() => client.send(makeCommand(), options, callback)).toThrow(error); + expect(callback).not.toHaveBeenCalled(); + expect(calls[0].args[1]).toBe(options); + expect(calls[0].args).toHaveLength(3); + }); + + it("does not swallow, retry or record an application callback exception", () => { + const error = new Error("application callback failure"); + const { client, exporter } = recordingClient({ synchronousCallback: true }); + const callback = jest.fn(() => { throw error; }); + + expect(() => client.send(makeCommand(), callback)).toThrow(error); + expect(callback).toHaveBeenCalledTimes(1); + const spans = exporter.getFinishedSpans(); + expect(spans.some((span) => span.status.message === error.message)).toBe(false); + }); + }); + + it("passes every trailing argument and the exact return value through for unknown commands", () => { + const result = {}; + const calls: unknown[][] = []; + class BedrockRuntimeClient { + send(...args: unknown[]) { + calls.push(args); + return result; + } + } + const instrumentation = new BedrockInstrumentation({ instrumentationConfig: { enabled: false } }); + instrumentation.manuallyInstrument({ + BedrockRuntimeClient, + } as unknown as Parameters[0]); + const client = new BedrockRuntimeClient(); + const args = [new UnrecognizedCommand(), abortedOptions(), undefined, "extra argument"]; + expect(client.send(...args)).toBe(result); + expect(calls).toEqual([args]); + }); + + it.each(["promise", "callback"])("preserves stream chunks with %s send", async (overload) => { + const chunks = [ + { chunk: { bytes: new TextEncoder().encode('{"contentBlockDelta":{"delta":{"text":"hello"}}}') } }, + { chunk: { bytes: new TextEncoder().encode('{"metadata":{"usage":{"inputTokens":1,"outputTokens":1}}}') } }, + ]; + const response = { + body: (async function* () { yield* chunks; })(), + contentType: "application/json", + }; + const { client, calls, complete } = recordingClient({ response }); + const command = new InvokeModelWithResponseStreamCommand({ modelId, body: JSON.stringify({ messages }) }); + const options = abortedOptions(); + let output: unknown; + if (overload === "callback") { + const callback = jest.fn((_error: unknown, value: unknown) => { output = value; }); + const result = client.send(command, options, callback); + if (result instanceof Promise) await result; + complete(); + expect(result).toBeUndefined(); + expect(callback).toHaveBeenCalledTimes(1); + } else { + output = await client.send(command, options); + } + + const streamResponse = output as { body: AsyncIterable; contentType: string }; + const received: unknown[] = []; + for await (const chunk of streamResponse.body) received.push(chunk); + expect(received).toEqual(chunks); + expect(streamResponse.contentType).toBe(response.contentType); + expect(calls[0].args[1]).toBe(options); + }); +}); diff --git a/typescript/packages/traceai_bedrock/src/__tests__/stream-finalization.test.ts b/typescript/packages/traceai_bedrock/src/__tests__/stream-finalization.test.ts new file mode 100644 index 000000000..321142d5b --- /dev/null +++ b/typescript/packages/traceai_bedrock/src/__tests__/stream-finalization.test.ts @@ -0,0 +1,68 @@ +import { afterEach, describe, expect, it, jest } from "@jest/globals"; +import { Span, SpanStatusCode } from "@opentelemetry/api"; +import { BasicTracerProvider, InMemorySpanExporter, SimpleSpanProcessor } from "@opentelemetry/sdk-trace-base"; +import { LLMSystem } from "@traceai/fi-semantic-conventions"; + +const OriginalRegistry = globalThis.FinalizationRegistry; +afterEach(() => { globalThis.FinalizationRegistry = OriginalRegistry; }); + +function setup() { + let cleanup!: (held: unknown) => void; + const registrations: { target: object; held: unknown; token?: object }[] = []; + const unregister = jest.fn((_token: object) => true); + class TestRegistry { + constructor(callback: (held: unknown) => void) { cleanup = callback; } + register(target: object, held: unknown, token?: object) { registrations.push({ target, held, token }); } + unregister = unregister; + } + globalThis.FinalizationRegistry = TestRegistry as unknown as typeof FinalizationRegistry; + let observe!: typeof import("../attributes/invoke-model-streaming-response-attributes").observeBedrockStream; + jest.isolateModules(() => { + observe = require("../attributes/invoke-model-streaming-response-attributes").observeBedrockStream; + }); + const exporter = new InMemorySpanExporter(); + const provider = new BasicTracerProvider({ spanProcessors: [new SimpleSpanProcessor(exporter)] }); + const span: Span = provider.getTracer("offline").startSpan("stream"); + const next = jest.fn(async () => ({ done: true as const, value: undefined })); + const source = { next, [Symbol.asyncIterator]() { return this; } }; + const body = observe({ stream: source, span, modelType: LLMSystem.ANTHROPIC, + onEnd: () => { span.setStatus({ code: SpanStatusCode.OK }); span.end(); }, + onError: () => { span.setStatus({ code: SpanStatusCode.ERROR }); span.end(); }, + }); + return { body, exporter, next, registrations, unregister, cleanup: (held: unknown) => cleanup(held) }; +} + +describe("Bedrock unconsumed stream finalization", () => { + it("registers best-effort cleanup without pulling or creating timers", () => { + const schedule = jest.spyOn(globalThis, "setTimeout"); + try { + const run = setup(); + expect(run.next).not.toHaveBeenCalled(); + expect(schedule).not.toHaveBeenCalled(); + expect(run.exporter.getFinishedSpans()).toHaveLength(0); + expect(run.registrations).toHaveLength(1); + // Deterministic delivery of the GC notification, not a timing assertion. + run.cleanup(run.registrations[0].held); + const spans = run.exporter.getFinishedSpans(); + expect(spans).toHaveLength(1); + expect(spans[0].status.code).toBe(SpanStatusCode.UNSET); + expect(spans[0].attributes["fi.stream.consumed"]).toBe(false); + expect(run.next).not.toHaveBeenCalled(); + run.cleanup(run.registrations[0].held); + expect(run.exporter.getFinishedSpans()).toHaveLength(1); + } finally { schedule.mockRestore(); } + }); + + it("disarms unconsumed cleanup when the caller pulls", async () => { + const run = setup(); + expect(run.registrations).toHaveLength(1); + await run.body[Symbol.asyncIterator]().next(); + expect(run.unregister).toHaveBeenCalled(); + // A queued notification must not turn a consumed stream into UNSET/unconsumed. + run.cleanup(run.registrations[0].held); + const spans = run.exporter.getFinishedSpans(); + expect(spans).toHaveLength(1); + expect(spans[0].status.code).toBe(SpanStatusCode.OK); + expect(spans[0].attributes["fi.stream.consumed"]).not.toBe(false); + }); +}); diff --git a/typescript/packages/traceai_bedrock/src/__tests__/streaming-errors.test.ts b/typescript/packages/traceai_bedrock/src/__tests__/streaming-errors.test.ts new file mode 100644 index 000000000..d8b74b21d --- /dev/null +++ b/typescript/packages/traceai_bedrock/src/__tests__/streaming-errors.test.ts @@ -0,0 +1,160 @@ +import { beforeEach, describe, expect, it } from "@jest/globals"; +import { InvokeModelWithResponseStreamCommand } from "@aws-sdk/client-bedrock-runtime"; +import { SpanStatusCode } from "@opentelemetry/api"; +import { + BasicTracerProvider, + InMemorySpanExporter, + SimpleSpanProcessor, +} from "@opentelemetry/sdk-trace-base"; +import { BedrockInstrumentation } from "../instrumentation"; +import { safelySplitStream } from "../attributes/invoke-model-streaming-response-attributes"; + +function nextTurn(): Promise { + return new Promise((resolve) => setImmediate(resolve)); +} + +async function collect(stream: AsyncIterable): Promise { + const chunks: unknown[] = []; + for await (const chunk of stream) { + chunks.push(chunk); + } + return chunks; +} + +function command(): InvokeModelWithResponseStreamCommand { + return new InvokeModelWithResponseStreamCommand({ + modelId: "anthropic.claude-3-haiku-20240307-v1:0", + contentType: "application/json", + body: JSON.stringify({ + anthropic_version: "bedrock-2023-05-31", + max_tokens: 8, + messages: [{ role: "user", content: [{ type: "text", text: "hello" }] }], + }), + }); +} + +const chunk = { chunk: { bytes: new TextEncoder().encode("{}") } }; + +describe("T8-TS-CLOUD-002: Bedrock streaming errors", () => { + const exporter = new InMemorySpanExporter(); + const provider = new BasicTracerProvider({ + spanProcessors: [new SimpleSpanProcessor(exporter)], + }); + + beforeEach(() => { + exporter.reset(); + }); + + function clientFor(body: AsyncIterable) { + // Fake only the SDK transport; exercise the real send instrumentation. + class BedrockRuntimeClient { + async send(_command: unknown) { + return { body, contentType: "application/json" }; + } + } + const instrumentation = new BedrockInstrumentation({ + instrumentationConfig: { enabled: false }, + tracerProvider: provider, + }); + instrumentation.manuallyInstrument({ + BedrockRuntimeClient, + } as unknown as Parameters[0]); + return new BedrockRuntimeClient(); + } + + it.each([false, true])( + "delayed iteration catches the original failure (yield chunk: %s)", + async (yieldChunk) => { + const failure = new Error("fake-stream-failure"); + let sourceFailed = false; + const body = (async function* () { + if (yieldChunk) { + yield chunk; + } + await nextTurn(); + sourceFailed = true; + throw failure; + })(); + + const response = await clientFor(body).send(command()); + // No iterator or error listener is attached to response.body during this + // delay. Before the fix, destroy(failure) emits an unhandled error here. + await nextTurn(); + await nextTurn(); + // Demand-only observation must not start the source during the delay. + expect(sourceFailed).toBe(false); + expect(exporter.getFinishedSpans()).toHaveLength(0); + + // No unhandled error/rejection may escape during the delay, and the + // original failure must still surface on the caller's pull. + await expect(collect(response.body)).rejects.toBe(failure); + expect(sourceFailed).toBe(true); + expect(exporter.getFinishedSpans()).toHaveLength(1); + expect(exporter.getFinishedSpans()[0].status.code).toBe(SpanStatusCode.ERROR); + expect(exporter.getFinishedSpans()[0].status.message).toBe(failure.message); + expect(exporter.getFinishedSpans()[0].events).toEqual(expect.arrayContaining([ + expect.objectContaining({ name: "exception" }), + ])); + }, + ); + + it("immediate iteration preserves the chunk and original failure", async () => { + const failure = new Error("fake-stream-failure"); + const body = (async function* () { + yield chunk; + await nextTurn(); + throw failure; + })(); + const response = await clientFor(body).send(command()); + const received: unknown[] = []; + + await expect( + (async () => { + for await (const event of response.body) { + received.push(event); + } + })(), + ).rejects.toBe(failure); + expect(received).toHaveLength(1); + expect(received[0]).toBe(chunk); + }); + + it("delayed successful iteration preserves chunks and response fields", async () => { + const secondChunk = { chunk: { bytes: new TextEncoder().encode("{}") } }; + const body = (async function* () { + yield chunk; + yield secondChunk; + })(); + const response = await clientFor(body).send(command()); + await nextTurn(); + + const received = await collect(response.body); + expect(received).toHaveLength(2); + expect(received[0]).toBe(chunk); + expect(received[1]).toBe(secondChunk); + expect(response.contentType).toBe("application/json"); + expect(exporter.getFinishedSpans()).toHaveLength(1); + expect(exporter.getFinishedSpans()[0].status.code).toBe(SpanStatusCode.OK); + }); + + it("retains the failure when both tee consumers start late", async () => { + const failure = new Error("fake-stream-failure"); + const body = (async function* () { + yield chunk; + await nextTurn(); + throw failure; + })(); + const { instrumentationStream, userStream } = safelySplitStream({ + originalStream: body, + }); + await nextTurn(); + await nextTurn(); + + expect(instrumentationStream).toBeDefined(); + if (!instrumentationStream) { + throw new Error("Expected an instrumentation stream"); + } + await expect(collect(instrumentationStream)).rejects.toBe(failure); + await expect(collect(userStream)).rejects.toBe(failure); + }); +}); diff --git a/typescript/packages/traceai_bedrock/src/__tests__/streaming-semantics.test.ts b/typescript/packages/traceai_bedrock/src/__tests__/streaming-semantics.test.ts new file mode 100644 index 000000000..7b284aaeb --- /dev/null +++ b/typescript/packages/traceai_bedrock/src/__tests__/streaming-semantics.test.ts @@ -0,0 +1,123 @@ +import { afterEach, describe, expect, it, jest } from "@jest/globals"; +import { InvokeModelWithResponseStreamCommand } from "@aws-sdk/client-bedrock-runtime"; +import { SpanStatusCode } from "@opentelemetry/api"; +import { BasicTracerProvider, InMemorySpanExporter, SimpleSpanProcessor } from "@opentelemetry/sdk-trace-base"; +import { SemanticConventions } from "@traceai/fi-semantic-conventions"; +import { BedrockInstrumentation } from "../instrumentation"; + +const chunk = (data: unknown) => ({ chunk: { bytes: new TextEncoder().encode(JSON.stringify(data)) } }); +const partial = chunk({ type: "content_block_delta", delta: { text: "partial" } }); +const command = () => new InvokeModelWithResponseStreamCommand({ + modelId: "anthropic.claude-3-haiku-20240307-v1:0", + body: JSON.stringify({ messages: [{ role: "user", content: "hello" }] }), +}); + +function harness(send: () => Promise<{ body: AsyncIterable }>) { + const exporter = new InMemorySpanExporter(); + const provider = new BasicTracerProvider({ spanProcessors: [new SimpleSpanProcessor(exporter)] }); + class BedrockRuntimeClient { send(_command: unknown) { return send(); } } + const instrumentation = new BedrockInstrumentation({ + instrumentationConfig: { enabled: false }, tracerProvider: provider, + }); + instrumentation.manuallyInstrument({ BedrockRuntimeClient } as unknown as Parameters[0]); + return { exporter, client: new BedrockRuntimeClient() }; +} + +afterEach(() => { jest.useRealTimers(); }); + +describe("Bedrock demand-only span semantics", () => { + it.each(["validation", "throttling", "auth"])("ends immediately when send rejects: %s", async (kind) => { + const failure = new Error(`synthetic ${kind} failure`); + const { client, exporter } = harness(async () => { throw failure; }); + await expect(client.send(command())).rejects.toBe(failure); + const spans = exporter.getFinishedSpans(); + expect(spans).toHaveLength(1); + expect(spans[0].status).toEqual({ code: SpanStatusCode.ERROR, message: failure.message }); + expect(spans[0].events).toEqual(expect.arrayContaining([expect.objectContaining({ name: "exception" })])); + }); + + it("ends ERROR on the surfacing pull, retaining partial output without unhandled rejection", async () => { + const failure = new Error("synthetic pull failure"); + const unhandled: unknown[] = []; + const listener = (error: unknown) => { unhandled.push(error); }; + process.on("unhandledRejection", listener); + let pulls = 0; + async function* body() { pulls++; yield partial; pulls++; throw failure; } + const { client, exporter } = harness(async () => ({ body: body() })); + try { + const response = await client.send(command()); + await new Promise((resolve) => setImmediate(resolve)); + expect(pulls).toBe(0); + expect(exporter.getFinishedSpans()).toHaveLength(0); + const iterator = response.body[Symbol.asyncIterator](); + expect((await iterator.next()).value).toBe(partial); + expect(exporter.getFinishedSpans()).toHaveLength(0); + await expect(iterator.next()).rejects.toBe(failure); + await new Promise((resolve) => setImmediate(resolve)); + expect(unhandled).toEqual([]); + expect(pulls).toBe(2); + const spans = exporter.getFinishedSpans(); + expect(spans).toHaveLength(1); + expect(spans[0].status).toEqual({ code: SpanStatusCode.ERROR, message: failure.message }); + expect(spans[0].events).toEqual(expect.arrayContaining([expect.objectContaining({ name: "exception", attributes: expect.objectContaining({ "exception.message": failure.message }) })])); + expect(JSON.parse(spans[0].attributes[SemanticConventions.OUTPUT_VALUE] as string).text).toBe("partial"); + } finally { process.removeListener("unhandledRejection", listener); } + }); + + it.each(["break", "return", "throw"])("ends UNSET on caller %s and does not pull the remainder", async (control) => { + let pulls = 0; + let closed = false; + async function* body() { + try { pulls++; yield partial; pulls++; yield partial; } + finally { closed = true; } + } + const { client, exporter } = harness(async () => ({ body: body() })); + const response = await client.send(command()); + if (control === "break") { + for await (const event of response.body) { expect(event).toBe(partial); break; } + } else { + const iterator = response.body[Symbol.asyncIterator](); + await iterator.next(); + if (control === "return") await iterator.return?.(); + else { + const cancellation = new Error("caller cancelled"); + await expect(iterator.throw?.(cancellation)).rejects.toBe(cancellation); + } + } + expect(closed).toBe(true); + expect(pulls).toBe(1); + const spans = exporter.getFinishedSpans(); + expect(spans).toHaveLength(1); + expect(spans[0].status.code).toBe(SpanStatusCode.UNSET); + expect(spans[0].events.map(event => event.name)).toEqual(["stream.cancelled"]); + expect(JSON.parse(spans[0].attributes[SemanticConventions.OUTPUT_VALUE] as string).text).toBe("partial"); + }); + + it("measures TTFT at the first caller-pulled chunk and exports usage and finish reason on completion", async () => { + jest.useFakeTimers({ doNotFake: ["nextTick", "setImmediate"] }); + const events = [ + chunk({ type: "message_start", message: { usage: { input_tokens: 3 } } }), + partial, + chunk({ type: "message_delta", delta: { stop_reason: "end_turn" }, usage: { output_tokens: 2 } }), + ]; + let pulls = 0; + async function* body() { for (const event of events) { pulls++; yield event; } } + const { client, exporter } = harness(async () => ({ body: body() })); + const response = await client.send(command()); + jest.advanceTimersByTime(2000); + expect(pulls).toBe(0); + const iterator = response.body[Symbol.asyncIterator](); + expect((await iterator.next()).value).toBe(events[0]); + jest.advanceTimersByTime(3000); + expect((await iterator.next()).value).toBe(events[1]); + expect((await iterator.next()).value).toBe(events[2]); + expect((await iterator.next()).done).toBe(true); + const spans = exporter.getFinishedSpans(); + expect(spans).toHaveLength(1); + expect(spans[0].status.code).toBe(SpanStatusCode.OK); + expect(spans[0].attributes[SemanticConventions.GEN_AI_SERVER_TIME_TO_FIRST_TOKEN]).toBe(2000); + expect(spans[0].attributes[SemanticConventions.GEN_AI_RESPONSE_FINISH_REASONS]).toEqual(["end_turn"]); + expect(spans[0].attributes[SemanticConventions.LLM_TOKEN_COUNT_PROMPT]).toBe(3); + expect(spans[0].attributes[SemanticConventions.LLM_TOKEN_COUNT_COMPLETION]).toBe(2); + }); +}); diff --git a/typescript/packages/traceai_bedrock/src/__tests__/streaming-tool-arguments.test.ts b/typescript/packages/traceai_bedrock/src/__tests__/streaming-tool-arguments.test.ts new file mode 100644 index 000000000..0749f5f93 --- /dev/null +++ b/typescript/packages/traceai_bedrock/src/__tests__/streaming-tool-arguments.test.ts @@ -0,0 +1,237 @@ +import { describe, it, expect } from "@jest/globals"; +import { InvokeModelWithResponseStreamCommand } from "@aws-sdk/client-bedrock-runtime"; +import { SpanStatusCode } from "@opentelemetry/api"; +import { + BasicTracerProvider, + InMemorySpanExporter, + ReadableSpan, + SimpleSpanProcessor, +} from "@opentelemetry/sdk-trace-base"; +import { SemanticConventions } from "@traceai/fi-semantic-conventions"; +import { BedrockInstrumentation } from "../instrumentation"; + +function toolStart( + index: number, + id = "toolu_demo", + name = "weather", + input: Record = {}, +) { + return { + type: "content_block_start", + index, + content_block: { type: "tool_use", id, name, input }, + }; +} + +function jsonDelta(index: number, partial_json: string) { + return { + type: "content_block_delta", + index, + delta: { type: "input_json_delta", partial_json }, + }; +} + +function blockStop(index: number) { + return { type: "content_block_stop", index }; +} + +/** Exercise the real send wrapper and stream splitter without an AWS request. */ +async function captureStream( + events: Record[], + streamError?: Error, +) { + const chunks = events.map((event) => + Object.freeze({ + chunk: Object.freeze({ + bytes: new TextEncoder().encode(JSON.stringify(event)), + }), + }), + ); + const originalBytes = chunks.map(({ chunk }) => Array.from(chunk.bytes)); + async function* body() { + for (const chunk of chunks) { + yield chunk; + // Let both split streams consume each event before a possible error. + await new Promise((resolve) => setImmediate(resolve)); + } + if (streamError) throw streamError; + } + class BedrockRuntimeClient { + async send(_command: unknown) { + return { body: body(), contentType: "application/json" }; + } + } + + const exporter = new InMemorySpanExporter(); + let finish!: () => void; + const finished = new Promise((resolve) => { finish = resolve; }); + class NotifyingSpanProcessor extends SimpleSpanProcessor { + onEnd(span: ReadableSpan): void { + super.onEnd(span); + finish(); + } + } + const provider = new BasicTracerProvider({ + spanProcessors: [new NotifyingSpanProcessor(exporter)], + }); + const instrumentation = new BedrockInstrumentation({ + tracerProvider: provider, + instrumentationConfig: { enabled: false }, + }); + instrumentation.manuallyInstrument({ + BedrockRuntimeClient, + } as unknown as Parameters[0]); + + try { + const response = await new BedrockRuntimeClient().send( + new InvokeModelWithResponseStreamCommand({ + modelId: "anthropic.claude-3-haiku-20240307-v1:0", + body: JSON.stringify({ + messages: [{ role: "user", content: "What is the weather?" }], + }), + }), + ); + const received: unknown[] = []; + let error: unknown; + try { + for await (const chunk of response.body) received.push(chunk); + } catch (caught) { + error = caught; + } + await finished; + await provider.forceFlush(); + + expect(response.contentType).toBe("application/json"); + expect(received).toHaveLength(chunks.length); + received.forEach((chunk, index) => expect(chunk).toBe(chunks[index])); + expect(chunks.map(({ chunk }) => Array.from(chunk.bytes))).toEqual(originalBytes); + expect(error).toBe(streamError); + const spans = exporter.getFinishedSpans(); + expect(spans).toHaveLength(1); + return { span: spans[0], error }; + } finally { + instrumentation.disable(); + await provider.shutdown(); + } +} + +function outputMessage(span: ReadableSpan) { + return JSON.parse(span.attributes[SemanticConventions.LLM_OUTPUT_MESSAGES] as string)[0]; +} + +function outputValue(span: ReadableSpan) { + return JSON.parse(span.attributes[SemanticConventions.OUTPUT_VALUE] as string); +} + +function toolCall(id: string, name: string, args: string) { + return { id, function: { name, arguments: args } }; +} + +describe("T8-TS-CLOUD-005 Bedrock streamed tool arguments", () => { + it("exports JSON deltas from the instrumented response", async () => { + const args = JSON.stringify({ city: "Paris" }); + const { span } = await captureStream([ + toolStart(0), + jsonDelta(0, args), + blockStop(0), + ]); + + expect(outputMessage(span)).toEqual({ + role: "assistant", + tool_calls: [toolCall("toolu_demo", "weather", args)], + }); + expect(outputValue(span).tool_calls).toEqual([ + { id: "toolu_demo", name: "weather", input: { city: "Paris" } }, + ]); + expect(span.status.code).toBe(SpanStatusCode.OK); + }); + + it("joins fragments by block index without losing text or usage", async () => { + const args = JSON.stringify({ city: 'Par"is', units: "°C" }); + const split = args.indexOf("\\") + 1; + const { span } = await captureStream([ + { type: "message_start", message: { usage: { input_tokens: 5 } } }, + { type: "content_block_start", index: 0, content_block: { type: "text", text: "" } }, + { type: "content_block_delta", index: 0, delta: { type: "text_delta", text: "Checking " } }, + toolStart(1), + toolStart(7, "toolu_time", "time"), + jsonDelta(99, '{"ignored":true}'), + jsonDelta(0, '{"ignored":true}'), + jsonDelta(1, args.slice(0, split)), + jsonDelta(7, '{"zone":'), + jsonDelta(1, args.slice(split)), + jsonDelta(7, '"UTC"}'), + blockStop(7), + blockStop(1), + { type: "content_block_delta", index: 0, delta: { type: "text_delta", text: "weather." } }, + blockStop(0), + { type: "message_delta", usage: { output_tokens: 9 } }, + ]); + + expect(outputMessage(span)).toEqual({ + role: "assistant", + content: "Checking weather.", + tool_calls: [ + toolCall("toolu_demo", "weather", args), + toolCall("toolu_time", "time", '{"zone":"UTC"}'), + ], + }); + expect(outputValue(span).tool_calls.map((tool: { input: unknown }) => tool.input)).toEqual([ + { city: 'Par"is', units: "°C" }, + { zone: "UTC" }, + ]); + expect(span.attributes[SemanticConventions.LLM_TOKEN_COUNT_PROMPT]).toBe(5); + expect(span.attributes[SemanticConventions.LLM_TOKEN_COUNT_COMPLETION]).toBe(9); + }); + + it("preserves initial tool input when no JSON delta arrives", async () => { + const { span } = await captureStream([ + toolStart(0, "toolu_demo", "weather", { city: "Paris" }), + blockStop(0), + ]); + expect(outputMessage(span).tool_calls).toEqual([ + toolCall("toolu_demo", "weather", '{"city":"Paris"}'), + ]); + expect(outputValue(span).tool_calls[0].input).toEqual({ city: "Paris" }); + }); + + it.each([ + { label: "complete JSON without a block stop", args: '{"city":"Paris"}', input: { city: "Paris" }, stop: false, incomplete: false }, + { label: "partial JSON without a block stop", args: '{"city":"Par', input: "", stop: false, incomplete: true }, + { label: "malformed JSON at a block stop", args: '{"city":oops}', input: "", stop: true, incomplete: true }, + ])("preserves $label", async ({ args, input, stop, incomplete }) => { + const { span } = await captureStream([ + toolStart(0), + jsonDelta(0, args), + ...(stop ? [blockStop(0)] : []), + ]); + expect(outputMessage(span).tool_calls).toEqual([ + toolCall("toolu_demo", "weather", incomplete ? "" : args), + ]); + expect(outputValue(span).tool_calls[0].input).toEqual(input); + expect(span.status.code).toBe(SpanStatusCode.OK); + if (incomplete) { + expect(span.attributes["fi.tool_call.arguments_incomplete"]).toBe(true); + expect(span.attributes["fi.tool_call.arguments_received_length"]).toBe(args.length); + expect(JSON.stringify(span.attributes)).not.toContain(JSON.stringify(args).slice(1, -1)); + } + }); + + it("retains partial arguments on stream failure and preserves the application error", async () => { + const failure = new Error("offline stream interrupted"); + const args = '{"city":"Par'; + const { span, error } = await captureStream([ + toolStart(0), + jsonDelta(0, args), + ], failure); + + expect(error).toBe(failure); + expect(span.status.code).toBe(SpanStatusCode.ERROR); + expect(outputMessage(span).tool_calls).toEqual([ + toolCall("toolu_demo", "weather", ""), + ]); + expect(outputValue(span).tool_calls[0].input).toBe(""); + expect(span.attributes["fi.tool_call.arguments_incomplete"]).toBe(true); + expect(span.attributes["fi.tool_call.arguments_received_length"]).toBe(args.length); + }); +}); diff --git a/typescript/packages/traceai_bedrock/src/__tests__/streaming.test.ts b/typescript/packages/traceai_bedrock/src/__tests__/streaming.test.ts new file mode 100644 index 000000000..322c888a4 --- /dev/null +++ b/typescript/packages/traceai_bedrock/src/__tests__/streaming.test.ts @@ -0,0 +1,237 @@ +import { beforeEach, describe, expect, it, jest } from "@jest/globals"; +import { SpanStatusCode } from "@opentelemetry/api"; +import { + BasicTracerProvider, + InMemorySpanExporter, + SimpleSpanProcessor, +} from "@opentelemetry/sdk-trace-base"; +import { InvokeModelWithResponseStreamCommand } from "@aws-sdk/client-bedrock-runtime"; +import { SemanticConventions } from "@traceai/fi-semantic-conventions"; +import { BedrockInstrumentation } from "../instrumentation"; + +function chunk(data: Record) { + return { chunk: { bytes: new TextEncoder().encode(JSON.stringify(data)) } }; +} + +const nextTurn = () => new Promise((resolve) => setImmediate(resolve)); + +describe("T8-TS-CLOUD-003: Bedrock streaming lifecycle", () => { + const exporter = new InMemorySpanExporter(); + const provider = new BasicTracerProvider({ + spanProcessors: [new SimpleSpanProcessor(exporter)], + }); + + beforeEach(() => exporter.reset()); + + async function responseFor(body: AsyncIterable) { + const response = { body, requestId: "offline-response" }; + class BedrockRuntimeClient { + send(_command: unknown) { + return Promise.resolve(response); + } + } + const instrumentation = new BedrockInstrumentation({ + instrumentationConfig: { enabled: false }, + tracerProvider: provider, + }); + instrumentation.manuallyInstrument({ + BedrockRuntimeClient, + } as unknown as Parameters[0]); + return new BedrockRuntimeClient().send( + new InvokeModelWithResponseStreamCommand({ + modelId: "amazon.titan-text-express-v1", + body: JSON.stringify({ inputText: "offline prompt" }), + }), + ); + } + + it("does not pull before consumption or ahead of a slow consumer", async () => { + const event = chunk({ outputText: "part", tokenCount: 1 }); + let pulled = 0; + async function* body() { + for (let i = 0; i < 100; i++) { + pulled++; + yield event; + } + } + const response = await responseFor(body()); + const iterator = response.body[Symbol.asyncIterator](); + try { + await nextTurn(); + expect(pulled).toBe(0); + expect(exporter.getFinishedSpans()).toHaveLength(0); + expect((await iterator.next()).value).toBe(event); + await nextTurn(); + expect(pulled).toBe(1); + expect(exporter.getFinishedSpans()).toHaveLength(0); + expect((await iterator.next()).value).toBe(event); + await nextTurn(); + expect(pulled).toBe(2); + } finally { + await iterator.return?.(); + } + }); + + it("closes upstream on break and never drains the gated remainder", async () => { + const event = chunk({ outputText: "first", tokenCount: 1 }); + let release!: () => void; + const gate = new Promise((resolve) => { release = resolve; }); + let pulled = 0; + let closed = false; + async function* body() { + try { + pulled++; + yield event; + await gate; + for (let i = 1; i < 64; i++) { + pulled++; + yield event; + } + } finally { + closed = true; + } + } + const response = await responseFor(body()); + let closedOnBreak = false; + try { + for await (const received of response.body) { + expect(received).toBe(event); + break; + } + closedOnBreak = closed; + } finally { + release(); + await nextTurn(); + } + expect(closedOnBreak).toBe(true); + expect(pulled).toBe(1); + const spans = exporter.getFinishedSpans(); + expect(spans).toHaveLength(1); + // Cancellation must end the span without claiming a complete generation. + expect(spans[0].status.code).toBe(SpanStatusCode.UNSET); + expect(JSON.parse(spans[0].attributes[SemanticConventions.OUTPUT_VALUE] as string).text) + .toBe("first"); + }); + + it("forwards next, handled throw, and return with their values and receiver", async () => { + const first = { done: false as const, value: chunk({ outputText: "first" }) }; + const recovered = { done: false as const, value: chunk({ outputText: "recovered" }) }; + const returned = { done: true as const, value: "upstream-return" }; + let calls = 0; + const next = jest.fn(async (_value?: unknown) => ++calls === 1 ? first : returned); + const thrown = jest.fn(async (_error?: unknown) => recovered); + const finish = jest.fn(async (_value?: unknown) => returned); + const upstream: AsyncIterableIterator = { + next, + throw: thrown, + return: finish, + [Symbol.asyncIterator]() { return this; }, + }; + const response = await responseFor(upstream); + const iterator = response.body[Symbol.asyncIterator](); + const error = new Error("consumer control"); + expect(await iterator.next("resume")).toBe(first); + expect(await iterator.throw?.(error)).toBe(recovered); + expect(exporter.getFinishedSpans()).toHaveLength(0); + expect(await iterator.return?.("stop")).toBe(returned); + expect(next).toHaveBeenCalledTimes(1); + expect(next).toHaveBeenCalledWith("resume"); + expect(thrown).toHaveBeenCalledWith(error); + expect(finish).toHaveBeenCalledWith("stop"); + expect(next.mock.contexts[0]).toBe(upstream); + expect(thrown.mock.contexts[0]).toBe(upstream); + expect(finish.mock.contexts[0]).toBe(upstream); + // Repeated terminal operations must not export another span. + await iterator.return?.("again"); + expect(exporter.getFinishedSpans()).toHaveLength(1); + }); + + it("forwards return before the first next and ends the cancelled span", async () => { + const returned = { done: true as const, value: "cancelled" }; + const next = jest.fn(async () => returned); + const finish = jest.fn(async (_value?: unknown) => returned); + const upstream: AsyncIterableIterator = { + next, + return: finish, + [Symbol.asyncIterator]() { return this; }, + }; + const response = await responseFor(upstream); + const iterator = response.body[Symbol.asyncIterator](); + expect(await iterator.return?.("cancelled")).toBe(returned); + expect(next).not.toHaveBeenCalled(); + expect(finish).toHaveBeenCalledWith("cancelled"); + expect(exporter.getFinishedSpans()).toHaveLength(1); + expect(exporter.getFinishedSpans()[0].status.code).toBe(SpanStatusCode.UNSET); + }); + + it("preserves events and completed output attributes without a second consumer", async () => { + const events = [ + chunk({ outputText: "hello ", tokenCount: 1 }), + { chunk: { bytes: new TextEncoder().encode("not json") } }, + chunk({ + outputText: "world", + tokenCount: 1, + "amazon-bedrock-invocationMetrics": { inputTokenCount: 3, outputTokenCount: 2 }, + }), + ]; + async function* body() { yield* events; } + const response = await responseFor(body()); + expect(response.requestId).toBe("offline-response"); + const received = []; + for await (const event of response.body) received.push(event); + expect(received).toHaveLength(events.length); + received.forEach((event, index) => expect(event).toBe(events[index])); + const spans = exporter.getFinishedSpans(); + expect(spans).toHaveLength(1); + expect(spans[0].status.code).toBe(SpanStatusCode.OK); + expect(JSON.parse(spans[0].attributes[SemanticConventions.OUTPUT_VALUE] as string).text) + .toBe("hello world"); + expect(spans[0].attributes[SemanticConventions.LLM_TOKEN_COUNT_PROMPT]).toBe(3); + expect(spans[0].attributes[SemanticConventions.LLM_TOKEN_COUNT_COMPLETION]).toBe(2); + }); + + it("propagates the original upstream error and ends the span once", async () => { + const error = new Error("offline upstream failure"); + async function* body() { + yield chunk({ outputText: "partial" }); + throw error; + } + const response = await responseFor(body()); + const iterator = response.body[Symbol.asyncIterator](); + await iterator.next(); + await expect(iterator.next()).rejects.toBe(error); + await iterator.return?.(); + const spans = exporter.getFinishedSpans(); + expect(spans).toHaveLength(1); + expect(spans[0].status).toEqual({ + code: SpanStatusCode.ERROR, + message: "offline upstream failure", + }); + }); + + it("preserves a return failure rather than replacing or swallowing it", async () => { + const error = new Error("offline close failure"); + let calls = 0; + const finish = jest.fn(async () => { throw error; }); + const upstream: AsyncIterableIterator = { + async next() { + return ++calls === 1 + ? { done: false, value: chunk({ outputText: "partial" }) } + : { done: true, value: undefined }; + }, + return: finish, + [Symbol.asyncIterator]() { return this; }, + }; + const response = await responseFor(upstream); + const iterator = response.body[Symbol.asyncIterator](); + await iterator.next(); + await expect(iterator.return?.()).rejects.toBe(error); + expect(finish).toHaveBeenCalledTimes(1); + const spans = exporter.getFinishedSpans(); + expect(spans).toHaveLength(1); + expect(spans[0].status).toEqual({ + code: SpanStatusCode.ERROR, + message: "offline close failure", + }); + }); +}); diff --git a/typescript/packages/traceai_bedrock/src/attributes/attribute-helpers.ts b/typescript/packages/traceai_bedrock/src/attributes/attribute-helpers.ts index 846ff1e35..432b4c507 100644 --- a/typescript/packages/traceai_bedrock/src/attributes/attribute-helpers.ts +++ b/typescript/packages/traceai_bedrock/src/attributes/attribute-helpers.ts @@ -248,23 +248,13 @@ export function getAttributesFromMessage(message: Message): Attributes { } /** - * Extracts clean model name from full Bedrock model ID - * Removes vendor prefix and version suffixes to get the base model name + * Preserves the requested Bedrock model identifier for gen_ai.request.model. + * Model IDs, cross-region inference profiles, and ARNs do not share a fixed + * dot-separated layout; shortening them can discard the model or its version. * * @param modelId The full Bedrock model identifier - * @returns {string} The cleaned model name (e.g., "claude-3-sonnet" from "anthropic.claude-3-sonnet-20240229-v1:0") + * @returns {string} The original model identifier, including region and version */ export function extractModelName(modelId: string): string { - const parts = modelId.split("."); - if (parts.length > 1) { - const modelPart = parts[1]; - if (modelId.includes("anthropic")) { - const versionIndex = modelPart.indexOf("-v"); - if (versionIndex > 0) { - return modelPart.substring(0, versionIndex); - } - } - return modelPart; - } return modelId; } \ No newline at end of file diff --git a/typescript/packages/traceai_bedrock/src/attributes/invoke-model-streaming-response-attributes.ts b/typescript/packages/traceai_bedrock/src/attributes/invoke-model-streaming-response-attributes.ts index e949da0a5..02f5fff56 100644 --- a/typescript/packages/traceai_bedrock/src/attributes/invoke-model-streaming-response-attributes.ts +++ b/typescript/packages/traceai_bedrock/src/attributes/invoke-model-streaming-response-attributes.ts @@ -6,10 +6,10 @@ * - Tool call extraction from streams * - Token usage accumulation * - Real-time span attribute setting - * - Safe stream splitting with original stream preservation + * - Pull-driven observation with upstream iterator lifecycle preservation */ -import { Span, diag } from "@opentelemetry/api"; +import { Span, SpanStatusCode, diag } from "@opentelemetry/api"; import { withSafety, isObjectWithStringKeys, @@ -39,6 +39,7 @@ interface StreamChunk { */ interface StreamEventData { type?: string; + index?: number; message?: { usage?: Record; }; @@ -50,10 +51,17 @@ interface StreamEventData { input?: Record; }; delta?: { + type?: string; text?: string; + partial_json?: string; + stop_reason?: string; }; usage?: Record; + stop_reason?: string; + completionReason?: string; + messageStop?: { stopReason?: string }; + // Amazon-specific fields outputText?: string; tokenCount?: number; @@ -105,9 +113,30 @@ interface StreamProcessingState { text?: string; id?: string; name?: string; - input?: Record; + input?: Record | string; + partialJson?: string; + argumentsIncomplete?: boolean; }>; + toolBlocksByIndex: Map; rawUsageData: Record; + finishReason?: string; +} + +/** Exported in place of tool arguments that never formed valid JSON. */ +const INCOMPLETE_TOOL_ARGUMENTS = ""; + +/** Parse complete tool input; never export an unparsed argument fragment. */ +function finalizeToolInput( + block: StreamProcessingState["contentBlocks"][number], +): void { + if (block.partialJson === undefined) return; + try { + const input: unknown = JSON.parse(block.partialJson); + block.input = isObjectWithStringKeys(input) ? input : block.partialJson; + } catch { + block.input = INCOMPLETE_TOOL_ARGUMENTS; + block.argumentsIncomplete = true; + } } /** @@ -132,9 +161,13 @@ function processAnthropicStreamChunk( } if (data.type === "content_block_start" && data.content_block) { - // Store tool use blocks for later processing, don't add duplicates if (data.content_block.type === "tool_use") { - state.contentBlocks.push(data.content_block); + // Accumulate into an owned block, never into the application's events. + const block = { ...data.content_block }; + state.contentBlocks.push(block); + if (typeof data.index === "number") { + state.toolBlocksByIndex.set(data.index, block); + } } } @@ -143,6 +176,30 @@ function processAnthropicStreamChunk( state.outputText += data.delta.text; } + if ( + data.type === "content_block_delta" && + data.delta?.type === "input_json_delta" && + typeof data.delta.partial_json === "string" && + typeof data.index === "number" + ) { + const block = state.toolBlocksByIndex.get(data.index); + if (block) { + block.partialJson = (block.partialJson ?? "") + data.delta.partial_json; + } + } + + if (data.type === "content_block_stop" && typeof data.index === "number") { + const block = state.toolBlocksByIndex.get(data.index); + if (block) { + finalizeToolInput(block); + state.toolBlocksByIndex.delete(data.index); + } + } + + if (data.type === "message_delta" && typeof data.delta?.stop_reason === "string") { + state.finishReason = data.delta.stop_reason; + } + if (data.type === "message_delta" && data.usage) { state.rawUsageData = { ...state.rawUsageData, ...data.usage }; } @@ -162,6 +219,8 @@ function processMetaStreamChunk( data: StreamEventData, state: StreamProcessingState, ): StreamProcessingState { + if (typeof data.stop_reason === "string") state.finishReason = data.stop_reason; + // Meta text generation - accumulate all text if (typeof data.generation === "string") { state.outputText += data.generation; @@ -186,6 +245,7 @@ function processMetaStreamChunk( function isTitanStreamChunk(data: StreamEventData): boolean { return !!( typeof data.outputText === "string" || + typeof data.completionReason === "string" || typeof data.tokenCount === "number" || data["amazon-bedrock-invocationMetrics"] ); @@ -197,6 +257,7 @@ function isTitanStreamChunk(data: StreamEventData): boolean { */ function isNovaStreamChunk(data: StreamEventData): boolean { return !!( + data.messageStop?.stopReason || data.contentBlockDelta?.delta?.text || data.metadata?.usage || // Nova usage without type field - check for camelCase token names @@ -222,6 +283,8 @@ function processTitanStreamChunk( data: StreamEventData, state: StreamProcessingState, ): StreamProcessingState { + if (typeof data.completionReason === "string") state.finishReason = data.completionReason; + // Titan text output if (typeof data.outputText === "string") { state.outputText += data.outputText; @@ -259,6 +322,8 @@ function processNovaStreamChunk( data: StreamEventData, state: StreamProcessingState, ): StreamProcessingState { + if (typeof data.messageStop?.stopReason === "string") state.finishReason = data.messageStop.stopReason; + // Nova text output if ( data.contentBlockDelta?.delta?.text && @@ -359,19 +424,32 @@ function setStreamingOutputAttributes({ outputText, contentBlocks, usage, + finishReason, }: { span: Span; outputText: string; contentBlocks: StreamProcessingState["contentBlocks"]; usage: UsageAttributes; + finishReason?: string; }): void { + const incompleteBlocks = contentBlocks.filter(block => block.argumentsIncomplete); + if (incompleteBlocks.length > 0) { + setSpanAttribute(span, "fi.tool_call.arguments_incomplete", true); + // Sum of UTF-16 code units received for incomplete calls; no argument text. + setSpanAttribute(span, "fi.tool_call.arguments_received_length", + incompleteBlocks.reduce((length, block) => length + (block.partialJson?.length ?? 0), 0)); + } + if (finishReason) { + setSpanAttribute(span, SemanticConventions.GEN_AI_RESPONSE_FINISH_REASONS, [finishReason]); + } + // Create the output value structure like the original streaming code const toolCalls = contentBlocks .filter((block) => block.type === "tool_use") .map((block) => ({ id: block.id || "unknown", name: block.name || "unknown", - input: block.input || {}, + input: block.input ?? {}, })); const outputValue = { @@ -407,7 +485,9 @@ function setStreamingOutputAttributes({ id: block.id || "unknown", function: { name: block.name || "unknown", - arguments: JSON.stringify(block.input || {}), + arguments: block.argumentsIncomplete + ? INCOMPLETE_TOOL_ARGUMENTS + : block.partialJson ?? JSON.stringify(block.input || {}), }, })); } @@ -456,14 +536,7 @@ function setStreamingOutputAttributes({ } } -/** - * Safely splits an async iterable stream into two streams for parallel consumption - * Uses Node.js built-in PassThrough streams for reliable stream duplication - * Creates separate streams for instrumentation and user consumption - * - * @param originalStream The original stream to split - * @returns Object with instrumentation and user streams, or just user stream on fallback - */ +/** Compatibility helper; send instrumentation uses pull-driven observation below. */ export function safelySplitStream({ originalStream, }: { @@ -477,6 +550,11 @@ export function safelySplitStream({ const instrumentationStream = new PassThrough({ objectMode: true }); const userStream = new PassThrough({ objectMode: true }); + // Handle error events even before either consumer starts iterating. Node + // retains destroy(error) in the stream state, so later iteration still rejects. + instrumentationStream.on("error", () => {}); + userStream.on("error", () => {}); + // Function to consume the original stream and write to both outputs const consumeAndDuplicate = async () => { try { @@ -514,96 +592,185 @@ export function safelySplitStream({ } } +interface UnconsumedStreamSpan { + span: Span; + consumed: boolean; + ended: boolean; +} + +// Holdings contain only span state, never the source, wrapper, iterator or a +// callback closing over them. The weak token lives with the returned stream and +// its iterators, so abandoning an unconsumed response can actually trigger GC. +const unconsumedStreams = typeof FinalizationRegistry === "undefined" + ? undefined + : new FinalizationRegistry((state) => { + if (state.consumed || state.ended) return; + state.ended = true; + try { + try { + state.span.setAttribute("fi.stream.consumed", false); + state.span.setStatus({ code: SpanStatusCode.UNSET }); + } finally { + state.span.end(); + } + } catch (error) { + diag.warn("Error finalizing unconsumed Bedrock stream:", error); + } + }); + /** - * Consumes AWS Bedrock streaming response chunks and extracts attributes for OpenTelemetry span. - * Supports multiple providers: Anthropic, Amazon Titan, Meta Llama, and Amazon Nova. - * - * This function processes different Bedrock streaming formats and accumulates response content, - * setting appropriate semantic convention attributes on the provided span. It runs in the - * background without blocking the user's stream consumption. + * Observes Bedrock events only when the user requests them. No tee or background + * reader is created, so an idle/slow consumer cannot queue extra response data. + * Iterator control methods and upstream failures are forwarded unchanged, while + * telemetry failures are isolated from the application. * - * @param stream - The Bedrock response stream (AsyncIterable), typically from splitStream() - * @param span - The OpenTelemetry span to set attributes on - * @param modelType - The LLM system type to determine parsing strategy - * @throws {Error} If critical stream processing errors occur - * - * @example - * ```typescript - * // Background processing after stream splitting - * const [instrumentationStream, userStream] = await splitStream(response.body); - * consumeBedrockStreamChunks({ stream: instrumentationStream, span, modelType }) - * .then(() => span.end()) - * .catch(error => { span.recordException(error); span.end(); }); - * ``` + * gen_ai.server.time_to_first_token is measured in milliseconds from send() until the + * first chunk the CALLER pulls, including caller delay (not network arrival). + * Never-iterated streams end best-effort on GC as UNSET with + * fi.stream.consumed=false; no timer or upstream pull is scheduled. */ -export const consumeBedrockStreamChunks = withSafety({ - fn: async ({ - stream, - span, - modelType, - }: { - stream: AsyncIterable; - span: Span; - modelType: LLMSystem; - }): Promise => { - // Initialize processing state - const state: StreamProcessingState = { - outputText: "", - contentBlocks: [], - rawUsageData: {}, - }; +export function observeBedrockStream({ + stream, + span, + modelType, + onEnd, + onError, + startedAt = performance.now(), +}: { + stream: AsyncIterable; + span: Span; + modelType: LLMSystem; + onEnd: (cancelled: boolean) => void; + onError: (error: unknown) => void; + startedAt?: number; +}): AsyncIterable { + const state: StreamProcessingState = { + outputText: "", + contentBlocks: [], + toolBlocksByIndex: new Map(), + rawUsageData: {}, + }; + const lifetime = {}; + const lifecycle: UnconsumedStreamSpan = { span, consumed: false, ended: false }; + unconsumedStreams?.register(lifetime, lifecycle, lifetime); + let receivedFirstChunk = false; + + const consume = () => { + if (lifecycle.consumed) return; + lifecycle.consumed = true; + unconsumedStreams?.unregister(lifetime); + }; - for await (const chunk of stream) { - // Type guard for chunk structure - if (isValidStreamChunk(chunk)) { - const text = new TextDecoder().decode(chunk.chunk.bytes); - const lines = text.split("\n").filter((line) => line.trim()); - - for (const line of lines) { - if (line.trim()) { - try { - const rawData = JSON.parse(line); - if (!isValidStreamEventData(rawData)) { - continue; // Skip invalid event data - } - const data: StreamEventData = rawData; - - // Process based on provider format using dedicated helpers - if (modelType === LLMSystem.ANTHROPIC) { - processAnthropicStreamChunk(data, state); - } else if (modelType === LLMSystem.AMAZON) { - if (isTitanStreamChunk(data)) { - processTitanStreamChunk(data, state); - } else if (isNovaStreamChunk(data)) { - processNovaStreamChunk(data, state); - } - } else if (modelType === LLMSystem.META) { - processMetaStreamChunk(data, state); - } - } catch (parseError) { - // Skip malformed JSON lines silently - continue; + // These safety boundaries are synchronous and never wrap upstream operations. + const processChunk = withSafety({ + fn: (chunk: unknown) => { + if (lifecycle.ended || !isValidStreamChunk(chunk)) return; + const text = new TextDecoder().decode(chunk.chunk.bytes); + const lines = text.split("\n").filter((line) => line.trim()); + for (const line of lines) { + try { + const rawData = JSON.parse(line); + if (!isValidStreamEventData(rawData)) continue; + const data: StreamEventData = rawData; + if (modelType === LLMSystem.ANTHROPIC) { + processAnthropicStreamChunk(data, state); + } else if (modelType === LLMSystem.AMAZON) { + if (isTitanStreamChunk(data)) { + processTitanStreamChunk(data, state); + } else if (isNovaStreamChunk(data)) { + processNovaStreamChunk(data, state); } + } else if (modelType === LLMSystem.META) { + processMetaStreamChunk(data, state); } + } catch (parseError) { + // Malformed JSON must not prevent delivery of the original event. + continue; } } + }, + onError: (error) => diag.warn("Error observing Bedrock stream chunk:", error), + }); + + const finish = withSafety({ + fn: (notify: () => void) => { + if (lifecycle.ended) return; + lifecycle.ended = true; + unconsumedStreams?.unregister(lifetime); + try { + // Preserve complete or partial arguments on completion, failure or cancellation. + for (const block of state.toolBlocksByIndex.values()) { + finalizeToolInput(block); + } + setStreamingOutputAttributes({ + span, + outputText: state.outputText, + contentBlocks: state.contentBlocks, + usage: normalizeStreamUsageData(state.rawUsageData, modelType), + finishReason: state.finishReason, + }); + } finally { + // End even a cancelled span, including when attribute extraction fails. + notify(); + } + }, + onError: (error) => diag.warn("Error finalizing Bedrock stream attributes:", error), + }); + + const observe = async ( + operation: () => Promise>, + control: "next" | "return" | "throw" = "next", + thrown?: unknown, + ): Promise> => { + if (control === "next") consume(); + try { + const result = await operation(); + if (!result.done) { + consume(); + if (!receivedFirstChunk && !lifecycle.ended) { + receivedFirstChunk = true; + // Isolate telemetry, including the timing attribute, from delivery. + try { + setSpanAttribute(span, SemanticConventions.GEN_AI_SERVER_TIME_TO_FIRST_TOKEN, + Math.max(0, performance.now() - startedAt)); + } catch (error) { diag.warn("Error recording Bedrock first chunk:", error); } + } + processChunk(result.value); + } + // A handled throw that yields another value is not a cancellation. + if (control === "return" || result.done) finish(() => onEnd(control !== "next")); + return result; + } catch (error) { + // The caller's own terminating throw is cancellation, not a provider + // failure. A different upstream error (including return failure) stays ERROR. + finish(() => control === "throw" && error === thrown ? onEnd(true) : onError(error)); + throw error; } + }; - // Normalize usage data once at the end - const normalizedUsage = normalizeStreamUsageData( - state.rawUsageData, - modelType, - ); - - setStreamingOutputAttributes({ - span, - outputText: state.outputText, - contentBlocks: state.contentBlocks, - usage: normalizedUsage, - }); - }, - onError: (error) => { - diag.warn("Error consuming Bedrock stream chunks:", error); - throw error; - }, -}); \ No newline at end of file + return { + [Symbol.asyncIterator](): AsyncIterableIterator { + let iterator: AsyncIterator; + try { + iterator = stream[Symbol.asyncIterator](); + } catch (error) { + finish(() => onError(error)); + throw error; + } + return { + next: (...args) => observe(() => iterator.next(...args)), + return: (...args) => + observe( + () => iterator.return + ? iterator.return(...args) + : Promise.resolve({ done: true as const, value: args[0] }), + "return", + ), + throw: iterator.throw + ? (...args) => observe(() => iterator.throw!(...args), "throw", args[0]) + : undefined, + [Symbol.asyncIterator]() { return this; }, + }; + }, + }; +} diff --git a/typescript/packages/traceai_bedrock/src/instrumentation.ts b/typescript/packages/traceai_bedrock/src/instrumentation.ts index 75e0b0261..1969f5c0d 100644 --- a/typescript/packages/traceai_bedrock/src/instrumentation.ts +++ b/typescript/packages/traceai_bedrock/src/instrumentation.ts @@ -32,10 +32,7 @@ import { extractInvokeModelRequestAttributes } from "./attributes/invoke-model-r import { extractInvokeModelResponseAttributes } from "./attributes/invoke-model-response-attributes"; import { extractConverseRequestAttributes } from "./attributes/converse-request-attributes"; import { extractConverseResponseAttributes } from "./attributes/converse-response-attributes"; -import { - consumeBedrockStreamChunks, - safelySplitStream, -} from "./attributes/invoke-model-streaming-response-attributes"; +import { observeBedrockStream } from "./attributes/invoke-model-streaming-response-attributes"; import { getSystemFromModelId, setBasicSpanAttributes, @@ -43,7 +40,7 @@ import { const MODULE_NAME = "@aws-sdk/client-bedrock-runtime"; const INSTRUMENTATION_NAME = "@traceai/bedrock"; -const INSTRUMENTATION_VERSION = "0.1.3"; // x-release-please-version +const INSTRUMENTATION_VERSION = "0.1.4"; // x-release-please-version /** * AWS SDK module interface for proper typing @@ -199,12 +196,14 @@ export class BedrockInstrumentation extends InstrumentationBase { // eslint-disable-next-line @typescript-eslint/no-explicit-any (original: any) => { // eslint-disable-next-line @typescript-eslint/no-explicit-any - return function patchedSend(this: unknown, command: any) { + return function patchedSend(this: unknown, ...args: any[]) { + const [command] = args; if (command?.constructor?.name === "InvokeModelCommand") { return instrumentation.handleInvokeModelCommand( command as InvokeModelCommand, original, this, + args, ); } @@ -216,6 +215,7 @@ export class BedrockInstrumentation extends InstrumentationBase { command as InvokeModelWithResponseStreamCommand, original, this, + args, ); } @@ -224,11 +224,12 @@ export class BedrockInstrumentation extends InstrumentationBase { command as ConverseCommand, original, this, + args, ); } // Pass through other commands without instrumentation - return original.apply(this, [command]); + return original.apply(this, args); }; }, ); @@ -258,6 +259,62 @@ export class BedrockInstrumentation extends InstrumentationBase { } } + /** + * Observe either send overload without converting callback calls into promises. + * Only the callback is replaced; command, options, trailing arguments and receiver + * are forwarded unchanged. Response processing stays synchronous for callbacks. + */ + private sendWithResponse( + original: (...args: unknown[]) => Promise | void, + client: unknown, + args: unknown[], + span: Span, + onResponse: (response: T) => T, + ): Promise | void { + const callbackIndex = typeof args[1] === "function" + ? 1 + : typeof args[2] === "function" ? 2 : -1; + const onError = (error: unknown) => { + recordBedrockError(span, error); + span.end(); + }; + let callbackInvoked = false; + + try { + if (callbackIndex !== -1) { + const callback = args[callbackIndex] as (...args: unknown[]) => unknown; + args[callbackIndex] = function (this: unknown, ...callbackArgs: unknown[]) { + callbackInvoked = true; + try { + if (callbackArgs[0]) { + onError(callbackArgs[0]); + } else { + callbackArgs[1] = onResponse(callbackArgs[1] as T); + } + } catch (error) { + // Instrumentation failures must not prevent delivery to the application. + diag.warn("Error processing Bedrock callback response:", error); + span.end(); + } + // Keep application exceptions outside the instrumentation error boundary. + return callback.apply(this, callbackArgs); + }; + return original.apply(client, args); + } + + const result = original.apply(client, args) as Promise; + return result.then(onResponse).catch((error: unknown) => { + onError(error); + throw error; + }); + } catch (error) { + // A synchronous application callback can throw through send(). Do not record + // it as an SDK failure or invoke the callback a second time. + if (!callbackInvoked) onError(error); + throw error; + } + } + /** * Handles instrumentation for synchronous InvokeModel commands * Creates a span, extracts request attributes, executes the command, and processes the response @@ -265,7 +322,8 @@ export class BedrockInstrumentation extends InstrumentationBase { * @param command The InvokeModelCommand to instrument * @param original The original send method from the Bedrock client * @param client The Bedrock client instance - * @returns {Promise} The response from the InvokeModel call + * @param args The complete send argument list + * @returns A response promise, or void for the callback overload */ private handleInvokeModelCommand( command: InvokeModelCommand, @@ -273,7 +331,8 @@ export class BedrockInstrumentation extends InstrumentationBase { original: any, // eslint-disable-next-line @typescript-eslint/no-explicit-any client: any, - ): Promise { + args: unknown[], + ): Promise | void { const span = this.fiTracer.startSpan("bedrock.invoke_model", { kind: SpanKind.INTERNAL, }); @@ -287,43 +346,31 @@ export class BedrockInstrumentation extends InstrumentationBase { extractInvokeModelRequestAttributes({ span, command, system }); - try { - const result = original.apply(client, [ - command, - ]) as Promise; - - // AWS SDK v3 send() method always returns a Promise - return result - .then((response: InvokeModelResponse) => { - extractInvokeModelResponseAttributes({ span, response, system }); - span.setStatus({ code: SpanStatusCode.OK }); - span.end(); - return response; - }) - .catch((error: unknown) => { - recordBedrockError(span, error); - span.end(); - throw error; - }); - } catch (error) { - // Handle errors that occur before the Promise is returned (e.g. invalid parameters) - recordBedrockError(span, error); - span.end(); - throw error; - } + return this.sendWithResponse( + original, + client, + args, + span, + (response: InvokeModelResponse) => { + extractInvokeModelResponseAttributes({ span, response, system }); + span.setStatus({ code: SpanStatusCode.OK }); + span.end(); + return response; + }, + ); } /** * Handles instrumentation for streaming InvokeModel commands with guaranteed stream preservation * - * This method ensures the original user stream is preserved regardless of instrumentation - * success or failure. The user stream is returned immediately while instrumentation - * happens asynchronously in the background using stream splitting. + * Observes events as the application pulls them, without a background reader. + * Consumer cancellation and iterator control are forwarded to the original stream. * * @param command The InvokeModelWithResponseStreamCommand to instrument * @param original The original send method from the Bedrock client * @param client The Bedrock client instance - * @returns {Promise<{body: AsyncIterable}>} Promise resolving to the response with preserved user stream + * @param args The complete send argument list + * @returns A response promise with preserved user stream, or void for callbacks */ private handleInvokeModelWithResponseStreamCommand( command: InvokeModelWithResponseStreamCommand, @@ -331,7 +378,9 @@ export class BedrockInstrumentation extends InstrumentationBase { original: any, // eslint-disable-next-line @typescript-eslint/no-explicit-any client: any, + args: unknown[], ) { + const startedAt = performance.now(); const span = this.fiTracer.startSpan("bedrock.invoke_model", { kind: SpanKind.INTERNAL, }); @@ -349,14 +398,12 @@ export class BedrockInstrumentation extends InstrumentationBase { system, }); - // Execute AWS SDK call and handle stream splitting outside error boundaries - // This ensures the user stream is ALWAYS returned, regardless of instrumentation failures - const result = original.apply(client, [command]) as Promise<{ - body: AsyncIterable; - }>; - - return result - .then((response: { body: AsyncIterable }) => { + return this.sendWithResponse( + original, + client, + args, + span, + (response: { body: AsyncIterable }) => { // Guard against missing response body - return original response if (!response.body) { span.recordException(new Error("Response body is undefined")); @@ -368,42 +415,31 @@ export class BedrockInstrumentation extends InstrumentationBase { return response; } - const { instrumentationStream, userStream } = safelySplitStream({ - originalStream: response.body, - }); - - // Start background instrumentation processing (non-blocking) - // Only the instrumentation is wrapped in error handling - the user stream is protected - if (instrumentationStream) { - // @ts-expect-error - instrumentationStream is guaranteed non-null by the if check above - consumeBedrockStreamChunks({ - stream: instrumentationStream, - span, - modelType: system, - }) - .then(() => { - span.setStatus({ code: SpanStatusCode.OK }); + const body = observeBedrockStream({ + stream: response.body, + span, + modelType: system, + startedAt, + onEnd: (cancelled) => { + try { + if (cancelled) span.addEvent("stream.cancelled"); + else span.setStatus({ code: SpanStatusCode.OK }); + } finally { span.end(); - }) - .catch((error: unknown) => { + } + }, + onError: (error) => { + try { recordBedrockError(span, error); + } finally { span.end(); - }); - } else { - // No instrumentation stream available, end span cleanly - span.setStatus({ code: SpanStatusCode.OK }); - span.end(); - } + } + }, + }); - // Return user stream immediately - instrumentation cannot interfere - return { ...response, body: userStream }; - }) - .catch((error: unknown) => { - // If the AWS SDK call itself fails, record error but still try to return original response - recordBedrockError(span, error); - span.end(); - throw error; // Re-throw since we have no stream to return - }); + return { ...response, body }; + }, + ); } /** @@ -413,7 +449,8 @@ export class BedrockInstrumentation extends InstrumentationBase { * @param command The ConverseCommand to instrument * @param original The original send method from the Bedrock client * @param client The Bedrock client instance - * @returns {Promise} The response from the Converse call + * @param args The complete send argument list + * @returns A response promise, or void for the callback overload */ private handleConverseCommand( command: ConverseCommand, @@ -421,7 +458,8 @@ export class BedrockInstrumentation extends InstrumentationBase { original: any, // eslint-disable-next-line @typescript-eslint/no-explicit-any client: any, - ): Promise { + args: unknown[], + ): Promise | void { const span = this.fiTracer.startSpan("bedrock.converse", { kind: SpanKind.INTERNAL, }); @@ -435,29 +473,17 @@ export class BedrockInstrumentation extends InstrumentationBase { extractConverseRequestAttributes({ span, command }); - try { - const result = original.apply(client, [ - command, - ]) as Promise; - - // AWS SDK v3 send() method always returns a Promise - return result - .then((response: ConverseResponse) => { - extractConverseResponseAttributes({ span, response }); - span.setStatus({ code: SpanStatusCode.OK }); - span.end(); - return response; - }) - .catch((error: unknown) => { - recordBedrockError(span, error); - span.end(); - throw error; - }); - } catch (error) { - // Handle errors that occur before the Promise is returned (e.g. invalid parameters) - recordBedrockError(span, error); - span.end(); - throw error; - } + return this.sendWithResponse( + original, + client, + args, + span, + (response: ConverseResponse) => { + extractConverseResponseAttributes({ span, response }); + span.setStatus({ code: SpanStatusCode.OK }); + span.end(); + return response; + }, + ); } } \ No newline at end of file diff --git a/typescript/packages/traceai_cohere/CHANGELOG.md b/typescript/packages/traceai_cohere/CHANGELOG.md index 996c68d3c..9b6c256fa 100644 --- a/typescript/packages/traceai_cohere/CHANGELOG.md +++ b/typescript/packages/traceai_cohere/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +## [0.1.2] - pending release + +### Fixed +- Integer (int8 and uint8) embedding vectors are recorded. +- Binary and ubinary embedding vectors are recorded as returned, packed with one value per eight dimensions; cohere.embedding_type names the recorded type. + ## [0.1.1](https://github.com/future-agi/traceAI/compare/npm-traceai-cohere-v0.1.0...npm-traceai-cohere-v0.1.1) (2026-11-05) diff --git a/typescript/packages/traceai_cohere/package.json b/typescript/packages/traceai_cohere/package.json index 8b3e4c357..ab9a21979 100644 --- a/typescript/packages/traceai_cohere/package.json +++ b/typescript/packages/traceai_cohere/package.json @@ -1,6 +1,6 @@ { "name": "@traceai/cohere", - "version": "0.1.1", + "version": "0.1.2", "description": "OpenTelemetry instrumentation for Cohere AI", "main": "./dist/src/index.js", "module": "./dist/esm/index.js", diff --git a/typescript/packages/traceai_cohere/src/__tests__/embeddings.test.ts b/typescript/packages/traceai_cohere/src/__tests__/embeddings.test.ts new file mode 100644 index 000000000..5a76c9e74 --- /dev/null +++ b/typescript/packages/traceai_cohere/src/__tests__/embeddings.test.ts @@ -0,0 +1,159 @@ +import { BasicTracerProvider, InMemorySpanExporter, SimpleSpanProcessor } from "@opentelemetry/sdk-trace-base"; +import { generateTraceConfig } from "@traceai/fi-core"; +import { SemanticConventions as S } from "@traceai/fi-semantic-conventions"; +import { outputAttributes } from "../attributes"; +import { CohereInstrumentation } from "../instrumentation"; + +const captureConfig = generateTraceConfig({ + hideInputs: false, hideInputMessages: false, hideInputText: false, hideInputImages: false, + hideOutputs: false, hideOutputMessages: false, hideOutputText: false, + hideEmbeddingVectors: false, piiRedaction: false, +}); +const vectorKey = (index: number) => `${S.EMBEDDING_EMBEDDINGS}.${index}.${S.EMBEDDING_VECTOR}`; +const exporter = new InMemorySpanExporter(); +const provider = new BasicTracerProvider({ spanProcessors: [new SimpleSpanProcessor(exporter)] }); +let instrumentation: CohereInstrumentation | undefined; +beforeEach(() => exporter.reset()); +afterEach(() => instrumentation?.disable()); +afterAll(async () => provider.shutdown()); + +describe("integer embedding attributes", () => { + describe.each([ + { type: "int8", values: [[-128, 0, 127], [1, -2, 3]] }, + { type: "uint8", values: [[0, 128, 255], [1, 2, 3]] }, + ])("$type", ({ type, values }) => { + const vectors = Object.freeze(values.map(vector => Object.freeze(vector))); + + it.each([false, true])("exports every vector through the wrapped SDK without changing the response (v2=%s)", async v2 => { + const response = Object.freeze({ embeddings: Object.freeze({ [type]: vectors }) }); + const request = Object.freeze({ + model: "embed-english-v3.0", texts: ["Paris", "London"], + inputType: "search_document", embeddingTypes: [type], + }); + const embed = jest.fn(async (_request: unknown) => response); + // Fake the generated SDK's v1 method and v2 getter; neither performs I/O. + class V2Client { embed(request: unknown) { return embed(request); } } + class Client { + embed(request: unknown) { return embed(request); } + get v2() { return new V2Client(); } + } + instrumentation = new CohereInstrumentation({ traceConfig: captureConfig }); + instrumentation.setTracerProvider(provider); + instrumentation.manuallyInstrument({ CohereClient: Client }); + const client = new Client(); + const result = await (v2 ? client.v2 : client).embed(request); + + expect(result).toBe(response); + expect(result.embeddings[type]).toBe(vectors); + expect(embed).toHaveBeenCalledTimes(1); + expect(embed).toHaveBeenCalledWith(request); + const spans = exporter.getFinishedSpans(); + expect(spans).toHaveLength(1); + const attrs = spans[0].attributes; + vectors.forEach((vector, index) => expect(attrs[vectorKey(index)]).toEqual(vector)); + expect(attrs["cohere.embedding_type"]).toBe(type); + expect(JSON.parse(attrs[S.RAW_OUTPUT] as string)).toEqual(response); + }); + + it.each(["hideEmbeddingVectors", "hideOutputs", "hideOutputMessages", "hideOutputText"] as const)( + "honors %s for standard and raw vectors", + flag => { + const attrs = outputAttributes("embed", { embeddings: { [type]: vectors } }, { ...captureConfig, [flag]: true }); + expect(attrs[vectorKey(0)]).toBeUndefined(); + expect(attrs[vectorKey(1)]).toBeUndefined(); + expect(attrs[S.RAW_OUTPUT]).toBeUndefined(); + expect(attrs["cohere.embedding_type"]).toBeUndefined(); + }, + ); + + it("retains vectors when only input capture is hidden", () => { + const response = { embeddings: { [type]: vectors }, texts: ["Paris", "London"] }; + const attrs = outputAttributes("embed", response, { ...captureConfig, hideInputs: true }); + vectors.forEach((vector, index) => expect(attrs[vectorKey(index)]).toEqual(vector)); + expect(JSON.parse(attrs[S.RAW_OUTPUT] as string)).toEqual({ embeddings: response.embeddings }); + expect(response.texts).toEqual(["Paris", "London"]); + }); + }); + + it("preserves legacy array embeddings", () => { + const response = { embeddings: [[0.1, 0.2], [0.3, 0.4]] }; + const attrs = outputAttributes("embed", response, captureConfig); + response.embeddings.forEach((vector, index) => expect(attrs[vectorKey(index)]).toEqual(vector)); + expect(JSON.parse(attrs[S.RAW_OUTPUT] as string)).toEqual(response); + }); + + it.each([ + { type: "float", embeddings: { float: [[0.1, 0.2]], int8: [[1, -2]], uint8: [[1, 2]] } }, + { type: "int8", embeddings: { int8: [[1, -2]], uint8: [[1, 2]], binary: [[-1]] } }, + { type: "uint8", embeddings: { float: null, uint8: [[1, 2]], ubinary: [[255]] } }, + ])("selects $type deterministically without mixing representations", ({ type, embeddings }) => { + const attrs = outputAttributes("embed", { embeddings }, captureConfig); + const expected = type === "float" ? [0.1, 0.2] : type === "int8" ? [1, -2] : [1, 2]; + expect(attrs[vectorKey(0)]).toEqual(expected); + expect(attrs[vectorKey(1)]).toBeUndefined(); + expect(attrs["cohere.embedding_type"]).toBe(type); + expect(JSON.parse(attrs[S.RAW_OUTPUT] as string)).toEqual({ embeddings }); + }); +}); + +describe("packed binary embedding attributes", () => { + describe.each([ + { type: "binary", values: [[-1, 5, 7], [-128, 0, 127]] }, + { type: "ubinary", values: [[255, 5, 7], [0, 128, 1]] }, + ])("$type", ({ type, values }) => { + const vectors = Object.freeze(values.map(vector => Object.freeze(vector))); + + it.each([false, true])("exports every packed vector as returned through the wrapped SDK (v2=%s)", async v2 => { + const response = Object.freeze({ embeddings: Object.freeze({ [type]: vectors }) }); + const request = Object.freeze({ + model: "embed-english-v3.0", texts: ["Paris", "London"], + inputType: "search_document", embeddingTypes: [type], + }); + const embed = jest.fn(async (_request: unknown) => response); + // Fake the generated SDK's v1 method and v2 getter; neither performs I/O. + class V2Client { embed(request: unknown) { return embed(request); } } + class Client { + embed(request: unknown) { return embed(request); } + get v2() { return new V2Client(); } + } + instrumentation = new CohereInstrumentation({ traceConfig: captureConfig }); + instrumentation.setTracerProvider(provider); + instrumentation.manuallyInstrument({ CohereClient: Client }); + const client = new Client(); + const result = await (v2 ? client.v2 : client).embed(request); + + expect(result).toBe(response); + expect(result.embeddings[type]).toBe(vectors); + expect(embed).toHaveBeenCalledTimes(1); + const spans = exporter.getFinishedSpans(); + expect(spans).toHaveLength(1); + const attrs = spans[0].attributes; + vectors.forEach((vector, index) => expect(attrs[vectorKey(index)]).toEqual(vector)); + expect(attrs[vectorKey(2)]).toBeUndefined(); + expect(attrs["cohere.embedding_type"]).toBe(type); + expect(JSON.parse(attrs[S.RAW_OUTPUT] as string)).toEqual(response); + }); + + it.each(["hideEmbeddingVectors", "hideOutputs", "hideOutputMessages", "hideOutputText"] as const)( + "honors %s for standard and raw vectors", + flag => { + const attrs = outputAttributes("embed", { embeddings: { [type]: vectors } }, { ...captureConfig, [flag]: true }); + expect(attrs[vectorKey(0)]).toBeUndefined(); + expect(attrs[vectorKey(1)]).toBeUndefined(); + expect(attrs[S.RAW_OUTPUT]).toBeUndefined(); + expect(attrs["cohere.embedding_type"]).toBeUndefined(); + }, + ); + }); + + it.each([ + { order: "binary first", embeddings: { binary: [[-1, 5]], ubinary: [[255, 5]] } }, + { order: "ubinary first", embeddings: { ubinary: [[255, 5]], binary: [[-1, 5]] } }, + ])("selects binary over ubinary deterministically ($order)", ({ embeddings }) => { + const attrs = outputAttributes("embed", { embeddings }, captureConfig); + expect(attrs[vectorKey(0)]).toEqual([-1, 5]); + expect(attrs[vectorKey(1)]).toBeUndefined(); + expect(attrs["cohere.embedding_type"]).toBe("binary"); + expect(JSON.parse(attrs[S.RAW_OUTPUT] as string)).toEqual({ embeddings }); + }); +}); diff --git a/typescript/packages/traceai_cohere/src/attributes.ts b/typescript/packages/traceai_cohere/src/attributes.ts index 1eb7e663b..4b8d1479c 100644 --- a/typescript/packages/traceai_cohere/src/attributes.ts +++ b/typescript/packages/traceai_cohere/src/attributes.ts @@ -149,12 +149,18 @@ export function outputAttributes(operation: Operation, response: Data, config: T if (operation === "embed") { // raw.output would otherwise bypass hideEmbeddingVectors. if (!hideOutput(config) && !config.hideEmbeddingVectors) { - const vectors = Array.isArray(response.embeddings) ? response.embeddings : object(response.embeddings).float; + const embeddings = object(response.embeddings); + // Prefer float coordinates, then integer coordinates, then packed binary + // and ubinary values as returned (one value per eight dimensions); + // cohere.embedding_type tells readers which representation was recorded. + const embeddingType = ["float", "int8", "uint8", "binary", "ubinary"].find(type => Array.isArray(embeddings[type])); + const vectors = Array.isArray(response.embeddings) ? response.embeddings : embeddingType ? embeddings[embeddingType] : undefined; list(vectors).forEach((vector, index) => { if (Array.isArray(vector) && vector.every((n) => typeof n === "number")) { attrs[`${S.EMBEDDING_EMBEDDINGS}.${index}.${S.EMBEDDING_VECTOR}`] = vector; } }); + if (embeddingType) attrs["cohere.embedding_type"] = embeddingType; attrs[S.RAW_OUTPUT] = json(response); } } else if (operation === "rerank") { diff --git a/typescript/packages/traceai_cohere/src/version.ts b/typescript/packages/traceai_cohere/src/version.ts index b322647c1..5447861b8 100644 --- a/typescript/packages/traceai_cohere/src/version.ts +++ b/typescript/packages/traceai_cohere/src/version.ts @@ -1 +1 @@ -export const VERSION = '0.1.1'; // x-release-please-version +export const VERSION = '0.1.2'; // x-release-please-version diff --git a/typescript/packages/traceai_langchain/CHANGELOG.md b/typescript/packages/traceai_langchain/CHANGELOG.md index cdf02bb16..7569adced 100644 --- a/typescript/packages/traceai_langchain/CHANGELOG.md +++ b/typescript/packages/traceai_langchain/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +## [0.1.8] - pending release + +### Fixed +- Normalized tool calls and the links from tool results back to their calls are recorded. +- Importing @traceai/langchain as an ES module no longer prints a TypeError stack trace on every run. + ## [0.1.6] - pending release ### Fix - Function-call arguments are kept in exported messages. Legacy OpenAI `function_call.arguments` was read as `args`, so `gen_ai.input.messages` / `gen_ai.output.messages` dropped the arguments; `args` is still accepted as a fallback. diff --git a/typescript/packages/traceai_langchain/README.md b/typescript/packages/traceai_langchain/README.md index 47529e398..ed9ac7e33 100644 --- a/typescript/packages/traceai_langchain/README.md +++ b/typescript/packages/traceai_langchain/README.md @@ -40,8 +40,8 @@ import * as ChatModelModule from "@langchain/core/language_models/chat_models"; import * as RunnableModule from "@langchain/core/runnables"; import { awaitAllCallbacks } from "@langchain/core/callbacks/promises"; -// Enable OpenTelemetry internal diagnostics (optional, for debugging) -diag.setLogger(new DiagConsoleLogger(), DiagLogLevel.DEBUG); +// For troubleshooting only: uncomment to print OpenTelemetry's internal debug logs. +// diag.setLogger(new DiagConsoleLogger(), DiagLogLevel.DEBUG); // 1. Register FI Core TracerProvider const tracerProvider = register({ diff --git a/typescript/packages/traceai_langchain/contract/lifecycle.mjs b/typescript/packages/traceai_langchain/contract/lifecycle.mjs index eba6cdde7..8531fcf2a 100644 --- a/typescript/packages/traceai_langchain/contract/lifecycle.mjs +++ b/typescript/packages/traceai_langchain/contract/lifecycle.mjs @@ -16,6 +16,7 @@ const { LangChainInstrumentation, isPatched } = await load("@traceai/langchain") const { BasicTracerProvider, SimpleSpanProcessor } = await load("@opentelemetry/sdk-trace-base"); const { OTLPTraceExporter } = await load("@opentelemetry/exporter-trace-otlp-http"); const { SemanticConventions } = await load("@traceai/fi-semantic-conventions"); +const { diag, DiagLogLevel } = await load("@opentelemetry/api"); const requests = []; const collector = createServer(async (request, response) => { @@ -45,14 +46,28 @@ const instrumentation = new LangChainInstrumentation({ tracerProvider: provider, traceConfig: hidden ? { hideInputs: true, hideOutputs: true, hideErrorDetails: true } : {}, }); +// Patching and unpatching must not log failures. An ES module namespace cannot +// take new properties, so setup must not try to store a flag on it. +const setupFailures = []; +let watchingSetup = false; +const watchSetup = (level) => (message, ...args) => { + if (watchingSetup && (level === "warn" || level === "error" || args.some((arg) => arg instanceof Error))) { + setupFailures.push(`${level}: ${message} ${args.join(" ")}`); + } +}; +diag.setLogger(Object.fromEntries( + ["error", "warn", "info", "debug", "verbose"].map((level) => [level, watchSetup(level)]), +), DiagLogLevel.ALL); try { + watchingSetup = true; // Loading these AFTER initialization is essential to the automatic hook test. const manager = await load("@langchain/core/callbacks/manager"); if (mode !== "automatic") { instrumentation.manuallyInstrument(manager); instrumentation.manuallyInstrument(manager); // Repeated setup must not duplicate spans. } + watchingSetup = false; assert.equal(isPatched(), true, `${mode} must patch core ${require("@langchain/core/package.json").version}`); const { RunnableLambda } = await load("@langchain/core/runnables"); const { awaitAllCallbacks } = await load("@langchain/core/callbacks/promises"); @@ -110,6 +125,10 @@ try { if (hidden) assert.ok(!JSON.stringify(requests).includes(secret), "private content must not reach OTLP"); else assert.ok(JSON.stringify(requests).includes(secret)); } + watchingSetup = true; + instrumentation.disable(); + watchingSetup = false; + assert.deepEqual(setupFailures, [], "patching and unpatching must not log failures"); console.log(JSON.stringify({ core: require("@langchain/core/package.json").version, mode, hidden, spans: names, result: "PASS" })); } finally { instrumentation.disable(); diff --git a/typescript/packages/traceai_langchain/contract/readme.mjs b/typescript/packages/traceai_langchain/contract/readme.mjs index f69c6a94f..c4c37d9c9 100644 --- a/typescript/packages/traceai_langchain/contract/readme.mjs +++ b/typescript/packages/traceai_langchain/contract/readme.mjs @@ -108,6 +108,7 @@ if (phase === "install") { assert.equal(code, 0, `README command failed (${signal}): ${stderr}`); } finally { clearTimeout(watchdog); } assert.ok(stdout.includes("Synthetic tracing example.")); + assert.doesNotMatch(stdout + stderr, /TypeError/, "README run must not print instrumentation errors"); assert.deepEqual(requests.sort(), ["/tracer/v1/traces", "/v1/chat/completions"]); assert.equal(spans.length, 1, "No missing or duplicate model span"); assert.equal(spans[0].name, "ChatOpenAI"); diff --git a/typescript/packages/traceai_langchain/package.json b/typescript/packages/traceai_langchain/package.json index 6dbf6397d..8e6b12c28 100644 --- a/typescript/packages/traceai_langchain/package.json +++ b/typescript/packages/traceai_langchain/package.json @@ -1,6 +1,6 @@ { "name": "@traceai/langchain", - "version": "0.1.7", + "version": "0.1.8", "description": "OpenTelemetry instrumentation for LangChain.js", "main": "./dist/src/index.js", "module": "./dist/esm/index.js", diff --git a/typescript/packages/traceai_langchain/src/__tests__/immutableModule.test.ts b/typescript/packages/traceai_langchain/src/__tests__/immutableModule.test.ts new file mode 100644 index 000000000..a689aa5fc --- /dev/null +++ b/typescript/packages/traceai_langchain/src/__tests__/immutableModule.test.ts @@ -0,0 +1,108 @@ +import { describe, it, expect, beforeEach, afterEach } from "@jest/globals"; +import { types } from "util"; +import * as CallbackManagerModule from "@langchain/core/callbacks/manager"; +import { awaitAllCallbacks } from "@langchain/core/callbacks/promises"; +import { RunnableLambda } from "@langchain/core/runnables"; +import { diag, DiagLogLevel } from "@opentelemetry/api"; +import { isWrapped } from "@opentelemetry/instrumentation"; +import { BasicTracerProvider, InMemorySpanExporter, SimpleSpanProcessor } from "@opentelemetry/sdk-trace-base"; +import { LangChainInstrumentation, isPatched, _resetPatchedStateForTesting } from "../instrumentation"; + +type DiagEntry = { level: string; message: string; args: unknown[] }; + +// Stands in for an ES module namespace (or a bundler's frozen exports): the +// object cannot gain properties, so assigning one throws in strict-mode code. +function moduleNamespace(exports: T): T { + const namespace: Record = Object.create(null); + for (const key of Object.keys(exports)) namespace[key] = (exports as Record)[key]; + Object.defineProperty(namespace, Symbol.toStringTag, { value: "Module" }); + return Object.freeze(namespace) as T; +} + +function captureDiag(): DiagEntry[] { + const entries: DiagEntry[] = []; + const record = (level: string) => (message: string, ...args: unknown[]) => { + entries.push({ level, message, args }); + }; + diag.disable(); + diag.setLogger({ + error: record("error"), + warn: record("warn"), + info: record("info"), + debug: record("debug"), + verbose: record("verbose"), + }, DiagLogLevel.ALL); + return entries; +} + +function problems(entries: DiagEntry[]) { + return entries + .filter(({ level, args }) => level === "warn" || level === "error" || args.some((arg) => types.isNativeError(arg))) + .map(({ level, message, args }) => `${level}: ${message} ${args.map(String).join(" ")}`); +} + +describe("instrumenting an immutable module namespace", () => { + // Newer @langchain/core wraps _configureSync; the 0.2.0 floor only has configure. + const configureSync = () => { + const manager = CallbackManagerModule.CallbackManager as unknown as Record; + return ("_configureSync" in manager ? manager._configureSync : manager.configure) as unknown as { __original?: unknown }; + }; + let exporter: InMemorySpanExporter; + let provider: BasicTracerProvider; + let instrumentation: LangChainInstrumentation; + let namespace: typeof CallbackManagerModule; + let logged: DiagEntry[]; + + beforeEach(() => { + _resetPatchedStateForTesting(); + exporter = new InMemorySpanExporter(); + provider = new BasicTracerProvider({ spanProcessors: [new SimpleSpanProcessor(exporter)] }); + instrumentation = new LangChainInstrumentation({ tracerProvider: provider }); + namespace = moduleNamespace(CallbackManagerModule); + logged = captureDiag(); + }); + + afterEach(async () => { + instrumentation.disable(); + diag.disable(); + await provider.shutdown(); + }); + + it("uses a namespace that rejects new properties", () => { + expect(Object.prototype.toString.call(namespace)).toBe("[object Module]"); + expect(() => { + (namespace as unknown as Record).fiPatched = true; + }).toThrow(TypeError); + }); + + it("patches and unpatches without throwing or logging failures", () => { + expect(() => instrumentation.manuallyInstrument(namespace)).not.toThrow(); + expect(isPatched()).toBe(true); + expect(isWrapped(configureSync())).toBe(true); + + expect(() => instrumentation.disable()).not.toThrow(); + expect(isPatched()).toBe(false); + expect(isWrapped(configureSync())).toBe(false); + expect(problems(logged)).toEqual([]); + }); + + it("wraps the callback manager once when instrumented twice", () => { + instrumentation.manuallyInstrument(namespace); + instrumentation.manuallyInstrument(namespace); + + expect(isWrapped(configureSync())).toBe(true); + expect(isWrapped(configureSync().__original)).toBe(false); + }); + + it("still exports one span per run", async () => { + instrumentation.manuallyInstrument(namespace); + instrumentation.manuallyInstrument(namespace); + + const chain = RunnableLambda.from((input: number) => input + 1).withConfig({ runName: "add_one" }); + expect(await chain.invoke(1)).toBe(2); + await awaitAllCallbacks(); + await provider.forceFlush(); + + expect(exporter.getFinishedSpans().map((span) => span.name)).toEqual(["add_one"]); + }); +}); diff --git a/typescript/packages/traceai_langchain/src/__tests__/instrumentation.test.ts b/typescript/packages/traceai_langchain/src/__tests__/instrumentation.test.ts index 354f6c74d..c826562d1 100644 --- a/typescript/packages/traceai_langchain/src/__tests__/instrumentation.test.ts +++ b/typescript/packages/traceai_langchain/src/__tests__/instrumentation.test.ts @@ -367,17 +367,36 @@ describe('LangChain Instrumentation', () => { }); describe('Module Flag Management', () => { - it('should set fiPatched flag on successful patch', () => { - const moduleWithFlag: any = { + it('should not write a patched flag onto the module', () => { + const writableModule: any = { CallbackManager: { configure: jest.fn(), _configureSync: jest.fn(), }, }; - // Ensure property is writable - Object.defineProperty(moduleWithFlag, 'fiPatched', { value: undefined, writable: true, configurable: true }); - instrumentation.manuallyInstrument(moduleWithFlag); - expect(moduleWithFlag.fiPatched).toBe(true); + instrumentation.manuallyInstrument(writableModule); + expect(isPatched()).toBe(true); + expect(Object.prototype.hasOwnProperty.call(writableModule, 'fiPatched')).toBe(false); + + instrumentation.disable(); + expect(isPatched()).toBe(false); + expect(Object.prototype.hasOwnProperty.call(writableModule, 'fiPatched')).toBe(false); + }); + + it('should respect a patched flag that is already set on the module', () => { + const configure = jest.fn(); + const configureSync = jest.fn(); + const flaggedModule = { + fiPatched: true, + CallbackManager: { + configure, + _configureSync: configureSync, + }, + }; + instrumentation.manuallyInstrument(flaggedModule as any); + expect(flaggedModule.CallbackManager.configure).toBe(configure); + expect(flaggedModule.CallbackManager._configureSync).toBe(configureSync); + expect(isPatched()).toBe(false); }); it('should handle flag setting failure gracefully', () => { diff --git a/typescript/packages/traceai_langchain/src/__tests__/normalizedToolCalls.test.ts b/typescript/packages/traceai_langchain/src/__tests__/normalizedToolCalls.test.ts new file mode 100644 index 000000000..7a20ce2ee --- /dev/null +++ b/typescript/packages/traceai_langchain/src/__tests__/normalizedToolCalls.test.ts @@ -0,0 +1,211 @@ +import { describe, expect, it } from "@jest/globals"; +import * as manager from "@langchain/core/callbacks/manager"; +import { awaitAllCallbacks } from "@langchain/core/callbacks/promises"; +import * as chatModels from "@langchain/core/language_models/chat_models"; +import { AIMessage, HumanMessage, ToolMessage } from "@langchain/core/messages"; +import * as runnables from "@langchain/core/runnables"; +import { + BasicTracerProvider, + InMemorySpanExporter, + SimpleSpanProcessor, +} from "@opentelemetry/sdk-trace-base"; +import { SemanticConventions as SC } from "@traceai/fi-semantic-conventions"; +import { LangChainInstrumentation } from "../instrumentation"; +import { safelyFormatInputMessages, safelyFormatOutputMessages } from "../utils"; + +const normalizedCalls = [{ + type: "tool_call" as const, + id: "call_demo", + name: "lookup", + args: { city: "Paris" }, +}]; +const canonicalCalls = [{ + id: "call_demo", + function: { name: "lookup", arguments: JSON.stringify({ city: "Paris" }) }, +}]; +const legacyCalls = canonicalCalls.map((call) => ({ ...call, type: "function" })); + +function aiMessage( + kwargs: Record, + fields: Record = {}, +) { + return { + lc_id: ["langchain_core", "messages", "AIMessage"], + lc_kwargs: { content: "", ...kwargs }, + ...fields, + }; +} + +function serializedMessages( + direction: "input" | "output", + messages: Record[], +) { + const attributes = direction === "input" + ? safelyFormatInputMessages({ messages: [messages] }) + : safelyFormatOutputMessages({ generations: [messages.map((message) => ({ message }))] }); + const key = direction === "input" ? SC.LLM_INPUT_MESSAGES : SC.LLM_OUTPUT_MESSAGES; + const serialized = (attributes as Record | null)?.[key]; + expect(typeof serialized).toBe("string"); + if (typeof serialized !== "string") throw new Error("Missing canonical messages"); + return JSON.parse(serialized); +} + +const cases = [ + { + name: "normalized lc_kwargs without additional kwargs", + message: aiMessage({ tool_calls: normalizedCalls }), + }, + { + name: "normalized message fields", + message: aiMessage({ additional_kwargs: {} }, { tool_calls: normalizedCalls }), + }, + { + name: "normalized fields without lc_kwargs", + message: { + lc_id: ["langchain_core", "messages", "AIMessage"], + content: "", + tool_calls: normalizedCalls, + }, + }, + { + name: "normalized calls instead of a duplicate legacy copy", + message: aiMessage({ + tool_calls: normalizedCalls, + additional_kwargs: { tool_calls: legacyCalls }, + }), + }, + { + name: "current message fields ahead of serialized and legacy copies", + message: aiMessage({ + tool_calls: [{ name: "stale", args: { city: "Rome" }, id: "call_stale" }], + additional_kwargs: { tool_calls: legacyCalls }, + }, { tool_calls: normalizedCalls }), + }, + { + name: "serialized normalized calls when message fields are empty", + message: aiMessage({ tool_calls: normalizedCalls }, { tool_calls: [] }), + }, + { + name: "legacy fallback without normalized calls", + message: aiMessage({ additional_kwargs: { tool_calls: legacyCalls } }), + }, + { + name: "legacy fallback when normalized arrays are empty", + message: aiMessage({ + tool_calls: [], + additional_kwargs: { tool_calls: legacyCalls }, + }, { tool_calls: [] }), + }, +]; + +describe("T7-TS-LANGCHAIN-001 normalized tool calls", () => { + for (const direction of ["input", "output"] as const) { + for (const { name, message } of cases) { + it(`${direction} preserves ${name} without mutation or duplication`, () => { + const before = JSON.stringify(message); + expect(serializedMessages(direction, [message])).toEqual([ + { role: "assistant", tool_calls: canonicalCalls }, + ]); + expect(JSON.stringify(message)).toBe(before); + }); + } + + it(`${direction} serializes object arguments once and preserves call order and optional IDs`, () => { + const args = { cities: ["Paris", "Rome"], options: { limit: 2 }, enabled: true }; + const message = aiMessage({ tool_calls: [ + { name: "lookup", args, id: "call_first" }, + { name: "finish", args: {} }, + ] }); + expect(serializedMessages(direction, [message])).toEqual([{ + role: "assistant", + tool_calls: [ + { id: "call_first", function: { name: "lookup", arguments: JSON.stringify(args) } }, + { function: { name: "finish", arguments: "{}" } }, + ], + }]); + }); + + for (const location of ["message", "lc_kwargs"] as const) { + it(`${direction} preserves the tool result link from ${location}`, () => { + const message = { + lc_id: ["langchain_core", "messages", "ToolMessage"], + lc_kwargs: { + content: "Sunny", + ...(location === "lc_kwargs" ? { tool_call_id: "call_demo" } : {}), + }, + ...(location === "message" ? { tool_call_id: "call_demo" } : {}), + }; + expect(serializedMessages(direction, [message])).toEqual([ + { role: "tool", content: "Sunny", tool_call_id: "call_demo" }, + ]); + }); + } + + it(`${direction} retains legacy names and argument strings without an ID`, () => { + const message = aiMessage({ additional_kwargs: { tool_calls: [{ + function: { name: "lookup", arguments: '{ "city": "Paris" }' }, + }] } }); + expect(serializedMessages(direction, [message])).toEqual([{ + role: "assistant", + tool_calls: [{ function: { name: "lookup", arguments: '{ "city": "Paris" }' } }], + }]); + }); + } + + it("exports normalized-only BaseChatModel calls and their subsequent tool result links", async () => { + // Fake the provider, not LangChain's message or callback machinery. No network. + class ToolModel extends chatModels.BaseChatModel { + constructor() { super({}); } + _llmType() { return "offline-tool-model"; } + async _generate() { + return { generations: [{ + text: "", + message: new AIMessage({ content: "", tool_calls: normalizedCalls }), + }] }; + } + } + + const exporter = new InMemorySpanExporter(); + const provider = new BasicTracerProvider({ + spanProcessors: [new SimpleSpanProcessor(exporter)], + }); + const instrumentation = new LangChainInstrumentation({ + tracerProvider: provider, + instrumentationConfig: { enabled: false }, + }); + try { + instrumentation.manuallyInstrument(manager, chatModels, runnables); + const model = new ToolModel(); + const human = new HumanMessage("lookup Paris"); + const response = await model.invoke([human]); + expect(response).toBeInstanceOf(AIMessage); + if (!(response instanceof AIMessage)) throw new Error("Expected AIMessage"); + expect(response.tool_calls).toEqual(normalizedCalls); + expect(response.additional_kwargs.tool_calls).toBeUndefined(); + + await model.invoke([ + human, + response, + new ToolMessage({ content: "Sunny", tool_call_id: "call_demo" }), + ]); + await awaitAllCallbacks(); + await provider.forceFlush(); + + const spans = exporter.getFinishedSpans(); + expect(spans).toHaveLength(2); + expect(JSON.parse(spans[0].attributes[SC.LLM_OUTPUT_MESSAGES] as string)).toEqual([ + { role: "assistant", tool_calls: canonicalCalls }, + ]); + expect(JSON.parse(spans[1].attributes[SC.LLM_INPUT_MESSAGES] as string)).toEqual([ + { role: "user", content: "lookup Paris" }, + { role: "assistant", tool_calls: canonicalCalls }, + { role: "tool", content: "Sunny", tool_call_id: "call_demo" }, + ]); + expect(response.tool_calls).toEqual(normalizedCalls); + } finally { + await awaitAllCallbacks(); + instrumentation.disable(); + await provider.shutdown(); + } + }); +}); diff --git a/typescript/packages/traceai_langchain/src/instrumentation.ts b/typescript/packages/traceai_langchain/src/instrumentation.ts index b89ea39a8..9ec362169 100644 --- a/typescript/packages/traceai_langchain/src/instrumentation.ts +++ b/typescript/packages/traceai_langchain/src/instrumentation.ts @@ -19,10 +19,12 @@ const MODULE_NAME = "@langchain/core/callbacks"; const INSTRUMENTATION_NAME = "@traceai/langchain"; /** - * Flag to check if the openai module has been patched - * Note: This is a fallback in case the module is made immutable (e.x. Deno, webpack, etc.) + * Flag to check if the callback manager module has been patched. + * Patched state is kept here, never on the module: ES module namespaces and + * frozen bundler exports cannot take new properties. */ let _isFIPatched = false; +let _patchedModules = new WeakSet(); /** * function to check if instrumentation is enabled / disabled @@ -36,6 +38,7 @@ export function isPatched() { */ export function _resetPatchedStateForTesting(): void { _isFIPatched = false; + _patchedModules = new WeakSet(); } type CallbackManagerModule = typeof CallbackManagerModuleV02; @@ -239,7 +242,8 @@ export class LangChainInstrumentation extends InstrumentationBase { moduleVersion != null ? `@${moduleVersion}` : "" }`, ); - if (module?.fiPatched || _isFIPatched) { + // A flag already set on the module is still honored. + if (module?.fiPatched || _isFIPatched || _patchedModules.has(module)) { return module; } if (module?.CallbackManager == null) { @@ -283,12 +287,7 @@ export class LangChainInstrumentation extends InstrumentationBase { }); } _isFIPatched = true; - try { - // This can fail if the module is made immutable via the runtime or bundler - module.fiPatched = true; - } catch (e) { - diag.debug(`Failed to set ${MODULE_NAME} patched flag on the module`, e); - } + _patchedModules.add(module); return module; } @@ -321,12 +320,7 @@ export class LangChainInstrumentation extends InstrumentationBase { this._unwrap(module.CallbackManager, "_configureSync"); } _isFIPatched = false; - try { - // This can fail if the module is made immutable via the runtime or bundler - module.fiPatched = false; - } catch (e) { - diag.warn(`Failed to unset ${MODULE_NAME} patched flag on the module`, e); - } + _patchedModules.delete(module); return module; } } \ No newline at end of file diff --git a/typescript/packages/traceai_langchain/src/types.ts b/typescript/packages/traceai_langchain/src/types.ts index 959910237..29ba7a234 100644 --- a/typescript/packages/traceai_langchain/src/types.ts +++ b/typescript/packages/traceai_langchain/src/types.ts @@ -1,6 +1,7 @@ import { SemanticConventions } from "@traceai/fi-semantic-conventions"; type LLMMessageToolCall = { + [SemanticConventions.TOOL_CALL_ID]?: string; [SemanticConventions.TOOL_CALL_FUNCTION_NAME]?: string; [SemanticConventions.TOOL_CALL_FUNCTION_ARGUMENTS_JSON]?: string; }; @@ -18,6 +19,7 @@ export type LLMMessage = LLMMessageToolCalls & LLMMessageFunctionCall & { [SemanticConventions.MESSAGE_ROLE]?: string; [SemanticConventions.MESSAGE_CONTENT]?: string; + [SemanticConventions.MESSAGE_TOOL_CALL_ID]?: string; }; export type LLMMessagesAttributes = diff --git a/typescript/packages/traceai_langchain/src/utils.ts b/typescript/packages/traceai_langchain/src/utils.ts index d0b9edb21..7c03a8743 100644 --- a/typescript/packages/traceai_langchain/src/utils.ts +++ b/typescript/packages/traceai_langchain/src/utils.ts @@ -229,28 +229,34 @@ function getFunctionCallDataFromAdditionalKwargs( } /** - * Gets the tool calls from the langchain message additional kwargs and formats them into FI attributes. - * @param additionalKwargs - The langchain message additional kwargs to extract the tool calls from - * @returns the FI attributes for the tool calls + * Formats provider-neutral tool calls or the legacy OpenAI additional kwargs shape. + * Normalized calls contain object `args`; legacy calls contain string `function.arguments`. */ -function getToolCallDataFromAdditionalKwargs( - additionalKwargs: Record, +function getToolCallData( + toolCalls: unknown, + normalized: boolean, ): LLMMessageToolCalls { - const toolCalls = additionalKwargs.tool_calls; if (!Array.isArray(toolCalls)) { return {}; } const formattedToolCalls = toolCalls.map((toolCall) => { - if (!isObject(toolCall) && !isObject(toolCall.function)) { + if (!isObject(toolCall)) { return {}; } - const toolCallName = isString(toolCall.function.name) - ? toolCall.function.name - : undefined; - const toolCallArgs = isString(toolCall.function.arguments) - ? toolCall.function.arguments + const functionCall = normalized ? toolCall : toolCall.function; + if (!isObject(functionCall)) { + return {}; + } + const toolCallName = isString(functionCall.name) + ? functionCall.name : undefined; + const toolCallArgs = normalized + ? safelyJSONStringify(functionCall.args) ?? undefined + : isString(functionCall.arguments) + ? functionCall.arguments + : undefined; return { + [SemanticConventions.TOOL_CALL_ID]: isString(toolCall.id) ? toolCall.id : undefined, [SemanticConventions.TOOL_CALL_FUNCTION_NAME]: toolCallName, [SemanticConventions.TOOL_CALL_FUNCTION_ARGUMENTS_JSON]: toolCallArgs, }; @@ -273,23 +279,36 @@ function parseMessage(messageData: Record): LLMMessage { message[SemanticConventions.MESSAGE_ROLE] = maybeRole; } - const messageKwargs = messageData.lc_kwargs; - if (!isObject(messageKwargs)) { - return message; - } + const messageKwargs = isObject(messageData.lc_kwargs) + ? messageData.lc_kwargs + : messageData; const maybeContent = getContentFromMessageData(messageKwargs); if (maybeContent != null) { message[SemanticConventions.MESSAGE_CONTENT] = maybeContent; } - - const additionalKwargs = messageKwargs.additional_kwargs; - if (!isObject(additionalKwargs)) { - return message; - } + const toolCallId = isString(messageData.tool_call_id) + ? messageData.tool_call_id + : messageKwargs.tool_call_id; + if (isString(toolCallId)) { + message[SemanticConventions.MESSAGE_TOOL_CALL_ID] = toolCallId; + } + + const additionalKwargs: Record = isObject(messageKwargs.additional_kwargs) + ? messageKwargs.additional_kwargs + : {}; + // Runtime fields are authoritative; serialized messages retain these in lc_kwargs. + // Empty normalized arrays are common defaults on legacy adapters. Fall back in + // that case, but never append a duplicate legacy copy to normalized calls. + const normalizedToolCalls = isNonEmptyArray(messageData.tool_calls) + ? messageData.tool_calls + : messageKwargs.tool_calls; + const toolCalls = isNonEmptyArray(normalizedToolCalls) + ? getToolCallData(normalizedToolCalls, true) + : getToolCallData(additionalKwargs.tool_calls, false); return { ...message, ...getFunctionCallDataFromAdditionalKwargs(additionalKwargs), - ...getToolCallDataFromAdditionalKwargs(additionalKwargs), + ...toolCalls, }; } @@ -326,6 +345,7 @@ function formatInputMessages( const obj: Record = {}; if (msg[SemanticConventions.MESSAGE_ROLE]) obj.role = msg[SemanticConventions.MESSAGE_ROLE]; if (msg[SemanticConventions.MESSAGE_CONTENT]) obj.content = msg[SemanticConventions.MESSAGE_CONTENT]; + if (msg[SemanticConventions.MESSAGE_TOOL_CALL_ID] != null) obj.tool_call_id = msg[SemanticConventions.MESSAGE_TOOL_CALL_ID]; if (msg[SemanticConventions.MESSAGE_FUNCTION_CALL_NAME]) { obj.function_call = { name: msg[SemanticConventions.MESSAGE_FUNCTION_CALL_NAME], @@ -334,6 +354,7 @@ function formatInputMessages( } if (msg[SemanticConventions.MESSAGE_TOOL_CALLS]) { obj.tool_calls = msg[SemanticConventions.MESSAGE_TOOL_CALLS]?.map((tc) => ({ + id: tc[SemanticConventions.TOOL_CALL_ID], function: { name: tc[SemanticConventions.TOOL_CALL_FUNCTION_NAME], arguments: tc[SemanticConventions.TOOL_CALL_FUNCTION_ARGUMENTS_JSON], @@ -395,6 +416,7 @@ function formatOutputMessages( const obj: Record = {}; if (msg[SemanticConventions.MESSAGE_ROLE]) obj.role = msg[SemanticConventions.MESSAGE_ROLE]; if (msg[SemanticConventions.MESSAGE_CONTENT]) obj.content = msg[SemanticConventions.MESSAGE_CONTENT]; + if (msg[SemanticConventions.MESSAGE_TOOL_CALL_ID] != null) obj.tool_call_id = msg[SemanticConventions.MESSAGE_TOOL_CALL_ID]; if (msg[SemanticConventions.MESSAGE_FUNCTION_CALL_NAME]) { obj.function_call = { name: msg[SemanticConventions.MESSAGE_FUNCTION_CALL_NAME], @@ -403,6 +425,7 @@ function formatOutputMessages( } if (msg[SemanticConventions.MESSAGE_TOOL_CALLS]) { obj.tool_calls = msg[SemanticConventions.MESSAGE_TOOL_CALLS]?.map((tc) => ({ + id: tc[SemanticConventions.TOOL_CALL_ID], function: { name: tc[SemanticConventions.TOOL_CALL_FUNCTION_NAME], arguments: tc[SemanticConventions.TOOL_CALL_FUNCTION_ARGUMENTS_JSON], diff --git a/typescript/packages/traceai_langchain/src/version.ts b/typescript/packages/traceai_langchain/src/version.ts index 51ba382b6..9f12ba09b 100644 --- a/typescript/packages/traceai_langchain/src/version.ts +++ b/typescript/packages/traceai_langchain/src/version.ts @@ -1 +1 @@ -export const VERSION = "0.1.7"; // x-release-please-version \ No newline at end of file +export const VERSION = "0.1.8"; // x-release-please-version \ No newline at end of file diff --git a/typescript/packages/traceai_n8n/CHANGELOG.md b/typescript/packages/traceai_n8n/CHANGELOG.md index 7d1479099..a54ea854e 100644 --- a/typescript/packages/traceai_n8n/CHANGELOG.md +++ b/typescript/packages/traceai_n8n/CHANGELOG.md @@ -1,6 +1,12 @@ # Changelog -## 0.1.0 (unreleased) +## [0.1.1] - pending release + +### Fixed + +- The README shows how to install the package from npm instead of building it from source. + +## [0.1.0] - 2026-11-09 - Add workflow and node execution attempt spans for `n8n-core >=2.16.1 <3`. - Preserve workflow results, cancellable promises, parent context and retry diff --git a/typescript/packages/traceai_n8n/README.md b/typescript/packages/traceai_n8n/README.md index 5e34c434c..88f03fb71 100644 --- a/typescript/packages/traceai_n8n/README.md +++ b/typescript/packages/traceai_n8n/README.md @@ -14,23 +14,13 @@ under the active node span. ESM package entry points are available, but native ESM loading of n8n itself is not covered by the automatic loader hook. -This package is a source contribution and has not been released to npm yet. The -commands below build it from this repository: +Install it in the same installation that resolves `n8n-core`, together with the +OpenTelemetry packages the setup below uses: ```sh -cd typescript -pnpm --filter '@traceai/n8n...' install --frozen-lockfile --ignore-scripts -pnpm run build:deps -pnpm --filter @traceai/n8n build -pnpm --filter @traceai/n8n pack --pack-destination /path/to/artifacts +npm install @traceai/n8n @traceai/fi-core @opentelemetry/api @opentelemetry/context-async-hooks @opentelemetry/instrumentation ``` -Install the resulting tarball in the same application installation that resolves -`n8n-core`. The workspace dependencies must also be available there; when building -them from source, pack and install `@traceai/fi-semantic-conventions` and -`@traceai/fi-core` first. On Windows, use Git Bash for the existing dependency -build scripts. - ## Self-hosted setup Load instrumentation **before** n8n loads its engine, in every process that diff --git a/typescript/packages/traceai_n8n/package.json b/typescript/packages/traceai_n8n/package.json index 6609badd0..2370fb3d2 100644 --- a/typescript/packages/traceai_n8n/package.json +++ b/typescript/packages/traceai_n8n/package.json @@ -1,6 +1,6 @@ { "name": "@traceai/n8n", - "version": "0.1.0", + "version": "0.1.1", "description": "OpenTelemetry instrumentation for n8n workflow execution", "main": "./dist/src/index.js", "module": "./dist/esm/index.js", diff --git a/typescript/packages/traceai_n8n/src/instrumentation.ts b/typescript/packages/traceai_n8n/src/instrumentation.ts index 28081af2a..b0e609576 100644 --- a/typescript/packages/traceai_n8n/src/instrumentation.ts +++ b/typescript/packages/traceai_n8n/src/instrumentation.ts @@ -23,7 +23,7 @@ import { import type { WorkflowExecute } from "n8n-core"; import type { IRun, Workflow } from "n8n-workflow"; -const VERSION = "0.1.0"; // x-release-please-version +const VERSION = "0.1.1"; // x-release-please-version const SUPPORTED_VERSIONS = [">=2.16.1 <3"]; const EXECUTION = createContextKey("traceai.n8n.execution"); diff --git a/typescript/packages/traceai_openai/CHANGELOG.md b/typescript/packages/traceai_openai/CHANGELOG.md index 7834a5cb6..3dd0123f9 100644 --- a/typescript/packages/traceai_openai/CHANGELOG.md +++ b/typescript/packages/traceai_openai/CHANGELOG.md @@ -1,3 +1,9 @@ +## [0.1.18] - pending release + +### Fixed +- For non-streaming chat completion, completion, Responses and embedding calls, a provider error returned with HTTP 200 comes back to your code unchanged and is recorded as an error span, instead of the call throwing a TypeError. +- Token usage for embedding requests is recorded. + ## [0.1.17] - pending release ### Feature - Embeddings now also set `gen_ai.request.model` from the request and `gen_ai.response.model` from the response when present, while retaining `embedding.model_name`. diff --git a/typescript/packages/traceai_openai/package.json b/typescript/packages/traceai_openai/package.json index 5ffd2da06..384a59239 100644 --- a/typescript/packages/traceai_openai/package.json +++ b/typescript/packages/traceai_openai/package.json @@ -1,6 +1,6 @@ { "name": "@traceai/openai", - "version": "0.1.17", + "version": "0.1.18", "description": "OpenTelemetry instrumentation for OpenAI", "main": "./dist/src/index.js", "module": "./dist/esm/index.js", diff --git a/typescript/packages/traceai_openai/src/__tests__/embeddingUsage.test.ts b/typescript/packages/traceai_openai/src/__tests__/embeddingUsage.test.ts new file mode 100644 index 000000000..f1b81d35d --- /dev/null +++ b/typescript/packages/traceai_openai/src/__tests__/embeddingUsage.test.ts @@ -0,0 +1,85 @@ +import { SpanStatusCode } from "@opentelemetry/api"; +import { + BasicTracerProvider, + InMemorySpanExporter, + SimpleSpanProcessor, +} from "@opentelemetry/sdk-trace-base"; +import { privacyFilter, REDACTED_VALUE } from "@traceai/fi-core"; +import { OpenAIInstrumentation } from "../instrumentation"; + +// Use the real SDK parser with a fake transport; no network requests are made. +const sdk = require(process.env.OPENAI_TEST_MODULE || "openai"); +const savedEnv = { ...process.env }; +beforeEach(() => { + for (const key of Object.keys(process.env)) { + if (/^(FI_|OTEL_)/.test(key)) delete process.env[key]; + } +}); +afterEach(() => { process.env = { ...savedEnv }; }); + +const usageRows = [ + { label: "ordinary usage", usage: { prompt_tokens: 7, total_tokens: 7 } }, + { label: "distinct provider totals", usage: { prompt_tokens: 7, total_tokens: 11 } }, + { label: "zero usage", usage: { prompt_tokens: 0, total_tokens: 0 } }, + { label: "omitted usage", usage: undefined }, +]; + +describe.each([false, true])("T7-TS-OPENAI-002 embedding usage (hideOutputs=%s)", (hideOutputs) => { + test.each(usageRows)("retains $label without changing the response", async ({ usage }) => { + const exporter = new InMemorySpanExporter(); + const provider = new BasicTracerProvider({ + spanProcessors: [new SimpleSpanProcessor(exporter)], + }); + const traceConfig = { hideInputs: false, hideOutputs, hideEmbeddingVectors: false }; + const instrumentation = new OpenAIInstrumentation({ traceConfig }); + try { + instrumentation.setTracerProvider(provider); + instrumentation.manuallyInstrument(sdk); + const body = { + model: "text-embedding-3-small", + input: "hello", + encoding_format: "float" as const, + }; + const response = { + object: "list", + model: body.model, + data: [{ object: "embedding", index: 0, embedding: [0.1, 0.2] }], + ...(usage === undefined ? {} : { usage }), + }; + const fetch = jest.fn(async (_url: unknown, _init?: RequestInit) => new Response(JSON.stringify(response), { + headers: { "content-type": "application/json" }, + })); + const client = new sdk.OpenAI({ apiKey: "FAKE_OPENAI_KEY", maxRetries: 0, fetch }); + const result = await client.embeddings.create(body); + expect(fetch).toHaveBeenCalledTimes(1); + expect(JSON.parse(fetch.mock.calls[0][1]!.body as string)).toEqual(body); + expect(result).toEqual(response); + expect(result.usage).toEqual(usage); + await provider.forceFlush(); + expect(exporter.getFinishedSpans()).toHaveLength(1); + const native = exporter.getFinishedSpans()[0]; + expect(native.name).toBe("OpenAI Embeddings"); + expect(native.status.code).toBe(SpanStatusCode.OK); + if (hideOutputs) { + expect(native.attributes["raw.output"]).toBe(REDACTED_VALUE); + } else { + expect(JSON.parse(native.attributes["raw.output"] as string)).toEqual(response); + } + + // Canonical numeric usage must survive both native and FI export paths, + // even when raw.output is hidden. Never invent output tokens or totals. + const projected = privacyFilter(traceConfig).span(native); + for (const span of [native, projected]) { + expect(span.attributes["gen_ai.usage.input_tokens"]).toBe(usage?.prompt_tokens); + expect(span.attributes["gen_ai.usage.total_tokens"]).toBe(usage?.total_tokens); + expect(span.attributes["gen_ai.usage.output_tokens"]).toBeUndefined(); + if (hideOutputs) { + expect(span.attributes["raw.output"]).toBe(REDACTED_VALUE); + } + } + } finally { + instrumentation.disable(); + await provider.shutdown(); + } + }); +}); diff --git a/typescript/packages/traceai_openai/src/__tests__/errorBodyResponses.test.ts b/typescript/packages/traceai_openai/src/__tests__/errorBodyResponses.test.ts new file mode 100644 index 000000000..938e3325b --- /dev/null +++ b/typescript/packages/traceai_openai/src/__tests__/errorBodyResponses.test.ts @@ -0,0 +1,318 @@ +import { SpanStatusCode } from "@opentelemetry/api"; +import { + BasicTracerProvider, + InMemorySpanExporter, + SimpleSpanProcessor, +} from "@opentelemetry/sdk-trace-base"; +import { OpenAIInstrumentation } from "../instrumentation"; + +// Real SDK parsing and streaming, with an in-memory transport only. +const sdk = require(process.env.OPENAI_TEST_MODULE || "openai"); +type API = "chat" | "completion" | "responses"; +const apis: API[] = ["chat", "completion", "responses"]; +const errorBody = { error: { message: "synthetic provider failure", type: "server_error" } }; +const model = "fixture-model"; +const completed = { + chat: { + id: "chat-fixture", object: "chat.completion", created: 1, model, + choices: [{ index: 0, message: { role: "assistant", content: "hello" }, finish_reason: "stop" }], + }, + completion: { + id: "completion-fixture", object: "text_completion", created: 1, model, + choices: [{ index: 0, text: "hello", finish_reason: "stop", logprobs: null }], + }, + responses: { + id: "response-fixture", object: "response", status: "completed", model, + output: [{ type: "message", role: "assistant", content: [{ type: "output_text", text: "hello" }] }], + }, +}; +const spanNames = { + chat: "OpenAI Chat Completions", + completion: "OpenAI Completions", + responses: "OpenAI Responses", + embeddings: "OpenAI Embeddings", +}; + +function create(client: any, api: API, stream?: boolean) { + const options = stream === undefined ? {} : { stream }; + if (api === "chat") return client.chat.completions.create({ + model, messages: [{ role: "user", content: "hello" }], ...options, + }); + if (api === "completion") return client.completions.create({ model, prompt: "hello", ...options }); + return client.responses.create({ model, input: "hello", ...options }); +} + +function clientFor(payload: unknown, status = 200, sse = false) { + const fetch = jest.fn(async () => new Response(sse ? payload as string : JSON.stringify(payload), { + status, + headers: { "content-type": sse ? "text/event-stream" : "application/json" }, + })); + return { + client: new sdk.OpenAI({ apiKey: "synthetic-key", baseURL: "http://provider.invalid/v1", maxRetries: 0, fetch }), + fetch, + }; +} + +let exporter: InMemorySpanExporter; +let provider: BasicTracerProvider; +let instrumentation: OpenAIInstrumentation; +const savedEnv = { ...process.env }; +beforeEach(() => { + for (const key of Object.keys(process.env)) { + if (/^(FI_|OTEL_)/.test(key)) delete process.env[key]; + } + exporter = new InMemorySpanExporter(); + provider = new BasicTracerProvider({ spanProcessors: [new SimpleSpanProcessor(exporter)] }); + instrumentation = new OpenAIInstrumentation({ instrumentationConfig: { enabled: false } }); + instrumentation.setTracerProvider(provider); + instrumentation.manuallyInstrument(sdk); +}); +afterEach(async () => { + instrumentation.disable(); + await provider.shutdown(); + process.env = { ...savedEnv }; +}); + +async function onlySpan(api: API | "embeddings") { + await provider.forceFlush(); + const spans = exporter.getFinishedSpans(); + expect(spans).toHaveLength(1); + expect(spans[0].name).toBe(spanNames[api]); + return spans[0]; +} + +function expectNoOutput(attributes: Record) { + for (const name of ["output.value", "output.mime_type", "raw.output", "llm.output_messages"]) { + expect(attributes[name]).toBeUndefined(); + } +} + +describe.each(apis)("non-stream response regressions: %s", api => { + test.each([undefined, false])("returns an HTTP 200 error body and exports its error (stream=%s)", async stream => { + const { client, fetch } = clientFor(errorBody); + const promise = create(client, api, stream); + const result = await promise; + expect(result).toEqual(errorBody); + expect(await promise).toBe(result); + expect(fetch).toHaveBeenCalledTimes(1); + const span = await onlySpan(api); + expect(span.status).toEqual({ code: SpanStatusCode.ERROR, message: errorBody.error.message }); + expectNoOutput(span.attributes); + }); + + test("preserves withResponse data and exports the provider error once", async () => { + const { client, fetch } = clientFor(errorBody); + const promise = create(client, api); + const { data, response } = await promise.withResponse(); + expect(data).toEqual(errorBody); + expect(response.status).toBe(200); + expect(await promise).toBe(data); + expect(fetch).toHaveBeenCalledTimes(1); + expect((await onlySpan(api)).status).toEqual({ code: SpanStatusCode.ERROR, message: errorBody.error.message }); + }); + + test.each([ + { label: "empty object", body: {} }, + { label: "unknown envelope", body: { metadata: "accepted" } }, + { label: "non-callable iterator", body: { iterator: "not-a-function" } }, + { label: "null", body: null }, + { label: "primitive", body: "unexpected response" }, + { label: "invalid choices", body: { choices: null } }, + // The Responses SDK itself rejects null/primitives before instrumentation. + ].filter(({ body }) => api !== "responses" || (body !== null && typeof body === "object")))("returns $label without output attributes or a telemetry exception", async ({ body }) => { + const { client, fetch } = clientFor(body); + await expect(create(client, api)).resolves.toEqual(body); + expect(fetch).toHaveBeenCalledTimes(1); + const span = await onlySpan(api); + expect(span.status.code).toBe(SpanStatusCode.UNSET); + expectNoOutput(span.attributes); + }); +}); + +const encodings = [undefined, "float", "base64"] as const; +function createEmbedding(client: any, encoding?: typeof encodings[number]) { + return client.embeddings.create({ + model, input: "hello", ...(encoding === undefined ? {} : { encoding_format: encoding }), + }); +} + +describe("non-stream response regressions: embeddings", () => { + test.each(encodings)("returns an HTTP 200 error body unchanged (encoding=%s)", async encoding => { + const { client, fetch } = clientFor(errorBody); + const promise = createEmbedding(client, encoding); + const result = await promise; + expect(result).toEqual(errorBody); + expect(await promise).toBe(result); + expect(fetch).toHaveBeenCalledTimes(1); + const span = await onlySpan("embeddings"); + expect(span.status).toEqual({ code: SpanStatusCode.ERROR, message: errorBody.error.message }); + expectNoOutput(span.attributes); + }); + + test.each(encodings)("preserves withResponse data and ends once (encoding=%s)", async encoding => { + const { client, fetch } = clientFor(errorBody); + const promise = createEmbedding(client, encoding); + const { data, response } = await promise.withResponse(); + expect(data).toEqual(errorBody); + expect(response.status).toBe(200); + expect(await promise).toBe(data); + expect((await promise.withResponse()).data).toBe(data); + expect(fetch).toHaveBeenCalledTimes(1); + const span = await onlySpan("embeddings"); + expect(span.status).toEqual({ code: SpanStatusCode.ERROR, message: errorBody.error.message }); + expectNoOutput(span.attributes); + }); + + test("prioritizes the error over otherwise valid embedding data", async () => { + const body = { ...errorBody, data: [{ index: 0, embedding: [0.5] }] }; + const { client } = clientFor(body); + await expect(createEmbedding(client, "float")).resolves.toEqual(body); + const span = await onlySpan("embeddings"); + expect(span.status).toEqual({ code: SpanStatusCode.ERROR, message: errorBody.error.message }); + expect(span.attributes["embedding.embeddings.0.embedding.vector"]).toBeUndefined(); + expectNoOutput(span.attributes); + }); + + test.each([ + { label: "missing data", body: {} }, + { label: "null data", body: { data: null } }, + { label: "object data", body: { data: {} } }, + { label: "string data", body: { data: "unexpected" } }, + ])("does not throw while observing $label", async ({ body }) => { + // Explicit encoding bypasses the SDK's native base64 decoding validation. + const { client } = clientFor(body); + await expect(createEmbedding(client, "float")).resolves.toEqual(body); + const span = await onlySpan("embeddings"); + expect(span.attributes["embedding.embeddings.0.embedding.vector"]).toBeUndefined(); + }); +}); + +describe("base controls: embeddings", () => { + test.each(encodings)("the uninstrumented SDK returns the error body (encoding=%s)", async encoding => { + instrumentation.disable(); + const { client, fetch } = clientFor(errorBody); + await expect(createEmbedding(client, encoding)).resolves.toEqual(errorBody); + expect(fetch).toHaveBeenCalledTimes(1); + await provider.forceFlush(); + expect(exporter.getFinishedSpans()).toHaveLength(0); + }); + + test.each(encodings)("asResponse leaves the error body unread (encoding=%s)", async encoding => { + const { client, fetch } = clientFor(errorBody); + const promise = createEmbedding(client, encoding); + const response = await promise.asResponse(); + expect(response.status).toBe(200); + expect(response.bodyUsed).toBe(false); + expect(await promise.asResponse()).toBe(response); + await expect(response.json()).resolves.toEqual(errorBody); + expect(fetch).toHaveBeenCalledTimes(1); + // Raw-only access records HTTP metadata; telemetry must not parse the body. + const span = await onlySpan("embeddings"); + expect(span.status.code).toBe(SpanStatusCode.OK); + expect(span.attributes["http.response.status_code"]).toBe(200); + expectNoOutput(span.attributes); + }); + + test("normal embeddings keep vector and usage attributes", async () => { + const body = { object: "list", model, data: [{ object: "embedding", index: 0, embedding: [0.5] }], + usage: { prompt_tokens: 1, total_tokens: 1 } }; + const { client } = clientFor(body); + await expect(createEmbedding(client, "float")).resolves.toEqual(body); + const span = await onlySpan("embeddings"); + expect(span.status.code).toBe(SpanStatusCode.OK); + expect(span.attributes["embedding.embeddings.0.embedding.vector"]).toEqual([0.5]); + expect(span.attributes["gen_ai.usage.total_tokens"]).toBe(1); + }); +}); + +describe("base controls: streaming chat errors", () => { + test.each([ + { label: "first error envelope", event: "", afterContent: false }, + { label: "first named error event", event: "event: error\n", afterContent: false }, + { label: "error envelope after content", event: "", afterContent: true }, + { label: "named error event after content", event: "event: error\n", afterContent: true }, + ])("preserves the SDK rejection for $label", async ({ event, afterContent }) => { + const chunk = { id: "chat-fixture", object: "chat.completion.chunk", created: 1, model, + choices: [{ index: 0, delta: { content: "hello" }, finish_reason: null }] }; + const sse = (afterContent ? `data: ${JSON.stringify(chunk)}\n\n` : "") + + `${event}data: ${JSON.stringify(errorBody)}\n\ndata: [DONE]\n\n`; + async function consume() { + const { client, fetch } = clientFor(sse, 200, true); + const stream = await create(client, "chat", true); + const chunks = []; + let error: unknown; + try { for await (const value of stream) chunks.push(value); } + catch (caught) { error = caught; } + expect(fetch).toHaveBeenCalledTimes(1); + expect(chunks).toEqual(afterContent ? [chunk] : []); + expect(error).toBeInstanceOf(sdk.APIError); + expect(error).toMatchObject({ message: errorBody.error.message, error: errorBody.error }); + return { error, stream }; + } + instrumentation.disable(); + const native = await consume(); + expect(exporter.getFinishedSpans()).toHaveLength(0); + instrumentation.manuallyInstrument(sdk); + const traced = await consume(); + expect((traced.error as Error).constructor).toBe((native.error as Error).constructor); + const span = await onlySpan("chat"); + expect(span.status).toEqual({ code: SpanStatusCode.ERROR, message: errorBody.error.message }); + expect(span.events.filter(event => event.name === "exception")).toHaveLength(1); + expectNoOutput(span.attributes); + // SDK cleanup aborts its controller before rejecting; it must not end twice. + traced.stream.controller.abort(); + await provider.forceFlush(); + expect(exporter.getFinishedSpans()).toHaveLength(1); + }); +}); + +describe.each(apis)("base controls: %s", api => { + test("the uninstrumented SDK returns the same HTTP 200 error body", async () => { + instrumentation.disable(); + const { client, fetch } = clientFor(errorBody); + await expect(create(client, api)).resolves.toEqual(errorBody); + expect(fetch).toHaveBeenCalledTimes(1); + await provider.forceFlush(); + expect(exporter.getFinishedSpans()).toHaveLength(0); + }); + + test("a normal completion still exports one successful span", async () => { + const { client, fetch } = clientFor(completed[api]); + // The SDK may add conveniences such as output_text to the parsed body. + await expect(create(client, api)).resolves.toMatchObject(completed[api]); + expect(fetch).toHaveBeenCalledTimes(1); + const span = await onlySpan(api); + expect(span.status.code).toBe(SpanStatusCode.OK); + expect(span.attributes["output.value"]).toBeDefined(); + }); + + test("a real SDK stream stays lazy and exports one successful span on consumption", async () => { + const chunk = api === "chat" ? { + id: "chat-fixture", object: "chat.completion.chunk", created: 1, model, + choices: [{ index: 0, delta: { role: "assistant", content: "hello" }, finish_reason: "stop" }], + } : api === "completion" ? completed.completion : { type: "response.completed", response: completed.responses }; + const { client, fetch } = clientFor(`data: ${JSON.stringify(chunk)}\n\ndata: [DONE]\n\n`, 200, true); + const stream = await create(client, api, true); + expect(typeof stream[Symbol.asyncIterator]).toBe("function"); + expect(stream.controller).toBeInstanceOf(AbortController); + expect(exporter.getFinishedSpans()).toHaveLength(0); + const chunks = []; + for await (const value of stream) chunks.push(value); + expect(chunks).toEqual([chunk]); + expect(fetch).toHaveBeenCalledTimes(1); + const span = await onlySpan(api); + expect(span.status.code).toBe(SpanStatusCode.OK); + expect(span.attributes["output.value"]).toBeDefined(); + }); + + test("an HTTP 500 still rejects with the SDK error and exports one failed span", async () => { + const { client, fetch } = clientFor(errorBody, 500); + const promise = create(client, api); + await expect(promise).rejects.toBeInstanceOf(sdk.APIError); + await expect(promise).rejects.toMatchObject({ status: 500, error: errorBody.error }); + expect(fetch).toHaveBeenCalledTimes(1); + const span = await onlySpan(api); + expect(span.status.code).toBe(SpanStatusCode.ERROR); + expect(span.status.message).toContain(errorBody.error.message); + }); +}); diff --git a/typescript/packages/traceai_openai/src/instrumentation.ts b/typescript/packages/traceai_openai/src/instrumentation.ts index 339954520..a04bc72d0 100644 --- a/typescript/packages/traceai_openai/src/instrumentation.ts +++ b/typescript/packages/traceai_openai/src/instrumentation.ts @@ -287,7 +287,7 @@ export class OpenAIInstrumentation extends InstrumentationBase{ const wrappedPromise = observeAPIPromise(execPromise, (result) => { if (!span.isRecording()) return result; - if (isChatCompletionResponse(result)) { + if (isChatCompletionResponse(result) && !hasResponseError(result)) { const responseAttributes = { [SemanticConventions.OUTPUT_VALUE]: JSON.stringify(result), [SemanticConventions.OUTPUT_MIME_TYPE]: MimeType.JSON, @@ -304,11 +304,13 @@ export class OpenAIInstrumentation extends InstrumentationBase{ span.setAttributes(responseAttributes); span.setStatus({ code: SpanStatusCode.OK }); span.end(); - } else { + } else if (isStream(result)) { result = observeStream(result, span, requestStartedAt, (chunks) => consumeChatCompletionStreamChunks(chunks, span), args[1]?.signal, ); + } else { + endNonStreamResponseSpan(span, result); } return result; }, (error) => endSpanWithError(span, error), (response) => endRawResponseSpan(span, response)); @@ -376,7 +378,7 @@ export class OpenAIInstrumentation extends InstrumentationBase{ ); const wrappedPromise = observeAPIPromise(execPromise, (result) => { if (!span.isRecording()) return result; - if (isCompletionResponse(result)) { + if (isCompletionResponse(result) && !hasResponseError(result)) { span.setAttributes({ [SemanticConventions.OUTPUT_VALUE]: JSON.stringify(result), [SemanticConventions.OUTPUT_MIME_TYPE]: MimeType.JSON, @@ -391,7 +393,7 @@ export class OpenAIInstrumentation extends InstrumentationBase{ }); span.setStatus({ code: SpanStatusCode.OK }); span.end(); - } else { + } else if (isStream(result)) { result = observeStream(result, span, requestStartedAt, (chunks) => { span.setAttributes({ ...getStreamMetadataAttributes(chunks), @@ -403,6 +405,8 @@ export class OpenAIInstrumentation extends InstrumentationBase{ }); span.setStatus({ code: SpanStatusCode.OK }); }, args[1]?.signal); + } else { + endNonStreamResponseSpan(span, result); } return result; }, (error) => endSpanWithError(span, error), (response) => endRawResponseSpan(span, response)); @@ -469,10 +473,18 @@ export class OpenAIInstrumentation extends InstrumentationBase{ ); const wrappedPromise = observeAPIPromise(execPromise, (result) => { if (!span.isRecording()) return result; + if (hasResponseError(result)) { + endNonStreamResponseSpan(span, result); + return result; + } if (result) { // Record the results span.setAttributes({ [SemanticConventions.GEN_AI_RESPONSE_MODEL]: result.model, + ...(result.usage && { + [SemanticConventions.LLM_TOKEN_COUNT_PROMPT]: result.usage.prompt_tokens, + [SemanticConventions.LLM_TOKEN_COUNT_TOTAL]: result.usage.total_tokens, + }), // Do not record the output data as it can be large ...getEmbeddingEmbeddingsAttributes(result), [SemanticConventions.RAW_OUTPUT]: safelyJSONStringify(result) ?? "", @@ -593,7 +605,7 @@ export class OpenAIInstrumentation extends InstrumentationBase{ }; if (isResponseCreateResponse(result)) { recordSpan(result); - } else { + } else if (isStream(result)) { result = observeStream( result, span, @@ -604,6 +616,8 @@ export class OpenAIInstrumentation extends InstrumentationBase{ args[1]?.signal, (event) => event.type === "response.completed" || event.type === "response.failed" || event.type === "response.incomplete", ); + } else { + endNonStreamResponseSpan(span, result); } return result; @@ -797,28 +811,57 @@ function endRawResponseSpan(span: Span, response: { status: number }): void { span.end(); } +function isObject(response: unknown): response is Record { + return response !== null && typeof response === "object"; +} + +/** Missing completion fields do not imply a stream, even on an HTTP 200. */ +function isStream(response: unknown): response is Stream { + return isObject(response) && ( + typeof response[Symbol.asyncIterator] === "function" || + typeof response.iterator === "function" + ); +} + +function hasResponseError(response: unknown): response is { error: Record } { + return isObject(response) && isObject(response.error); +} + +/** Preserve unexpected parsed bodies without inventing output or consuming them. */ +function endNonStreamResponseSpan(span: Span, response: unknown): void { + if (hasResponseError(response)) { + const error = new Error(typeof response.error.message === "string" + ? response.error.message : "Provider returned an error"); + // Use the same exception.type event attribute as SDK exceptions. + if (typeof response.error.type === "string") error.name = response.error.type; + endSpanWithError(span, error); + } else { + span.end(); + } +} + function isResponseCreateResponse( - response: Stream | ResponseType, + response: unknown, ): response is ResponseType { - return "object" in response && response.object === "response"; + return isObject(response) && response.object === "response"; } /** * type-guard that checks if the response is a chat completion response */ function isChatCompletionResponse( - response: Stream | ChatCompletion, + response: unknown, ): response is ChatCompletion { - return "choices" in response; + return isObject(response) && Array.isArray(response.choices); } /** * type-guard that checks if the response is a completion response */ function isCompletionResponse( - response: Stream | Completion, + response: unknown, ): response is Completion { - return "choices" in response; + return isObject(response) && Array.isArray(response.choices); } /** @@ -1133,6 +1176,7 @@ function getEmbeddingTextAttributes( function getEmbeddingEmbeddingsAttributes( response: CreateEmbeddingResponse, ): Attributes { + if (!Array.isArray(response?.data)) return {}; return response.data.reduce((acc, embedding, index) => { const indexPrefix = `${SemanticConventions.EMBEDDING_EMBEDDINGS}.${index}.`; acc[`${indexPrefix}${SemanticConventions.EMBEDDING_VECTOR}`] = diff --git a/typescript/packages/traceai_openai/src/version.ts b/typescript/packages/traceai_openai/src/version.ts index 93700aa47..ba8f7e776 100644 --- a/typescript/packages/traceai_openai/src/version.ts +++ b/typescript/packages/traceai_openai/src/version.ts @@ -1 +1 @@ -export const VERSION = '0.1.17'; // x-release-please-version \ No newline at end of file +export const VERSION = '0.1.18'; // x-release-please-version \ No newline at end of file diff --git a/typescript/packages/traceai_openrouter/CHANGELOG.md b/typescript/packages/traceai_openrouter/CHANGELOG.md new file mode 100644 index 000000000..4bee96470 --- /dev/null +++ b/typescript/packages/traceai_openrouter/CHANGELOG.md @@ -0,0 +1,8 @@ +# Changelog + +## [0.1.1] - pending release + +### Fixed +- The README shows the install command and how to send traces to Future AGI with @traceai/fi-core. +- Completed OpenRouter calls are marked OK in traces, like other traceAI TypeScript packages: a response whose JSON body is parsed, or a stream that reaches its final event. Provider failures stay errors. + diff --git a/typescript/packages/traceai_openrouter/README.md b/typescript/packages/traceai_openrouter/README.md index b72dad74a..eddbcc204 100644 --- a/typescript/packages/traceai_openrouter/README.md +++ b/typescript/packages/traceai_openrouter/README.md @@ -1,14 +1,22 @@ # OpenRouter TypeScript tracing -Trace the official `@openrouter/sdk` 1.5.33+ inference calls through its supported custom HTTP client. Install `@traceai/openrouter`, `@openrouter/sdk`, and a configured OpenTelemetry SDK exporter. Configure Future AGI's OTLP HTTP endpoint `https://api.futureagi.com/tracer/v1/traces` with `FI_API_KEY` and `FI_SECRET_KEY` in your exporter; never embed secrets in trace attributes. +Trace the official `@openrouter/sdk` 1.5.33+ inference calls through its supported custom HTTP client. Install the adapter, the SDK and `@traceai/fi-core`, which sets up the exporter to Future AGI: + +```sh +npm install @traceai/openrouter @openrouter/sdk @traceai/fi-core +``` + +Set `FI_API_KEY`, `FI_SECRET_KEY` and `OPENROUTER_API_KEY`, then: ```ts +import { register, ProjectType } from "@traceai/fi-core"; import { OpenRouter } from "@openrouter/sdk"; import { createOpenRouterTracingClient } from "@traceai/openrouter"; +const tracerProvider = register({ projectName: "openrouter-demo", projectType: ProjectType.OBSERVE }); const openrouter = new OpenRouter({ apiKey: process.env.OPENROUTER_API_KEY, - httpClient: createOpenRouterTracingClient(), + httpClient: createOpenRouterTracingClient({ tracer: tracerProvider.getTracer("@traceai/openrouter") }), }); const result = await openrouter.chat.send({ chatRequest: { model: "openai/gpt-4o-mini", messages: [{ role: "user", content: "Hello" }] }, diff --git a/typescript/packages/traceai_openrouter/package.json b/typescript/packages/traceai_openrouter/package.json index 5b3fc22f0..8ddf586e0 100644 --- a/typescript/packages/traceai_openrouter/package.json +++ b/typescript/packages/traceai_openrouter/package.json @@ -1,6 +1,6 @@ { "name": "@traceai/openrouter", - "version": "0.1.0", + "version": "0.1.1", "description": "OpenTelemetry tracing for the official OpenRouter TypeScript SDK", "main": "./dist/src/index.js", "types": "./dist/src/index.d.ts", diff --git a/typescript/packages/traceai_openrouter/src/index.ts b/typescript/packages/traceai_openrouter/src/index.ts index d6f3d86d3..844e4228d 100644 --- a/typescript/packages/traceai_openrouter/src/index.ts +++ b/typescript/packages/traceai_openrouter/src/index.ts @@ -5,7 +5,7 @@ import { HTTPClient, type Fetcher } from "@openrouter/sdk"; // eslint-disable-next-line @typescript-eslint/no-require-imports const sdkOperations = require("@openrouter/sdk/models/operations") as Record; -export const VERSION = "0.1.0"; // x-release-please-version +export const VERSION = "0.1.1"; // x-release-please-version /** Only official SDK inference endpoints are traced; unrelated OpenRouter APIs are left alone. */ const operations: Readonly> = { @@ -87,12 +87,13 @@ async function responseMetadata(response: Response): Promise<{ payload?: unknown } /** Observe complete bounded SSE frames, without retaining message content. */ -function observer(span: Span, markError: () => void) { +function observer(span: Span, markError: () => void, markDone: () => void = () => {}, markUnobserved: () => void = () => {}) { const decoder = new TextDecoder(); let pending = ""; return (chunk: Uint8Array) => { pending += decoder.decode(chunk, { stream: true }); - if (pending.length > 65536) { pending = ""; return; } + // An oversized frame is not observed, so the stream can no longer be shown to have succeeded. + if (pending.length > 65536) { pending = ""; markUnobserved(); return; } // Match the SDK's SSE boundary variants, including CRLF and LF-only frames. const boundary = /\r\n\r\n|\r\n\r|\r\n\n|\r\r\n|\n\r\n|\r\r|\n\r|\n\n/g; let start = 0; @@ -100,10 +101,12 @@ function observer(span: Span, markError: () => void) { const frame = pending.slice(start, match.index); start = match.index! + match[0].length; const data = frame.split(/\r\n|\r|\n/).filter(line => line.startsWith("data:")).map(line => line.slice(5).trimStart()).join("\n"); - if (!data || data === "[DONE]") continue; + if (!data) continue; + if (data === "[DONE]") { markDone(); continue; } try { const event = object(JSON.parse(data)); if (event?.type === "error" || event?.type === "response.failed" || event?.type === "response.incomplete") markError(); + else if (event?.type === "response.completed") markDone(); // Chat chunks carry usage directly; Responses events carry the response in a nested object. setOutput(span, event?.response ?? event); } catch { markError(); /* The SDK will report the parsing failure to its caller. */ } @@ -142,7 +145,17 @@ export function createOpenRouterTracingClient(options: OpenRouterTracingOptions const model = await requestModel(request); if (model) span.setAttribute("gen_ai.request.model", model); let ended = false; - const end = (error = false) => { if (!ended) { ended = true; if (error) span.setStatus({ code: SpanStatusCode.ERROR }); span.end(); } }; + let failed = false; + // OK needs positive evidence of a completed success, as in the other traceAI instrumentors: a + // validated JSON body, or a stream's final [DONE] or response.completed event. A provider failure + // seen in a stream stays ERROR; anything else keeps the default status. + const end = (error = false, succeeded = false) => { + if (ended) return; + ended = true; + if (error) span.setStatus({ code: SpanStatusCode.ERROR }); + else if (succeeded && !failed) span.setStatus({ code: SpanStatusCode.OK }); + span.end(); + }; try { // Keep only the SDK's transport in scope; do not wrap a separate HTTP span. const response = await context.with(trace.setSpan(context.active(), span), () => fetcher(input, init)); @@ -156,17 +169,23 @@ export function createOpenRouterTracingClient(options: OpenRouterTracingOptions } if (type === "text/event-stream" && response.body) { const reader = response.body.getReader(); - const observe = observer(span, () => span.setStatus({ code: SpanStatusCode.ERROR })); + let streamDone = false; + let unobserved = false; + let cancelling = false; + const observe = observer(span, () => { failed = true; span.setStatus({ code: SpanStatusCode.ERROR }); }, () => { streamDone = true; }, () => { unobserved = true; }); const body = new ReadableStream({ async pull(controller) { try { const { done, value } = await reader.read(); - if (done) { end(); controller.close(); return; } + // A cancel resolves a pending read as done; cancel() ends the span with its reason. + if (cancelling) return; + if (done) { end(false, streamDone && !unobserved); controller.close(); return; } observe(value); controller.enqueue(value); } catch (error) { end(true); controller.error(error); } }, - async cancel(reason) { try { await reader.cancel(reason); } finally { end(reason instanceof Error); } }, + // The SDK cancels the body after the final [DONE] or response.completed event; that is a completed stream. + async cancel(reason) { cancelling = true; try { await reader.cancel(reason); } finally { end(reason instanceof Error, streamDone && !unobserved); } }, }, { highWaterMark: 0 }); return new Response(body, { status: response.status, statusText: response.statusText, headers: response.headers }); } @@ -177,7 +196,8 @@ export function createOpenRouterTracingClient(options: OpenRouterTracingOptions end(true); } else { setOutput(span, metadata.payload); - end(); + // An oversized body skips the bounded validation, so it is not evidence of success. + end(false, metadata.payload !== undefined); } } else end(); return response; diff --git a/typescript/packages/traceai_openrouter/test/integration.test.cjs b/typescript/packages/traceai_openrouter/test/integration.test.cjs index acef47f6c..2501b993b 100644 --- a/typescript/packages/traceai_openrouter/test/integration.test.cjs +++ b/typescript/packages/traceai_openrouter/test/integration.test.cjs @@ -50,6 +50,8 @@ for (const moduleName of ['@openrouter/sdk-floor', '@openrouter/sdk']) { const stream = await client.chat.send({ chatRequest: { model: 'requested/model', messages: [{ role: 'user', content: 'synthetic' }], stream: true } }); for await (const _event of stream) { /* consume */ } assert.equal(exporter.getFinishedSpans().at(-1).attributes['gen_ai.usage.output_tokens'], 6); + assert.equal(exporter.getFinishedSpans().at(-1).status.code, 1); + assert.equal(chunkedSpan.status.code, 1); const malformedStreamClient = new OpenRouter({ apiKey: 'synthetic-only', retryConfig: { strategy: 'none' }, httpClient: createOpenRouterTracingClient({ fetcher: async () => new Response('data: {bad json}\n\n', { headers: { 'content-type': 'text/event-stream' } }), tracer: provider.getTracer('test') }) }); const malformed = await malformedStreamClient.chat.send({ chatRequest: { model: 'requested/model', messages: [{ role: 'user', content: 'synthetic' }], stream: true } }); @@ -92,6 +94,7 @@ for (const moduleName of ['@openrouter/sdk-floor', '@openrouter/sdk']) { await reader.read(); await reader.cancel(); assert.equal(exporter.getFinishedSpans().at(-1).attributes['gen_ai.operation.name'], 'responses'); + assert.equal(exporter.getFinishedSpans().at(-1).status.code, 0); assert.equal(paths.at(-1), '/api/v1/responses'); assert.equal(exporter.getFinishedSpans().at(-1).attributes['gen_ai.response.model'], 'early'); } finally { await provider.shutdown(); } @@ -136,7 +139,7 @@ for (const moduleName of ['@openrouter/sdk-floor', '@openrouter/sdk']) { ]) { reply = new Response(JSON.stringify(payload), { headers: { 'content-type': 'application/json' } }); await invoke(); - assert.equal(exporter.getFinishedSpans().at(-1).status.code, 0); + assert.equal(exporter.getFinishedSpans().at(-1).status.code, 1); assert.equal(exporter.getFinishedSpans().at(-1).attributes['fi.span.kind'], kind); } for (const [data, type] of [ @@ -169,3 +172,46 @@ for (const moduleName of ['@openrouter/sdk-floor', '@openrouter/sdk']) { } finally { await provider.shutdown(); } }); } + +test('span status is OK only on evidence of a completed success', async () => { + const exporter = new InMemorySpanExporter(); + const provider = new BasicTracerProvider({ spanProcessors: [new SimpleSpanProcessor(exporter)] }); + const tracer = provider.getTracer('test'); + const chat = () => new Request('https://openrouter.ai/api/v1/chat/completions', { method: 'POST', body: '{"model":"synthetic"}' }); + const last = () => exporter.getFinishedSpans().at(-1).status.code; + const pending = () => new Response(new ReadableStream({ start() { /* never enqueues */ } }), { headers: { 'content-type': 'text/event-stream' } }); + try { + // A cancel that lands while a read is pending is not a completed stream. + for (const [reason, code] of [[undefined, 0], [new Error('synthetic'), 2]]) { + const response = await createOpenRouterTracingClient({ fetcher: async () => pending(), tracer }).request(chat()); + const reader = response.body.getReader(); + const read = reader.read(); + await reader.cancel(reason); + await read; + assert.equal(last(), code); + } + // An event-stream response with no body never completes. + await createOpenRouterTracingClient({ fetcher: async () => new Response(null, { headers: { 'content-type': 'text/event-stream' } }), tracer }).request(chat()); + assert.equal(last(), 0); + // A JSON body too large for the bounded validation is not evidence of success. + const large = JSON.stringify({ unexpected: 'x'.repeat(66000) }); + await createOpenRouterTracingClient({ fetcher: async () => new Response(large, { headers: { 'content-type': 'application/json' } }), tracer }).request(chat()); + assert.equal(last(), 0); + // A stream frame too large to observe cannot mark the stream completed. + const huge = `data: ${JSON.stringify({ id: 'x', object: 'chat.completion.chunk', created: 1, model: 'm', choices: [{ index: 0, delta: { content: 'x'.repeat(66000) } }] })}\n\ndata: [DONE]\n\n`; + const streamed = await createOpenRouterTracingClient({ fetcher: async () => new Response(huge, { headers: { 'content-type': 'text/event-stream' } }), tracer }).request(chat()); + const hugeReader = streamed.body.getReader(); + while (!(await hugeReader.read()).done) { /* consume */ } + assert.notEqual(last(), 1); + // The same oversized frame followed by [DONE] in a later chunk is still not evidence of success. + const [frame, finish] = [huge.slice(0, huge.indexOf('data: [DONE]')), 'data: [DONE]\n\n']; + const split = await createOpenRouterTracingClient({ fetcher: async () => new Response(new ReadableStream({ start(controller) { + const encoder = new TextEncoder(); controller.enqueue(encoder.encode(frame)); controller.enqueue(encoder.encode(finish)); controller.close(); + } }), { headers: { 'content-type': 'text/event-stream' } }), tracer }).request(chat()); + const splitReader = split.body.getReader(); + while (!(await splitReader.read()).done) { /* consume */ } + assert.notEqual(last(), 1); + } finally { + await provider.shutdown(); + } +}); diff --git a/typescript/packages/traceai_vercel/CHANGELOG.md b/typescript/packages/traceai_vercel/CHANGELOG.md index aacec0cd4..5c91a3c69 100644 --- a/typescript/packages/traceai_vercel/CHANGELOG.md +++ b/typescript/packages/traceai_vercel/CHANGELOG.md @@ -1,5 +1,10 @@ # Changelog +## [0.1.5] - pending release + +### Fixed +- Embedding calls made with embed and embedMany keep their token usage in auto mode. + ## [0.1.4] - pending release ### Feature - `enableVercelInstrumentation()` (also exported as `enable`) and `VercelAIInstrumentation` trace the AI SDK's `generateText`, `streamText`, `generateObject`, `streamObject`, `embed` and `embedMany` without `experimental_telemetry`. The zero-code agent `@traceai/auto` uses it. diff --git a/typescript/packages/traceai_vercel/package.json b/typescript/packages/traceai_vercel/package.json index 3166d1362..858d63f38 100644 --- a/typescript/packages/traceai_vercel/package.json +++ b/typescript/packages/traceai_vercel/package.json @@ -1,6 +1,6 @@ { "name": "@traceai/vercel", - "version": "0.1.4", + "version": "0.1.5", "description": "OpenTelemetry instrumentation for Vercel", "main": "./dist/src/index.js", "module": "./dist/esm/index.js", diff --git a/typescript/packages/traceai_vercel/src/AISemanticConventions.ts b/typescript/packages/traceai_vercel/src/AISemanticConventions.ts index 3e83b357c..5249e2b93 100644 --- a/typescript/packages/traceai_vercel/src/AISemanticConventions.ts +++ b/typescript/packages/traceai_vercel/src/AISemanticConventions.ts @@ -50,6 +50,9 @@ const TOKEN_COUNT_COMPLETION = const TOKEN_COUNT_PROMPT = `${AI_PREFIX}.${AIPrefixes.usage}.${AIUsagePostfixes.promptTokens}` as const; +/** Embedding usage maps to both input and total counts, not a single FI key. */ +export const EMBEDDING_TOKEN_COUNT = `${AI_PREFIX}.${AIPrefixes.usage}.tokens` as const; + const RESPONSE_TEXT = `${AI_PREFIX}.${AIPrefixes.response}.${AIResponsePostfixes.text}` as const; diff --git a/typescript/packages/traceai_vercel/src/__tests__/embed-usage.test.ts b/typescript/packages/traceai_vercel/src/__tests__/embed-usage.test.ts new file mode 100644 index 000000000..6905b32cc --- /dev/null +++ b/typescript/packages/traceai_vercel/src/__tests__/embed-usage.test.ts @@ -0,0 +1,183 @@ +import { Attributes, context, trace } from "@opentelemetry/api"; +import { suppressTracing } from "@opentelemetry/core"; +import { AsyncLocalStorageContextManager } from "@opentelemetry/context-async-hooks"; +import { BasicTracerProvider, InMemorySpanExporter, SimpleSpanProcessor } from "@opentelemetry/sdk-trace-base"; +import { SemanticConventions } from "@traceai/fi-semantic-conventions"; +import { VercelAIInstrumentation } from "../auto"; +import { safelyGetFIAttributes } from "../utils"; + +const INPUT_TOKENS = SemanticConventions.LLM_TOKEN_COUNT_PROMPT; +const TOTAL_TOKENS = SemanticConventions.LLM_TOKEN_COUNT_TOTAL; +const OUTPUT_TOKENS = SemanticConventions.LLM_TOKEN_COUNT_COMPLETION; +const operations = ["ai.embed", "ai.embed.doEmbed", "ai.embedMany", "ai.embedMany.doEmbed"]; + +describe("AI SDK 5 embedding usage attribute shape", () => { + test.each(operations)("normalizes input and total tokens for %s with or without a function ID", operation => { + for (const suffix of ["", " customer-function"]) { + const attributes = { + "operation.name": `${operation}${suffix}`, + "ai.usage.tokens": operation.includes("embedMany") ? 11 : 7, + }; + const normalized = safelyGetFIAttributes(attributes); + expect(normalized).toMatchObject({ + [INPUT_TOKENS]: attributes["ai.usage.tokens"], + [TOTAL_TOKENS]: attributes["ai.usage.tokens"], + }); + expect(normalized?.[OUTPUT_TOKENS]).toBeUndefined(); + expect(attributes).not.toHaveProperty(INPUT_TOKENS); + expect(attributes).not.toHaveProperty(TOTAL_TOKENS); + } + }); + + test.each([INPUT_TOKENS, TOTAL_TOKENS])("preserves an existing zero %s while filling the other counter", key => { + const normalized = safelyGetFIAttributes({ + "operation.name": "ai.embed.doEmbed", + "ai.usage.tokens": 7, + [key]: 0, + }); + expect(normalized).toMatchObject({ [INPUT_TOKENS]: 7, [TOTAL_TOKENS]: 7, [key]: 0 }); + }); + + test("preserves both existing canonical counters", () => { + expect(safelyGetFIAttributes({ + "operation.name": "ai.embedMany", + "ai.usage.tokens": 11, + [INPUT_TOKENS]: 13, + [TOTAL_TOKENS]: 17, + })).toMatchObject({ [INPUT_TOKENS]: 13, [TOTAL_TOKENS]: 17 }); + }); + + test("keeps zero embedding usage", () => { + expect(safelyGetFIAttributes({ "operation.name": "ai.embed", "ai.usage.tokens": 0 })) + .toMatchObject({ [INPUT_TOKENS]: 0, [TOTAL_TOKENS]: 0 }); + }); + + test.each([undefined, "7", [7], Number.NaN, Number.POSITIVE_INFINITY, -1])( + "does not synthesize counters for invalid embedding usage %s", tokens => { + const normalized = safelyGetFIAttributes({ "operation.name": "ai.embed", "ai.usage.tokens": tokens }); + expect(normalized?.[INPUT_TOKENS]).toBeUndefined(); + expect(normalized?.[TOTAL_TOKENS]).toBeUndefined(); + }, + ); + + test.each(["ai.generateText", "ai.generateText.doGenerate", "ai.toolCall", "ai.embedOther", undefined])( + "does not apply embedding usage to unrelated operation %s", operation => { + const normalized = safelyGetFIAttributes({ "operation.name": operation, "ai.usage.tokens": 7 }); + expect(normalized?.[INPUT_TOKENS]).toBeUndefined(); + expect(normalized?.[TOTAL_TOKENS]).toBeUndefined(); + }, + ); + + test("chat prompt/completion usage remains unchanged", () => { + expect(safelyGetFIAttributes({ + "operation.name": "ai.generateText.doGenerate", + "ai.usage.promptTokens": 3, + "ai.usage.completionTokens": 4, + "ai.usage.tokens": 99, + })).toMatchObject({ [INPUT_TOKENS]: 3, [OUTPUT_TOKENS]: 4 }); + }); +}); + +describe("real AI SDK 6.0.320 embedding usage in auto mode", () => { + const exporter = new InMemorySpanExporter(); + const provider = new BasicTracerProvider(); + const manager = new AsyncLocalStorageContextManager(); + let original: any; + let sdk: any; + let createOpenAI: any; + let instrumentor: VercelAIInstrumentation; + + beforeAll(() => { + provider.addSpanProcessor(new SimpleSpanProcessor(exporter)); + context.setGlobalContextManager(manager.enable()); + trace.setGlobalTracerProvider(provider); + original = require("ai"); + ({ createOpenAI } = require("@ai-sdk/openai")); + instrumentor = new VercelAIInstrumentation({ instrumentationConfig: { enabled: false } }); + instrumentor.setTracerProvider(provider); + instrumentor.enable(); + sdk = instrumentor.manuallyInstrument(original, "6.0.320"); + }); + beforeEach(() => exporter.reset()); + afterAll(async () => { + instrumentor?.disable(); + await provider.shutdown(); + context.disable(); + trace.disable(); + manager.disable(); + }); + + test("keeps a canonical total that the span set itself when embedding usage adds the input counter", () => { + const fake = instrumentor.manuallyInstrument({ + embed({ experimental_telemetry }: any) { + const attributes: Attributes = { "operation.name": "ai.embed.doEmbed", [TOTAL_TOKENS]: 17, [OUTPUT_TOKENS]: 0 }; + const span = experimental_telemetry.tracer.startSpan("ai.embed.doEmbed", { attributes }); + span.setAttribute("ai.usage.tokens", 7); + span.end(); + }, + }, "6.0.320"); + fake.embed({}); + const spans = exporter.getFinishedSpans(); + expect(spans).toHaveLength(1); + expect(spans[0].attributes[INPUT_TOKENS]).toBe(7); + expect(spans[0].attributes[TOTAL_TOKENS]).toBe(17); + }); + + test.each([ + ["embed", ["hello"], 7], + ["embedMany", ["hello", "world"], 11], + ] as const)("%s keeps usage on the operation and doEmbed spans without content aliases", async (method, values, tokens) => { + const embeddings = values.map((_, index) => [0.123456 + index, 0.654321]); + const fetch = jest.fn(async (url: string, init: RequestInit) => { + expect(String(url)).toBe("https://embedding.test/v1/embeddings"); + const body = JSON.parse(init.body as string); + expect(body.input).toEqual(values); + return new Response(JSON.stringify({ + object: "list", + model: "text-embedding-3-small", + data: embeddings.map((embedding, index) => ({ object: "embedding", index, embedding })), + usage: { prompt_tokens: tokens, total_tokens: tokens }, + }), { status: 200, headers: { "content-type": "application/json" } }); + }); + const openai = createOpenAI({ apiKey: "test-key", baseURL: "https://embedding.test/v1", fetch }); + const options = { + model: openai.embedding("text-embedding-3-small"), + ...(method === "embed" ? { value: values[0] } : { values: [...values] }), + maxRetries: 0, + }; + const plain = await context.with(suppressTracing(context.active()), () => original[method](options)); + const result = await sdk[method](options); + expect(fetch).toHaveBeenCalledTimes(2); + expect(result.usage).toEqual(plain.usage); + expect(result.usage.tokens).toBe(tokens); + const resultKey = method === "embed" ? "embedding" : "embeddings"; + expect(result[resultKey]).toEqual(plain[resultKey]); + expect(result[resultKey]).toEqual(method === "embed" ? embeddings[0] : embeddings); + expect(options).not.toHaveProperty("experimental_telemetry"); + + const spans = exporter.getFinishedSpans(); + expect(spans.map(span => span.name).sort()).toEqual([`ai.${method}`, `ai.${method}.doEmbed`]); + for (const span of spans) { + expect(span.attributes).toMatchObject({ [INPUT_TOKENS]: tokens, [TOTAL_TOKENS]: tokens }); + expect(span.attributes[OUTPUT_TOKENS]).toBeUndefined(); + expect(Object.keys(span.attributes).some(key => key.startsWith("ai."))).toBe(false); + expect(JSON.stringify(span.attributes)).not.toContain("0.123456"); + } + }); + + test.each([INPUT_TOKENS, TOTAL_TOKENS])("auto mode preserves canonical %s during native span updates", key => { + const emit = instrumentor.manuallyInstrument({ + embed({ experimental_telemetry }: any) { + const attributes: Attributes = { "operation.name": "ai.embed.doEmbed", [key]: 0 }; + const span = experimental_telemetry.tracer.startSpan("ai.embed.doEmbed", { attributes }); + span.setAttribute("ai.usage.tokens", 7); + span.end(); + return "unchanged"; + }, + }, "6.0.320"); + expect(emit.embed({})).toBe("unchanged"); + expect(exporter.getFinishedSpans()).toHaveLength(1); + expect(exporter.getFinishedSpans()[0].attributes) + .toMatchObject({ [INPUT_TOKENS]: 7, [TOTAL_TOKENS]: 7, [key]: 0 }); + }); +}); diff --git a/typescript/packages/traceai_vercel/src/auto.ts b/typescript/packages/traceai_vercel/src/auto.ts index 79cf8545c..c5fdfc8aa 100644 --- a/typescript/packages/traceai_vercel/src/auto.ts +++ b/typescript/packages/traceai_vercel/src/auto.ts @@ -36,7 +36,10 @@ function nativeAttributes(attributes: Attributes): Attributes { else if (typeof attributes["gen_ai.system"] === "string") result["gen_ai.provider.name"] = attributes["gen_ai.system"]; const inputTokens = result["gen_ai.usage.input_tokens"]; const outputTokens = result["gen_ai.usage.output_tokens"]; - if (typeof inputTokens === "number" && typeof outputTokens === "number") result["gen_ai.usage.total_tokens"] = inputTokens + outputTokens; + // Derive the total only when the span did not set a canonical total itself. + if (typeof inputTokens === "number" && typeof outputTokens === "number" && typeof attributes["gen_ai.usage.total_tokens"] !== "number") { + result["gen_ai.usage.total_tokens"] = inputTokens + outputTokens; + } // Tool arguments are already normalized to input.value. The legacy alias is // outside both FI masking and the export boundary's covered content families. delete result["tool.parameters"]; diff --git a/typescript/packages/traceai_vercel/src/utils.ts b/typescript/packages/traceai_vercel/src/utils.ts index a36b3f610..10b0af80b 100644 --- a/typescript/packages/traceai_vercel/src/utils.ts +++ b/typescript/packages/traceai_vercel/src/utils.ts @@ -11,6 +11,7 @@ import { import { AISemanticConventions, AISemanticConventionsList, + EMBEDDING_TOKEN_COUNT, } from "./AISemanticConventions"; import { FIIOConventionKey, @@ -393,6 +394,27 @@ import { onError: onErrorCallback("metadata"), }); + /** SDK 5/6 report embedding usage on both operation and doEmbed spans. */ + const getEmbeddingTokenAttributes = (attributes: Attributes): Attributes => { + const operationName = attributes["operation.name"]; + const tokens = attributes[EMBEDDING_TOKEN_COUNT]; + if ( + typeof operationName !== "string" || + !["ai.embed", "ai.embedMany", "ai.embed.doEmbed", "ai.embedMany.doEmbed"].includes( + getVercelFunctionNameFromOperationName(operationName) ?? "", + ) || + typeof tokens !== "number" || !Number.isFinite(tokens) || tokens < 0 + ) { + return {}; + } + return { + [SemanticConventions.LLM_TOKEN_COUNT_PROMPT]: + attributes[SemanticConventions.LLM_TOKEN_COUNT_PROMPT] ?? tokens, + [SemanticConventions.LLM_TOKEN_COUNT_TOTAL]: + attributes[SemanticConventions.LLM_TOKEN_COUNT_TOTAL] ?? tokens, + }; + }; + /** * Gets the FI attributes associated with the span from the initial attributes * @param attributesWithSpanKind the initial attributes of the span and the FI span kind @@ -404,6 +426,7 @@ import { const spanKind = safelyGetFISpanKindFromAttributes(attributes); const fiAttributes = { [SemanticConventions.FI_SPAN_KIND]: spanKind ?? undefined, + ...getEmbeddingTokenAttributes(attributes), }; return AISemanticConventionsList.reduce( (fiAttributes: Attributes, convention) => { diff --git a/typescript/packages/traceai_vercel/src/version.ts b/typescript/packages/traceai_vercel/src/version.ts index ede1806b4..90d2111f1 100644 --- a/typescript/packages/traceai_vercel/src/version.ts +++ b/typescript/packages/traceai_vercel/src/version.ts @@ -1 +1 @@ -export const VERSION = '0.1.4'; // x-release-please-version \ No newline at end of file +export const VERSION = '0.1.5'; // x-release-please-version \ No newline at end of file diff --git a/typescript/packages/traceai_vitest_evals/CHANGELOG.md b/typescript/packages/traceai_vitest_evals/CHANGELOG.md new file mode 100644 index 000000000..334426519 --- /dev/null +++ b/typescript/packages/traceai_vitest_evals/CHANGELOG.md @@ -0,0 +1,7 @@ +# Changelog + +## [0.1.1] - pending release + +### Fixed +- Eval traces reach Future AGI: the reporter names the Observe project (the projectName option, else FI_PROJECT_NAME, else vitest-evals), which the backend requires. + diff --git a/typescript/packages/traceai_vitest_evals/README.md b/typescript/packages/traceai_vitest_evals/README.md index f3438d1dc..572ff3977 100644 --- a/typescript/packages/traceai_vitest_evals/README.md +++ b/typescript/packages/traceai_vitest_evals/README.md @@ -12,7 +12,7 @@ npm install -D @traceai/vitest-evals vitest-evals@0.17 vitest@4 The reporter only loads the OpenTelemetry packages at runtime; `vitest` and `vitest-evals` are optional peer dependencies that declare the supported versions. npm 10 can crash while installing `vitest-evals` 0.17 together with Vitest 4 (`Cannot read properties of null (reading 'edgesOut')`), with or without this package; use npm 11 or pnpm. -Set `FI_API_KEY` and `FI_SECRET_KEY` in the test process. In `vitest.config.ts`: +Set `FI_API_KEY` and `FI_SECRET_KEY` in the test process, and `FI_PROJECT_NAME` for the Observe project the eval traces go to (default `vitest-evals`; or pass `new VitestEvalsReporter({ projectName: '...' })`). In `vitest.config.ts`: ```ts import { defineConfig } from 'vitest/config'; diff --git a/typescript/packages/traceai_vitest_evals/package.json b/typescript/packages/traceai_vitest_evals/package.json index 4f5d24484..eae8b3631 100644 --- a/typescript/packages/traceai_vitest_evals/package.json +++ b/typescript/packages/traceai_vitest_evals/package.json @@ -1,5 +1,5 @@ { - "name": "@traceai/vitest-evals", "version": "0.1.0", + "name": "@traceai/vitest-evals", "version": "0.1.1", "description": "Export vitest-evals harness traces and scores to Future AGI via OpenTelemetry", "main": "./dist/index.js", "types": "./dist/index.d.ts", "exports": { ".": { "types": "./dist/index.d.ts", "require": "./dist/index.js", "import": "./dist/index.js" } }, diff --git a/typescript/packages/traceai_vitest_evals/src/index.ts b/typescript/packages/traceai_vitest_evals/src/index.ts index e67750b9b..bcd3da500 100644 --- a/typescript/packages/traceai_vitest_evals/src/index.ts +++ b/typescript/packages/traceai_vitest_evals/src/index.ts @@ -7,8 +7,13 @@ import { resourceFromAttributes } from '@opentelemetry/resources'; import type { ReadableSpan, SpanExporter } from '@opentelemetry/sdk-trace-base'; const DEFAULT_ENDPOINT = 'https://api.futureagi.com/tracer/v1/traces'; -const RESOURCE = resourceFromAttributes({ 'service.name': 'vitest-evals' }); -const SCOPE = { name: '@traceai/vitest-evals', version: '0.1.0' }; +// Future AGI files every span under its resource's project; a batch without project_name is refused. +const DEFAULT_PROJECT = 'vitest-evals'; +function projectResource(projectName?: string) { + const name = projectName || process.env.FI_PROJECT_NAME || DEFAULT_PROJECT; + return resourceFromAttributes({ 'service.name': 'vitest-evals', project_name: name, project_type: 'observe' }); +} +const SCOPE = { name: '@traceai/vitest-evals', version: '0.1.1' }; const SAFE_KEYS = new Set([ 'gen_ai.system', 'gen_ai.operation.name', 'gen_ai.request.model', 'gen_ai.response.model', 'gen_ai.usage.input_tokens', 'gen_ai.usage.output_tokens', 'gen_ai.usage.total_tokens', @@ -25,7 +30,7 @@ type EvalMeta = { scores?: Score[]; avgScore?: number | null; thresholdFailed?: type Run = { traces?: SourceTrace[]; usage?: { provider?: string; model?: string; inputTokens?: number; outputTokens?: number; totalTokens?: number } }; type TestLike = { name: string; meta(): { eval?: EvalMeta; harness?: { run?: Run } }; result(): { state: string } }; type ModuleLike = { children: { allTests(): Iterable } }; -export type ReporterOptions = { exporter?: SpanExporter; endpoint?: string; apiKey?: string; secretKey?: string }; +export type ReporterOptions = { exporter?: SpanExporter; endpoint?: string; apiKey?: string; secretKey?: string; projectName?: string }; const hexId = (value: string | undefined, length: number): value is string => typeof value === 'string' && new RegExp(`^[a-fA-F0-9]{${length}}$`).test(value) && !/^0+$/.test(value); @@ -67,7 +72,7 @@ function safeAttributes(span: SourceSpan, evalMeta?: EvalMeta, run?: Run): Attri } return attributes; } -function mappedTrace(trace: SourceTrace, evalMeta?: EvalMeta, run?: Run): ReadableSpan[] { +function mappedTrace(trace: SourceTrace, resource: ReadableSpan['resource'], evalMeta?: EvalMeta, run?: Run): ReadableSpan[] { const sourceTraceId = trace.id ?? trace.spans.find(span => hexId(span.traceId, 32))?.traceId; const traceId = hexId(sourceTraceId, 32) ? sourceTraceId.toLowerCase() : newId(16); const ids = new Map(); @@ -94,7 +99,7 @@ function mappedTrace(trace: SourceTrace, evalMeta?: EvalMeta, run?: Run): Readab startTime: start, endTime: end, duration, ended: true, status: { code: span.status === 'error' ? SpanStatusCode.ERROR : SpanStatusCode.OK }, attributes: safeAttributes(span, index === 0 ? evalMeta : undefined, index === 0 ? run : undefined), - events: [], links: [], resource: RESOURCE, instrumentationScope: SCOPE, + events: [], links: [], resource, instrumentationScope: SCOPE, droppedAttributesCount: 0, droppedEventsCount: 0, droppedLinksCount: 0, }; }); @@ -102,7 +107,9 @@ function mappedTrace(trace: SourceTrace, evalMeta?: EvalMeta, run?: Run): Readab export class VitestEvalsReporter { private readonly exporter: SpanExporter; + private readonly resource: ReadableSpan['resource']; constructor(options: ReporterOptions = {}) { + this.resource = projectResource(options.projectName); this.exporter = options.exporter ?? new OTLPTraceExporter({ url: options.endpoint ?? DEFAULT_ENDPOINT, headers: { @@ -120,7 +127,7 @@ export class VitestEvalsReporter { const run = harness?.run; const traces = run?.traces?.filter(trace => trace.spans?.length) ?? []; if (!traces.length) traces.push({ spans: [{ name: test.name, kind: 'run', status: test.result().state === 'failed' ? 'error' : 'ok' }] }); - for (const [index, trace] of traces.entries()) spans.push(...mappedTrace(trace, index === 0 ? evalMeta : undefined, index === 0 ? run : undefined)); + for (const [index, trace] of traces.entries()) spans.push(...mappedTrace(trace, this.resource, index === 0 ? evalMeta : undefined, index === 0 ? run : undefined)); } if (!spans.length) return; await new Promise((resolve, reject) => this.exporter.export(spans, result => { diff --git a/typescript/packages/traceai_vitest_evals/test/reporter.test.cjs b/typescript/packages/traceai_vitest_evals/test/reporter.test.cjs index 0b22ebf79..78bc9b5cb 100644 --- a/typescript/packages/traceai_vitest_evals/test/reporter.test.cjs +++ b/typescript/packages/traceai_vitest_evals/test/reporter.test.cjs @@ -45,3 +45,26 @@ test('uses provider trace ID and derives start from end and duration', async () assert.deepEqual(exported[0].startTime, [1767225600, 500000000]); assert.deepEqual(exported[0].duration, [0, 500000000]); }); + +test('every exported span names its Future AGI project, so the backend accepts the batch', async () => { + const run = async (options, env) => { + const saved = process.env.FI_PROJECT_NAME; + if (env === undefined) delete process.env.FI_PROJECT_NAME; else process.env.FI_PROJECT_NAME = env; + try { + let exported = []; + const reporter = new VitestEvalsReporter({ ...options, exporter: { export(spans, cb) { exported = spans; cb({ code: 0 }); }, shutdown: async () => {} } }); + const module = { children: { *allTests() { yield { name: 'synthetic eval', meta: () => ({ eval: { scores: [{ name: 'accuracy', score: 1 }] } }), result: () => ({ state: 'passed' }) }; } } }; + await reporter.onTestRunEnd([module]); + assert.ok(exported.length > 0); + return exported.map(span => span.resource.attributes); + } finally { + if (saved === undefined) delete process.env.FI_PROJECT_NAME; else process.env.FI_PROJECT_NAME = saved; + } + }; + for (const attrs of await run({}, undefined)) { + assert.equal(attrs.project_name, 'vitest-evals'); + assert.equal(attrs.project_type, 'observe'); + } + for (const attrs of await run({}, 'synthetic-env-project')) assert.equal(attrs.project_name, 'synthetic-env-project'); + for (const attrs of await run({ projectName: 'synthetic-option-project' }, 'synthetic-env-project')) assert.equal(attrs.project_name, 'synthetic-option-project'); +});