diff --git a/python/fi_instrumentation/fi_types.py b/python/fi_instrumentation/fi_types.py index 6974f4c6e..05f3a565d 100644 --- a/python/fi_instrumentation/fi_types.py +++ b/python/fi_instrumentation/fi_types.py @@ -1794,12 +1794,13 @@ def validate_fagi_system_eval_mapping( def to_dict(self) -> Dict[str, Any]: """Convert EvalTag to dictionary format for API responses""" - if isinstance(self.model, str): + # Custom evaluations may use their template's model without an override. + if self.model is None or isinstance(self.model, str): model_name = self.model elif isinstance(self.model, ModelChoices): model_name = self.model.value else: - raise ValueError(f"Model must be a string or ModelChoices, got {type(self.model)}") + raise ValueError(f"Model must be None, a string or ModelChoices, got {type(self.model)}") return { "type": self.type.value, diff --git a/python/fi_instrumentation/instrumentation/_attributes.py b/python/fi_instrumentation/instrumentation/_attributes.py index 708f46427..18443032c 100644 --- a/python/fi_instrumentation/instrumentation/_attributes.py +++ b/python/fi_instrumentation/instrumentation/_attributes.py @@ -172,7 +172,10 @@ def get_context_attributes( def get_session_attributes(*, session_id: str) -> Dict[str, AttributeValue]: - return {GEN_AI_CONVERSATION_ID: session_id} + return { + SESSION_ID: session_id, + GEN_AI_CONVERSATION_ID: session_id, + } def get_tag_attributes(*, tags: List[str]) -> Dict[str, AttributeValue]: @@ -577,6 +580,7 @@ def get_llm_tool_attributes( OUTPUT_MIME_TYPE = SpanAttributes.OUTPUT_MIME_TYPE OUTPUT_VALUE = SpanAttributes.OUTPUT_VALUE RETRIEVAL_DOCUMENTS = SpanAttributes.RETRIEVAL_DOCUMENTS +SESSION_ID = SpanAttributes.SESSION_ID GEN_AI_CONVERSATION_ID = SpanAttributes.GEN_AI_CONVERSATION_ID TAG_TAGS = SpanAttributes.TAG_TAGS GEN_AI_TOOL_DESCRIPTION = SpanAttributes.GEN_AI_TOOL_DESCRIPTION diff --git a/python/fi_instrumentation/instrumentation/_tracers.py b/python/fi_instrumentation/instrumentation/_tracers.py index 779854676..46449edfc 100644 --- a/python/fi_instrumentation/instrumentation/_tracers.py +++ b/python/fi_instrumentation/instrumentation/_tracers.py @@ -1,18 +1,23 @@ import asyncio import collections import inspect +import sys from collections.abc import Sequence from contextlib import contextmanager from contextvars import ContextVar from datetime import datetime +from functools import partial from secrets import randbits from types import ModuleType from typing import ( TYPE_CHECKING, Any, # type: ignore[attr-defined] + AsyncGenerator, Awaitable, Callable, + ContextManager, Dict, + Generator, Iterator, Literal, Optional, @@ -27,7 +32,14 @@ import wrapt # type: ignore[import-untyped] from fi_instrumentation.fi_types import FiSpanKindValues, SpanAttributes from opentelemetry.attributes import BoundedAttributes -from opentelemetry.context import _SUPPRESS_INSTRUMENTATION_KEY, Context, get_value +from opentelemetry.context import ( + _SUPPRESS_INSTRUMENTATION_KEY, + Context, + attach, + detach, + get_current, + get_value, +) from opentelemetry.sdk.trace.id_generator import IdGenerator, RandomIdGenerator from opentelemetry.sdk.trace.sampling import Sampler, SamplingResult from opentelemetry.trace import ( @@ -40,6 +52,7 @@ Status, StatusCode, Tracer, + set_span_in_context, use_span, ) from opentelemetry.util.types import Attributes @@ -380,16 +393,27 @@ def sync_wrapper( args: Tuple[Any, ...], kwargs: Dict[str, Any], ) -> ReturnType: - tracer = self - with _chain_context( - tracer=tracer, + context = partial( + _chain_context, + tracer=self, name=name, kind=kind, wrapped=wrapped, instance=instance, args=args, kwargs=kwargs, - ) as chain_context: + ) + if inspect.isgeneratorfunction(wrapped): + return cast( + ReturnType, + _trace_generator(wrapped(*args, **kwargs), context(streaming=True)), + ) + if inspect.isasyncgenfunction(wrapped): + return cast( + ReturnType, + _trace_async_generator(wrapped(*args, **kwargs), context(streaming=True)), + ) + with context() as chain_context: output = wrapped(*args, **kwargs) return chain_context.process_output(output) @@ -466,9 +490,9 @@ def sync_wrapper( args: Tuple[Any, ...], kwargs: Dict[str, Any], ) -> ReturnType: - tracer = self - with _tool_context( - tracer=tracer, + context = partial( + _tool_context, + tracer=self, name=name, description=description, parameters=parameters, @@ -476,7 +500,18 @@ def sync_wrapper( instance=instance, args=args, kwargs=kwargs, - ) as tool_context: + ) + if inspect.isgeneratorfunction(wrapped): + return cast( + ReturnType, + _trace_generator(wrapped(*args, **kwargs), context(streaming=True)), + ) + if inspect.isasyncgenfunction(wrapped): + return cast( + ReturnType, + _trace_async_generator(wrapped(*args, **kwargs), context(streaming=True)), + ) + with context() as tool_context: output = wrapped(*args, **kwargs) return tool_context.process_output(output) @@ -532,6 +567,7 @@ def _chain_context( instance: Any, args: Tuple[Any, ...], kwargs: Dict[str, Any], + streaming: bool = False, ) -> Iterator[_ChainContext]: span_name = name or _infer_span_name(instance=instance, callable=wrapped) bound_args = inspect.signature(wrapped).bind(*args, **kwargs) @@ -544,7 +580,8 @@ def _chain_context( else: input_attributes = get_input_attributes(arguments) - with tracer.start_as_current_span( + start_span = partial(_stream_span, tracer) if streaming else tracer.start_as_current_span + with start_span( span_name, fi_span_kind=kind, attributes=input_attributes, @@ -576,6 +613,7 @@ def _tool_context( instance: Any, args: Tuple[Any, ...], kwargs: Dict[str, Any], + streaming: bool = False, ) -> Iterator[_ToolContext]: tool_name = name or _infer_span_name(instance=instance, callable=wrapped) bound_args = inspect.signature(wrapped).bind(*args, **kwargs) @@ -593,7 +631,8 @@ def _tool_context( description=tool_description, parameters=tool_parameters, ) - with tracer.start_as_current_span( + start_span = partial(_stream_span, tracer) if streaming else tracer.start_as_current_span + with start_span( tool_name, fi_span_kind=FiSpanKindValues.TOOL, attributes={ @@ -605,6 +644,114 @@ def _tool_context( yield context +@contextmanager +def _stream_span(tracer: FITracer, name: str, **kwargs: Any) -> Iterator[FiSpan]: + """Own the lifetime without leaving a current-span token across a yield.""" + span = tracer.start_span(name, **kwargs) + try: + yield span + except GeneratorExit: + # Explicit close/aclose is normal termination, not an application error. + raise + except BaseException as exception: + if span.is_recording(): + span.record_exception(exception, escaped=True) + span.set_status( + Status(StatusCode.ERROR, f"{type(exception).__name__}: {exception}") + ) + raise + finally: + span.end() + + +class _Stream: + """Resume only the producer under its saved OTel context. + + A child span may itself cross a yield. Save that context as well, but restore + the consumer's context after every operation. Do not copy the Python context: + application ContextVar writes must still behave like an undecorated generator. + Only the last yielded value is retained, so infinite streams are not buffered. + """ + + def __init__(self, generator: Any, span: FiSpan) -> None: + self.generator = generator + self.context = set_span_in_context(cast(Span, span)) + self.output: Any = None + + @contextmanager + def activate(self) -> Iterator[None]: + token = attach(self.context) + try: + yield + finally: + self.context = get_current() + detach(token) + + def __iter__(self) -> "_Stream": + return self + + def __next__(self) -> Any: + return self.send(None) + + def send(self, value: Any) -> Any: + with self.activate(): + self.output = self.generator.send(value) + return self.output + + def throw(self, *args: Any) -> Any: + with self.activate(): + self.output = self.generator.throw(*args) + return self.output + + def close(self) -> Any: + with self.activate(): + return self.generator.close() + + async def step(self, method: Callable[..., Awaitable[Any]], *args: Any) -> Any: + with self.activate(): + self.output = await method(*args) + return self.output + + +def _trace_generator( + generator: Any, + context: ContextManager[Union[_ChainContext, _ToolContext]], +) -> Generator[Any, Any, Any]: + with context as span_context: + stream = _Stream(generator, span_context._span) + try: + # Native delegation preserves send/throw/close and StopIteration.value. + return (yield from stream) + finally: + span_context.process_output(stream.output) + + +async def _trace_async_generator( + generator: Any, + context: ContextManager[Union[_ChainContext, _ToolContext]], +) -> AsyncGenerator[Any, Any]: + with context as span_context: + stream = _Stream(generator, span_context._span) + try: + try: + output = await stream.step(generator.__anext__) + while True: + try: + value = yield output + except GeneratorExit: + with stream.activate(): + await generator.aclose() + raise + except BaseException: + output = await stream.step(generator.athrow, *sys.exc_info()) + else: + output = await stream.step(generator.asend, value) + except StopAsyncIteration: + return + finally: + span_context.process_output(stream.output) + + def _infer_span_name(*, instance: Any, callable: Callable[..., Any]) -> str: """ Makes a best-effort attempt to infer a span name from the bound instance diff --git a/python/tests/test_custom_eval_model.py b/python/tests/test_custom_eval_model.py new file mode 100644 index 000000000..11e04aa97 --- /dev/null +++ b/python/tests/test_custom_eval_model.py @@ -0,0 +1,116 @@ +"""Regression coverage for nullable custom-evaluation models (T1-PY-CORE-005).""" + +import json +from unittest.mock import patch + +import pytest + +from fi_instrumentation import register +from fi_instrumentation.fi_types import ( + EvalName, + EvalSpanKind, + EvalTag, + EvalTagType, + ModelChoices, + ProjectType, + prepare_eval_tags, +) +from fi_instrumentation.otel import EVAL_TAGS + + +@pytest.fixture +def custom_eval_template(): + with patch( + "fi_instrumentation.fi_types.get_custom_eval_template", + return_value={ + "isUserEvalTemplate": True, + "evalTemplate": {"config": {"requiredKeys": ["output"]}}, + }, + ) as lookup: + yield lookup + + +def test_custom_eval_without_model_serializes(custom_eval_template): + tag = EvalTag( + type=EvalTagType.OBSERVATION_SPAN, + value=EvalSpanKind.LLM, + eval_name="probe_custom_eval", + mapping={"output": "output.value"}, + ) + + assert tag.model is None + custom_eval_template.assert_called_once_with("probe_custom_eval") + serialized = tag.to_dict() + assert serialized["model"] is None + assert prepare_eval_tags([tag]) == [serialized] + + +@pytest.mark.parametrize( + ("model_kwargs", "expected_model"), + [ + ({}, None), + ({"model": None}, None), + ({"model": "custom-model"}, "custom-model"), + ({"model": ModelChoices.TURING_LARGE}, ModelChoices.TURING_LARGE.value), + ], + ids=["omitted", "explicit-none", "string", "enum"], +) +def test_register_custom_eval_model( + custom_eval_template, model_kwargs, expected_model +): + tag = EvalTag( + type=EvalTagType.OBSERVATION_SPAN, + value=EvalSpanKind.LLM, + eval_name="probe_custom_eval", + mapping={"output": "output.value"}, + **model_kwargs, + ) + expected_tags = [{ + "type": EvalTagType.OBSERVATION_SPAN.value, + "value": EvalSpanKind.LLM.value, + "eval_name": "probe_custom_eval", + "config": {}, + "mapping": {"output": "output.value"}, + "custom_eval_name": "probe_custom_eval", + "model": expected_model, + }] + + with patch( + "fi_instrumentation.otel.check_custom_eval_config_exists", return_value=False + ) as check_config, patch( + "fi_instrumentation.otel.TracerProvider" + ) as provider_class, patch( + "fi_instrumentation.otel.BatchSpanProcessor" + ) as processor_class: + provider = register( + project_name="probe", + project_version_name="test-version", + eval_tags=[tag], + verbose=False, + ) + + custom_eval_template.assert_called_once_with("probe_custom_eval") + check_config.assert_called_once_with( + project_name="probe", + project_type=ProjectType.EXPERIMENT.value, + eval_tags=expected_tags, + ) + provider_class.assert_called_once() + assert provider is provider_class.return_value + resource = provider_class.call_args.kwargs["resource"] + assert json.loads(resource.attributes[EVAL_TAGS]) == expected_tags + processor_class.assert_called_once() + provider.add_span_processor.assert_called_once_with(processor_class.return_value) + + +def test_system_eval_still_requires_model(): + with patch( + "fi_instrumentation.fi_types.get_custom_eval_template", + return_value={"isUserEvalTemplate": False, "evalTemplate": {}}, + ), pytest.raises(ValueError, match="model must be a of type ModelChoices"): + EvalTag( + type=EvalTagType.OBSERVATION_SPAN, + value=EvalSpanKind.LLM, + eval_name=EvalName.TOXICITY, + mapping={"input": "input.value"}, + ) diff --git a/python/tests/test_session_attributes.py b/python/tests/test_session_attributes.py new file mode 100644 index 000000000..06d165e5a --- /dev/null +++ b/python/tests/test_session_attributes.py @@ -0,0 +1,78 @@ +"""Regression coverage for T1-PY-CORE-006 manual session attributes.""" + +import json +from unittest.mock import Mock + +import pytest +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import ( + SimpleSpanProcessor, + SpanExporter, + SpanExportResult, +) +from opentelemetry.sdk.trace.sampling import ALWAYS_ON + +from fi_instrumentation.instrumentation import ( + get_context_attributes, + get_session_attributes, +) + + +@pytest.mark.parametrize( + "helper", + [get_session_attributes, get_context_attributes], + ids=["session", "context"], +) +@pytest.mark.parametrize("session_id", ["session-A", ""], ids=["nonempty", "empty"]) +def test_manual_session_helpers_emit_both_keys(helper, session_id): + assert helper(session_id=session_id) == { + "session.id": session_id, + "gen_ai.conversation.id": session_id, + } + + +def test_context_session_preserves_other_attributes(): + attributes = get_context_attributes( + session_id="session-A", + user_id="user-A", + metadata={"source": "unit-test"}, + tags=["manual"], + ) + + assert attributes.get("session.id") == "session-A" + assert attributes["gen_ai.conversation.id"] == "session-A" + assert attributes["user.id"] == "user-A" + assert json.loads(attributes["metadata"]) == {"source": "unit-test"} + assert attributes["tag.tags"] == ["manual"] + + +@pytest.mark.parametrize("kwargs", [{}, {"session_id": None}], ids=["omitted", "none"]) +def test_context_without_session_omits_both_keys(kwargs): + assert get_context_attributes(user_id="user-A", **kwargs) == {"user.id": "user-A"} + + +@pytest.mark.parametrize( + "helper", + [get_session_attributes, get_context_attributes], + ids=["session", "context"], +) +def test_manual_session_helpers_export_native_session_in_normal_scope(helper): + # Exercise ordinary OTel spans without a collector or network exporter. + exporter = Mock(spec=SpanExporter) + exporter.export.return_value = SpanExportResult.SUCCESS + provider = TracerProvider(sampler=ALWAYS_ON, shutdown_on_exit=False) + provider.add_span_processor(SimpleSpanProcessor(exporter)) + try: + with provider.get_tracer("my-app").start_as_current_span( + "manual-operation", attributes=helper(session_id="session-A") + ): + pass + finally: + provider.shutdown() + + exporter.export.assert_called_once() + (span,) = exporter.export.call_args.args[0] + assert span.instrumentation_scope.name == "my-app" + assert span.attributes["gen_ai.conversation.id"] == "session-A" + # Native FutureAGI session identity uses session.id, not the conversation key. + assert span.attributes.get("session.id") == "session-A" diff --git a/python/tests/test_tracer_generators.py b/python/tests/test_tracer_generators.py new file mode 100644 index 000000000..8fbc3dd46 --- /dev/null +++ b/python/tests/test_tracer_generators.py @@ -0,0 +1,282 @@ +"""T1-PY-CORE-002: offline decorator regressions using fake stream functions.""" + +import asyncio +from contextvars import ContextVar + +import pytest +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +from opentelemetry.trace import StatusCode, get_current_span + +from fi_instrumentation.fi_types import SpanAttributes +from fi_instrumentation.instrumentation._tracers import FITracer +from fi_instrumentation.instrumentation.config import TraceConfig + + +@pytest.fixture(params=["chain", "agent", "tool"]) +def tracing(request): + exporter = InMemorySpanExporter() + provider = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + tracer = FITracer(provider.get_tracer("generator-regression"), TraceConfig()) + yield tracer, exporter, request.param + provider.shutdown() + + +@pytest.fixture(params=[False, True], ids=["bare", "named"]) +def decorate(tracing, request): + tracer, _, kind = tracing + decorator = getattr(tracer, kind) + return decorator(name="stream") if request.param else decorator + + +def finished(exporter, name="stream"): + return [span for span in exporter.get_finished_spans() if span.name == name] + + +def assert_error(exporter): + spans = finished(exporter) + assert len(spans) == 1 + assert spans[0].status.status_code is StatusCode.ERROR + assert any(event.name == "exception" for event in spans[0].events) + + +def test_sync_generator_lifetime_output_parent_and_return(tracing, decorate): + tracer, exporter, kind = tracing + application_state = ContextVar("application_state", default="before") + result = object() + + @decorate + def stream(): + application_state.set("inside") + with tracer.start_as_current_span("work", fi_span_kind="chain"): + yield "first" + yield "second" + return result + + iterator = stream() + assert exporter.get_finished_spans() == () + with tracer.start_as_current_span("caller", fi_span_kind="chain") as caller: + assert next(iterator) == "first" + assert application_state.get() == "inside" + assert get_current_span() is caller + assert not finished(exporter) + with tracer.start_as_current_span("consumer", fi_span_kind="chain"): + pass + assert next(iterator) == "second" + assert get_current_span() is caller + assert not finished(exporter) + with pytest.raises(StopIteration) as stopped: + next(iterator) + assert stopped.value.value is result + assert get_current_span() is caller + + span, = finished(exporter) + child, = finished(exporter, "work") + consumer, = finished(exporter, "consumer") + assert span.attributes[SpanAttributes.GEN_AI_SPAN_KIND] == kind.upper() + assert span.attributes[SpanAttributes.OUTPUT_VALUE] == "second" + assert span.parent.span_id == caller.get_span_context().span_id + assert child.parent.span_id == span.context.span_id + assert consumer.parent.span_id == caller.get_span_context().span_id + assert span.start_time <= child.start_time <= child.end_time <= span.end_time + + +def test_sync_generator_late_exception(tracing, decorate): + tracer, exporter, _ = tracing + failure = ValueError("fake stream failure") + + @decorate + def stream(): + with tracer.start_as_current_span("work", fi_span_kind="chain"): + yield "partial" + raise failure + + iterator = stream() + assert not finished(exporter) + assert next(iterator) == "partial" + with pytest.raises(ValueError) as raised: + next(iterator) + assert raised.value is failure + assert_error(exporter) + assert finished(exporter)[0].attributes[SpanAttributes.OUTPUT_VALUE] == "partial" + + +def test_sync_generator_send_throw_and_close(tracing, decorate): + tracer, exporter, _ = tracing + cleanup_parents = [] + + @decorate + def stream(): + try: + value = yield "ready" + try: + yield value + except LookupError: + yield "recovered" + finally: + cleanup_parents.append(get_current_span().get_span_context().span_id) + + iterator = stream() + assert not finished(exporter) + with pytest.raises(TypeError): + iterator.send("not started") + assert not finished(exporter) + assert next(iterator) == "ready" + assert iterator.send("sent") == "sent" + assert iterator.throw(LookupError("fake recoverable failure")) == "recovered" + assert not finished(exporter) + iterator.close() + iterator.close() + span, = finished(exporter) + assert cleanup_parents == [span.context.span_id] + assert span.attributes[SpanAttributes.OUTPUT_VALUE] == "recovered" + assert span.status.status_code is not StatusCode.ERROR + + +def test_unstarted_generators_and_argument_errors(tracing, decorate): + _, exporter, _ = tracing + entered = [] + + @decorate + def stream(required): + entered.append(True) + yield required + + with pytest.raises(TypeError): + stream() + iterator = stream("unused") + iterator.close() + assert not entered + assert not finished(exporter) + + +def test_async_generator_lifetime_output_and_parent(tracing, decorate): + tracer, exporter, _ = tracing + application_state = ContextVar("async_application_state", default="before") + + @decorate + async def stream(): + application_state.set("inside") + with tracer.start_as_current_span("work", fi_span_kind="chain"): + await asyncio.sleep(0) + yield "first" + await asyncio.sleep(0) + yield "second" + + async def consume(): + iterator = stream() + assert not finished(exporter) + with tracer.start_as_current_span("caller", fi_span_kind="chain") as caller: + assert await iterator.__anext__() == "first" + assert application_state.get() == "inside" + assert get_current_span() is caller + assert not finished(exporter) + assert await iterator.__anext__() == "second" + assert get_current_span() is caller + assert not finished(exporter) + with pytest.raises(StopAsyncIteration): + await iterator.__anext__() + assert get_current_span() is caller + + asyncio.run(consume()) + span, = finished(exporter) + child, = finished(exporter, "work") + caller, = finished(exporter, "caller") + assert span.attributes[SpanAttributes.OUTPUT_VALUE] == "second" + assert span.parent.span_id == caller.context.span_id + assert child.parent.span_id == span.context.span_id + assert span.start_time <= child.start_time <= child.end_time <= span.end_time + + +def test_async_generator_asend_athrow_and_aclose(tracing, decorate): + _, exporter, _ = tracing + cleanup_parents = [] + + @decorate + async def stream(): + try: + value = yield "ready" + try: + yield value + except LookupError: + yield "recovered" + finally: + await asyncio.sleep(0) + cleanup_parents.append(get_current_span().get_span_context().span_id) + + async def consume(): + iterator = stream() + assert not finished(exporter) + with pytest.raises(TypeError): + await iterator.asend("not started") + assert not finished(exporter) + assert await iterator.__anext__() == "ready" + assert await iterator.asend("sent") == "sent" + assert await iterator.athrow(LookupError("fake recoverable failure")) == "recovered" + assert not finished(exporter) + await iterator.aclose() + await iterator.aclose() + + asyncio.run(consume()) + span, = finished(exporter) + assert cleanup_parents == [span.context.span_id] + assert span.attributes[SpanAttributes.OUTPUT_VALUE] == "recovered" + assert span.status.status_code is not StatusCode.ERROR + + +def test_async_generator_late_exception(tracing, decorate): + _, exporter, _ = tracing + failure = ValueError("fake async stream failure") + + @decorate + async def stream(): + yield "partial" + raise failure + + async def consume(): + iterator = stream() + assert not finished(exporter) + assert await iterator.__anext__() == "partial" + with pytest.raises(ValueError) as raised: + await iterator.__anext__() + assert raised.value is failure + + asyncio.run(consume()) + assert_error(exporter) + assert finished(exporter)[0].attributes[SpanAttributes.OUTPUT_VALUE] == "partial" + + +def test_async_generator_cancellation(tracing, decorate): + _, exporter, _ = tracing + + @decorate + async def stream(): + yield "ready" + raise asyncio.CancelledError() + + async def consume(): + iterator = stream() + assert not finished(exporter) + assert await iterator.__anext__() == "ready" + with pytest.raises(asyncio.CancelledError): + await iterator.__anext__() + + asyncio.run(consume()) + assert_error(exporter) + + +def test_explicit_stream_output_is_preserved(tracing, decorate): + _, exporter, _ = tracing + + @decorate + def stream(): + get_current_span().set_output("explicit output") + yield "automatic output" + + iterator = stream() + assert not finished(exporter) + assert list(iterator) == ["automatic output"] + span, = finished(exporter) + assert span.attributes[SpanAttributes.OUTPUT_VALUE] == "explicit output"