From f6622109d68e3056445aec8da4a717012616e32d Mon Sep 17 00:00:00 2001 From: nik13 Date: Mon, 26 Oct 2026 23:50:33 -0700 Subject: [PATCH 1/2] test(python): reconcile native core privacy and exporter contracts Preserve published-core expectations while asserting current error-detail suppression, message normalization, and delivery to registered plus later exporters. Reproduced the same 66 failures on pre-batch and current dev with identical dependency freezes. Related TH-8103. Co-Authored-By: Claude Code --- .../ag2-classic/tests/test_setup.py | 44 ++++++++++------- python/frameworks/ag2/tests/test_processor.py | 43 +++++++++++------ .../exa/tests/test_privacy_config.py | 35 +++++++++++--- .../mcp-use/tests/test_mcp_use_agent.py | 9 ++-- .../mcp-use/tests/test_mcp_use_callback.py | 16 ++++--- .../openai-realtime/tests/test_connect.py | 9 +++- .../openai-realtime/tests/test_privacy.py | 11 ++++- .../parallel/tests/test_parallel_config.py | 36 +++++++++----- .../parallel/tests/test_parallel_errors.py | 7 ++- .../replicate/tests/test_no_stacktrace.py | 12 ++++- python/frameworks/replicate/tests/test_run.py | 14 ++++-- .../tavily/tests/test_contract_harness.py | 9 +++- python/frameworks/tavily/tests/test_errors.py | 7 ++- .../frameworks/tavily/tests/test_isolation.py | 10 +++- .../frameworks/tavily/tests/test_privacy.py | 48 ++++++++++++++----- .../frameworks/typesafe/tests/test_errors.py | 9 +++- python/frameworks/voyage/tests/test_async.py | 8 +++- python/frameworks/voyage/tests/test_errors.py | 12 +++-- 18 files changed, 251 insertions(+), 88 deletions(-) diff --git a/python/frameworks/ag2-classic/tests/test_setup.py b/python/frameworks/ag2-classic/tests/test_setup.py index aa47805c2..8bf1fde2d 100644 --- a/python/frameworks/ag2-classic/tests/test_setup.py +++ b/python/frameworks/ag2-classic/tests/test_setup.py @@ -175,43 +175,53 @@ def test_register_then_setup_keeps_one_provider_resource_and_headers(monkeypatch def test_add_span_processor_after_register_and_setup_keeps_our_processor_first(monkeypatch, fake): - """fi's TracerProvider.add_span_processor shuts down and clears every - processor, ours included (fi_instrumentation/otel.py:336-339). setup() - re-prepends and re-enables ours so the new processor still sees filtered spans. + """Every active exporter receives filtered spans after a later processor add. + + Published core 1.1 replaces register's exporter on the first add. Current + core keeps it (#281); setup must normalize before BOTH exporters, not + silently restore the old replacement semantics. """ - from fi_instrumentation import register + from fi_instrumentation import TraceConfig, register from fi_instrumentation.fi_types import ProjectType monkeypatch.setenv("FI_API_KEY", "placeholder-api-key") monkeypatch.setenv("FI_SECRET_KEY", "placeholder-secret-key") monkeypatch.setenv("FI_BASE_URL", "http://127.0.0.1:9") saved = {sig: signal.getsignal(sig) for sig in (signal.SIGINT, signal.SIGTERM)} + original = InMemorySpanExporter() provider = register( project_type=ProjectType.OBSERVE, project_name="ag2-classic-add", verbose=False, + batch=False, + span_exporter=original, set_global_tracer_provider=False, ) tracing = setup(tracer_provider=provider, capture_content=False) memory = InMemorySpanExporter() + second = InMemorySpanExporter() try: - added = SimpleSpanProcessor(memory) - provider.add_span_processor(added) + provider.add_span_processor(SimpleSpanProcessor(memory)) + provider.add_span_processor(SimpleSpanProcessor(second)) processors = provider._active_span_processor._span_processors - assert processors == (tracing.processor, added) + assert sum(p is tracing.processor for p in processors) == 1 assert tracing.processor._disabled is False two_agent_tool_chat(tracing, fake) - spans = memory.get_finished_spans() - assert any(s.name == "conversation user" for s in spans) - _assert_no_content_and_no_promoted_usage_off_llm(spans) - conversation = next(s for s in spans if s.name == "conversation user").attributes - assert conversation["gen_ai.span.kind"] == "CHAIN" - - # A second add keeps exactly one copy of ours, still first. - second = SimpleSpanProcessor(InMemorySpanExporter()) - provider.add_span_processor(second) - assert provider._active_span_processor._span_processors == (tracing.processor, added, second) + assert provider.force_flush(timeout_millis=10_000) + expected_ids = {s.context.span_id for s in memory.get_finished_spans()} + exporters = [memory, second] + if hasattr(TraceConfig(), "hide_error_details"): + exporters.append(original) # Current register() keeps its exporter. + else: + assert original.get_finished_spans() == () # Published 1.1 contract. + for exporter in exporters: + spans = exporter.get_finished_spans() + assert {s.context.span_id for s in spans} == expected_ids + assert any(s.name == "conversation user" for s in spans) + _assert_no_content_and_no_promoted_usage_off_llm(spans) + conversation = next(s for s in spans if s.name == "conversation user").attributes + assert conversation["gen_ai.span.kind"] == "CHAIN" finally: tracing.uninstrument() provider.shutdown() diff --git a/python/frameworks/ag2/tests/test_processor.py b/python/frameworks/ag2/tests/test_processor.py index 4aa4dc22a..e6e98450e 100644 --- a/python/frameworks/ag2/tests/test_processor.py +++ b/python/frameworks/ag2/tests/test_processor.py @@ -778,25 +778,40 @@ def _fi_provider(exporter): def test_later_add_span_processor_on_fi_provider_keeps_ag2_processor_first(): - """fi's provider shuts down and clears every processor on the first - add_span_processor() after register() (otel.py 329-340); the user's new - exporter must still receive normalized AG2 spans.""" - provider = _fi_provider(InMemorySpanExporter()) + """Current register() keeps its exporter on later adds (#281). Both it + and the added exporter must receive exactly one normalized, private span. + Published core 1.1 still replaces the original exporter on the first add. + """ + original = InMemorySpanExporter() + provider = _fi_provider(original) try: - assert install_span_processor(provider) is True - replacement = InMemorySpanExporter() - provider.add_span_processor(SimpleSpanProcessor(replacement)) + assert install_span_processor(provider, config=TraceConfig(hide_inputs=True)) is True + added = InMemorySpanExporter() + provider.add_span_processor(SimpleSpanProcessor(added)) chain = provider._active_span_processor._span_processors - assert isinstance(chain[0], AG2SpanProcessor), chain assert sum(isinstance(p, AG2SpanProcessor) for p in chain) == 1 - assert [type(p) for p in chain[1:]] == [SimpleSpanProcessor] - _emit(provider, {"gen_ai.operation.name": "chat", "gen_ai.usage.thinking_tokens": 2}) - (span,) = replacement.get_finished_spans() - got = attrs(span) - assert got["gen_ai.span.kind"] == "LLM" - assert got["gen_ai.usage.reasoning.output_tokens"] == 2 + _emit(provider, { + "gen_ai.operation.name": "chat", + "gen_ai.usage.thinking_tokens": 2, + "gen_ai.input.messages": '[{"content": "PRIVATE-LATER-PROCESSOR"}]', + }) + assert provider.force_flush() + exporters = [added] + if hasattr(TraceConfig(), "hide_error_details"): + exporters.append(original) + else: + assert original.get_finished_spans() == () + ids = [] + for exporter in exporters: + (span,) = exporter.get_finished_spans() + ids.append(span.context.span_id) + got = attrs(span) + assert got["gen_ai.span.kind"] == "LLM" + assert got["gen_ai.usage.reasoning.output_tokens"] == 2 + assert "PRIVATE-LATER-PROCESSOR" not in span.to_json() + assert len(set(ids)) == 1 finally: provider.shutdown() diff --git a/python/frameworks/exa/tests/test_privacy_config.py b/python/frameworks/exa/tests/test_privacy_config.py index 2afb242fe..2d48f1b03 100644 --- a/python/frameworks/exa/tests/test_privacy_config.py +++ b/python/frameworks/exa/tests/test_privacy_config.py @@ -23,6 +23,9 @@ from exa_py import AsyncExa, Exa # noqa: E402 from fi_instrumentation import TraceConfig, using_session # noqa: E402 +# Published core 1.1 retains the legacy descriptions; current core owns error privacy. +NATIVE_ERROR_POLICY = hasattr(TraceConfig(), "hide_error_details") + HIDDEN = "__REDACTED__" PRIVATE = "private question" @@ -167,7 +170,7 @@ def test_error_text_has_the_key_redacted_and_is_capped(fake, call): @pytest.mark.parametrize("call", _echo_error_cases(), ids=["search", "stream", "async"]) -def test_hidden_inputs_also_leave_error_text(fake, call): +def test_hidden_inputs_apply_the_core_error_policy(fake, call): query = "{0} {1}".format(ECHO_QUERY, PRIVATE) with instrumented(config=TraceConfig(hide_inputs=True)) as traced: with pytest.raises(ValueError) as caught: @@ -176,8 +179,15 @@ def test_hidden_inputs_also_leave_error_text(fake, call): assert PRIVATE in str(caught.value) span = traced.one() assert PRIVATE not in traced.wire() - assert HIDDEN in span.status.description - assert HIDDEN in _exception_event(span)["exception.message"] + event = _exception_event(span) + assert event["exception.type"] == "ValueError" + if NATIVE_ERROR_POLICY: + assert span.status.description is None + assert "exception.message" not in event + assert "exception.stacktrace" not in event + else: + assert HIDDEN in span.status.description + assert HIDDEN in event["exception.message"] def test_short_error_text_is_kept_whole(fake): @@ -186,7 +196,8 @@ def test_short_error_text_is_kept_whole(fake): _client(fake).search(FAIL_QUERY) span = traced.one() - assert span.status.description == "ValueError: {0}".format(caught.value) + expected = str(caught.value) if NATIVE_ERROR_POLICY else "ValueError: {0}".format(caught.value) + assert span.status.description == expected assert _exception_event(span)["exception.message"] == str(caught.value) @@ -242,7 +253,9 @@ def test_sync_stream_failing_mid_iteration_is_an_error_not_a_cancel(): next(stream) span = traced.one() - assert span.status.description == "ConnectionError: connection dropped" + expected = "connection dropped" if NATIVE_ERROR_POLICY else "ConnectionError: connection dropped" + assert span.status.description == expected + assert _exception_event(span)["exception.type"] == "ConnectionError" assert "exa.cancelled" not in attrs(span) assert "exa.result_count" not in attrs(span) @@ -261,7 +274,9 @@ async def run(traced) -> None: asyncio.run(run(traced)) span = traced.one() - assert span.status.description == "ConnectionError: connection dropped" + expected = "connection dropped" if NATIVE_ERROR_POLICY else "ConnectionError: connection dropped" + assert span.status.description == expected + assert _exception_event(span)["exception.type"] == "ConnectionError" assert "exa.cancelled" not in attrs(span) @@ -296,7 +311,13 @@ def test_hidden_inputs_remove_requested_urls_from_error_text(fake): span = traced.one() assert "example.com" not in traced.wire() assert EXA_KEY not in traced.wire() - assert HIDDEN in span.status.description + if NATIVE_ERROR_POLICY: + assert span.status.description is None + assert "exception.message" not in _exception_event(span) + assert "exception.stacktrace" not in _exception_event(span) + else: + assert HIDDEN in span.status.description + assert _exception_event(span)["exception.type"] == "ValueError" assert attrs(span)["exa.url_count"] == 2 diff --git a/python/frameworks/mcp-use/tests/test_mcp_use_agent.py b/python/frameworks/mcp-use/tests/test_mcp_use_agent.py index 28e7aa4e7..426fae433 100644 --- a/python/frameworks/mcp-use/tests/test_mcp_use_agent.py +++ b/python/frameworks/mcp-use/tests/test_mcp_use_agent.py @@ -54,6 +54,9 @@ from traceai_mcp_use import FutureAGICallback +# Published core 1.1 retains the legacy descriptions; current core owns error privacy. +NATIVE_ERROR_POLICY = hasattr(TraceConfig(), "hide_error_details") + pytest.importorskip("uvicorn", reason="the loopback MCP server runs on uvicorn") @@ -186,7 +189,7 @@ def test_mcp_tool_error_fails_the_tool_span_and_the_agent_recovers(): spans = exporter.get_finished_spans() tool = only(spans, "execute_tool fail") assert tool.status.status_code is StatusCode.ERROR - assert tool.status.description == "RuntimeError" + assert tool.status.description == (None if NATIVE_ERROR_POLICY else "RuntimeError") assert tool.attributes["mcp_use.tool.error_type"] == "RuntimeError" assert [event.attributes["exception.type"] for event in tool.events] == ["RuntimeError"] agent = only(spans, AGENT) @@ -215,7 +218,7 @@ def test_tool_error_text_is_recorded_by_default_and_dropped_with_hide_inputs(): run_agent([tool_call("fail", {"reason": ARG}), answer()], [handler]) spans = exporter.get_finished_spans() tool = only(spans, "execute_tool fail") - assert tool.status.description == "RuntimeError" + assert tool.status.description == (None if NATIVE_ERROR_POLICY else "RuntimeError") (event,) = tool.events assert dict(event.attributes) == {"exception.type": "RuntimeError"} assert ARG not in wire(spans) and "PROMPT-MARKER" not in wire(spans) @@ -299,7 +302,7 @@ def test_hide_outputs_keeps_earlier_turns_out_of_later_turns(method, external): # outputs) is dropped, the exception type kept. assert attrs(only(third, AGENT))["input.value"] == "THIRD-PROMPT" for span in (only(third, AGENT), only(third, LLM_SPAN)): - assert span.status.description == "ValueError" + assert span.status.description == (None if NATIVE_ERROR_POLICY else "ValueError") (event,) = span.events assert dict(event.attributes) == {"exception.type": "ValueError"} diff --git a/python/frameworks/mcp-use/tests/test_mcp_use_callback.py b/python/frameworks/mcp-use/tests/test_mcp_use_callback.py index 32f6634d1..c9777010a 100644 --- a/python/frameworks/mcp-use/tests/test_mcp_use_callback.py +++ b/python/frameworks/mcp-use/tests/test_mcp_use_callback.py @@ -59,6 +59,9 @@ scrub_local, ) +# Published core 1.1 retains the legacy descriptions; current core owns error privacy. +NATIVE_ERROR_POLICY = hasattr(TraceConfig(), "hide_error_details") + TOOL_SPAN = "execute_tool add" METADATA = { "ls_provider": "fake", @@ -359,7 +362,7 @@ def test_hide_outputs_drops_earlier_turns_replayed_as_input(): assert llm_span["input.value"] == PROMPT for span in spans: # Error text is content: dropped with either hide flag. - assert span.status.description == "ValueError" + assert span.status.description == (None if NATIVE_ERROR_POLICY else "ValueError") def test_hide_outputs_with_a_replayed_reply_last_redacts_the_llm_input_value(): @@ -405,7 +408,8 @@ def test_tool_error_records_its_message_by_default_and_never_a_stack_trace(): spans = exporter.get_finished_spans() tool = only(spans, "execute_tool fail") assert tool.status.status_code is StatusCode.ERROR - assert tool.status.description == "ValueError: tool failed because " + ARG + expected = ("" if NATIVE_ERROR_POLICY else "ValueError: ") + "tool failed because " + ARG + assert tool.status.description == expected (event,) = tool.events assert event.name == "exception" assert dict(event.attributes) == { @@ -431,7 +435,7 @@ def test_either_hide_flag_drops_the_error_text(config): spans = exporter.get_finished_spans() tool = only(spans, "execute_tool fail") assert tool.status.status_code is StatusCode.ERROR - assert tool.status.description == "ValueError" + assert tool.status.description == (None if NATIVE_ERROR_POLICY else "ValueError") assert dict(tool.events[0].attributes) == {"exception.type": "ValueError"} assert "SERVER-SIDE-TEXT" not in wire(spans) and "tool failed because" not in wire(spans) @@ -629,7 +633,7 @@ def test_llm_error_fails_the_llm_and_agent_spans(hidden): for span in spans: assert span.status.status_code is StatusCode.ERROR if hidden: - assert span.status.description == "ConnectionError" + assert span.status.description == (None if NATIVE_ERROR_POLICY else "ConnectionError") else: assert span.status.description == "ConnectionError: provider down " + PROMPT assert [event.name for event in span.events] == ["exception"] @@ -671,7 +675,7 @@ def test_mcp_use_formatted_tool_error_is_an_error_span(): if capture: assert span.status.description == "RuntimeError: Error executing tool fail: tool failed because " + ARG else: - assert span.status.description == "RuntimeError" + assert span.status.description == (None if NATIVE_ERROR_POLICY else "RuntimeError") assert ARG not in wire(spans) @@ -978,7 +982,7 @@ def test_hide_flags_fail_closed_when_a_run_remembers_too_much(monkeypatch): for marker in CONTENT_MARKERS + (ANSWER,): assert marker not in text, marker tool = only(spans, "execute_tool fail") - assert tool.status.description == "ValueError" + assert tool.status.description == (None if NATIVE_ERROR_POLICY else "ValueError") assert dict(tool.events[0].attributes) == {"exception.type": "ValueError"} diff --git a/python/frameworks/openai-realtime/tests/test_connect.py b/python/frameworks/openai-realtime/tests/test_connect.py index 9f26c6156..9604ebb93 100644 --- a/python/frameworks/openai-realtime/tests/test_connect.py +++ b/python/frameworks/openai-realtime/tests/test_connect.py @@ -9,6 +9,7 @@ import time import pytest +from fi_instrumentation import TraceConfig from openai import AsyncOpenAI, OpenAI from websockets.exceptions import InvalidStatus from openai.resources.realtime import realtime as rt @@ -20,6 +21,9 @@ from traceai_openai_realtime._session import RESPONSE_NAME, ROOT_NAME, TOOL_NAME, TURN_NAME from voice_tab import assert_voice_tab +# Published core 1.1 retains the legacy descriptions; current core owns error privacy. +NATIVE_ERROR_POLICY = hasattr(TraceConfig(), "hide_error_details") + SDK3 = hasattr(rt.AsyncRealtimeConnectionManager, "send") # 3.x: pre-open queue, handlers, reconnect @@ -154,7 +158,7 @@ async def session(): async with _client(server, key=bad_key).realtime.connect(model="gpt-realtime-2.1"): pass - with pytest.raises(InvalidStatus): + with pytest.raises(InvalidStatus) as raised: asyncio.run(session()) spans = exporter.get_finished_spans() root = root_of(spans) @@ -163,7 +167,8 @@ async def session(): assert root.status.status_code == StatusCode.ERROR assert root.attributes["realtime.end_reason"] == "connect_failed" assert root.attributes["fi.span.kind"] == "CHAIN" - assert root.status.description.startswith("InvalidStatus") + expected = str(raised.value) if NATIVE_ERROR_POLICY else "InvalidStatus: " + str(raised.value) + assert root.status.description == expected assert [e.attributes["exception.type"] for e in root.events if e.name == "exception"] == [ "websockets.exceptions.InvalidStatus" ] diff --git a/python/frameworks/openai-realtime/tests/test_privacy.py b/python/frameworks/openai-realtime/tests/test_privacy.py index e9a66c518..900decbc5 100644 --- a/python/frameworks/openai-realtime/tests/test_privacy.py +++ b/python/frameworks/openai-realtime/tests/test_privacy.py @@ -7,6 +7,7 @@ import logging import pytest +from fi_instrumentation import TraceConfig from openai import AsyncOpenAI import voice_agent @@ -14,6 +15,9 @@ from traceai_openai_realtime._session import RESPONSE_NAME, TOOL_NAME, TURN_NAME from voice_tab import all_values, assert_voice_tab +# Published core 1.1 retains the legacy descriptions; current core owns error privacy. +NATIVE_ERROR_POLICY = hasattr(TraceConfig(), "hide_error_details") + CONTENT_KEYS = ( "input.value", "output.value", @@ -97,7 +101,12 @@ def test_hide_switches_through_the_sdk_including_errors_and_a_socket_drop(env, m assert needle not in blob, needle root = root_of(spans) assert root.attributes["realtime.end_reason"] == "socket_closed" - assert root.status.description == "ConnectionClosedError" + assert root.status.description == (None if NATIVE_ERROR_POLICY else "ConnectionClosedError") + assert root.status.status_code.name == "ERROR" + (event,) = [event for event in root.events if event.name == "exception"] + assert event.attributes["exception.type"].endswith("ConnectionClosedError") + assert "exception.message" not in event.attributes + assert "exception.stacktrace" not in event.attributes def test_full_mode_carries_content(instrument, exporter, stub_server): diff --git a/python/frameworks/parallel/tests/test_parallel_config.py b/python/frameworks/parallel/tests/test_parallel_config.py index 725c73685..be8d342e6 100644 --- a/python/frameworks/parallel/tests/test_parallel_config.py +++ b/python/frameworks/parallel/tests/test_parallel_config.py @@ -28,6 +28,9 @@ ) from traceai_parallel import ParallelInstrumentor # noqa: E402 +# Published core 1.1 retains the legacy descriptions; current core owns error privacy. +NATIVE_ERROR_POLICY = hasattr(TraceConfig(), "hide_error_details") + # Dropped by hide_inputs. input.value is kept as TraceConfig's placeholder. INPUT_KEYS = ("gen_ai.retrieval.query", "parallel.urls", "parallel.objective") EMAIL = "jane.doe@example.com" @@ -138,10 +141,17 @@ def test_pii_is_redacted_before_the_cap_so_no_partial_email_survives(fake): assert "jane.doe" not in traced.wire() -def _error_texts(span): +def _error_texts(span, *, hidden=False): """The three recorded error texts: status description, exception message, stacktrace.""" (event,) = span.events assert event.name == "exception" + if hidden and NATIVE_ERROR_POLICY: + assert span.status.status_code is StatusCode.ERROR + assert span.status.description is None + assert event.attributes["exception.type"] == "parallel.BadRequestError" + assert "exception.message" not in event.attributes + assert "exception.stacktrace" not in event.attributes + return [] # There are no text fields to redact under the native privacy policy. return [ span.status.description, event.attributes["exception.message"], @@ -170,7 +180,7 @@ def test_hide_inputs_removes_the_query_from_error_text(fake, pii): assert "rejected request: " + query in str(error) assert attrs(span)["input.value"] == REDACTED_VALUE assert span.status.status_code is StatusCode.ERROR - for text in _error_texts(span): + for text in _error_texts(span, hidden=True): assert "rejected request: " + REDACTED_VALUE in text assert FAIL_ECHO not in text wire = traced.wire() @@ -215,7 +225,7 @@ def test_hide_inputs_removes_every_query_url_and_objective_from_error_text( assert "secret" in str(error) # Each input becomes one placeholder; the server's own words stay. quoted = "rejected request: " + " | ".join([REDACTED_VALUE] * 3) - for text in _error_texts(span): + for text in _error_texts(span, hidden=True): assert quoted in text assert "secret" not in traced.wire() @@ -228,10 +238,14 @@ def test_hidden_inputs_are_replaced_longest_first_in_one_pass(fake): with instrumented(config=TraceConfig(hide_inputs=True)) as traced: _, span = _failed(traced, sync_client(fake).search, **kwargs) - description, message, _ = _error_texts(span) + texts = _error_texts(span, hidden=True) quoted = "rejected request: " + " | ".join([REDACTED_VALUE] * 3) + "'" - assert quoted in description - assert quoted in message + if NATIVE_ERROR_POLICY: + assert texts == [] + else: + description, message, _ = texts + assert quoted in description + assert quoted in message assert "alpha" not in traced.wire() @@ -242,7 +256,7 @@ def test_hide_inputs_removes_the_query_after_the_key_is_redacted(fake): with instrumented(config=TraceConfig(hide_inputs=True)) as traced: _, span = _failed(traced, sync_client(fake).search, search_queries=[query]) - for text in _error_texts(span): + for text in _error_texts(span, hidden=True): assert "rejected request: " + REDACTED_VALUE in text wire = traced.wire() assert PARALLEL_KEY not in wire @@ -274,7 +288,7 @@ def test_without_hide_inputs_error_text_is_recorded_as_thrown(fake): expected = "rejected request: {0} focus | https://x.example/a | https://x.example/b | goal" assert expected.format(FAIL_ECHO) in message description, recorded, stacktrace = _error_texts(span) - assert description == "BadRequestError: " + message + assert description == (message if NATIVE_ERROR_POLICY else "BadRequestError: " + message) assert recorded == message assert message in stacktrace assert REDACTED_VALUE not in traced.wire() @@ -331,7 +345,7 @@ async def run_async() -> None: assert len(sync_spans) == len(async_spans) == 3 for sync_span, async_span in zip(sync_spans[:2], async_spans[:2]): - sync_texts, async_texts = _error_texts(sync_span), _error_texts(async_span) + sync_texts, async_texts = _error_texts(sync_span, hidden=True), _error_texts(async_span, hidden=True) # Status description and exception message match the sync call's; # the stacktraces differ only in their frames. assert async_texts[:2] == sync_texts[:2] @@ -358,8 +372,8 @@ def unreadable(*_args, **_kwargs): ) assert "unread-secret" in str(error) - expected = ["BadRequestError: " + REDACTED_VALUE, REDACTED_VALUE, REDACTED_VALUE] - assert _error_texts(span) == expected + expected = [] if NATIVE_ERROR_POLICY else ["BadRequestError: " + REDACTED_VALUE, REDACTED_VALUE, REDACTED_VALUE] + assert _error_texts(span, hidden=True) == expected assert "secret" not in traced.wire() diff --git a/python/frameworks/parallel/tests/test_parallel_errors.py b/python/frameworks/parallel/tests/test_parallel_errors.py index fedcd3fc1..70e7e6552 100644 --- a/python/frameworks/parallel/tests/test_parallel_errors.py +++ b/python/frameworks/parallel/tests/test_parallel_errors.py @@ -6,6 +6,7 @@ import os import pytest +from fi_instrumentation import TraceConfig pytest.importorskip("parallel", reason="parallel-web must be installed to test its instrumentor") @@ -26,6 +27,9 @@ ) from traceai_parallel import _wrappers # noqa: E402 +# Published core 1.1 retains the legacy descriptions; current core owns error privacy. +NATIVE_ERROR_POLICY = hasattr(TraceConfig(), "hide_error_details") + STACKTRACE_LIMIT = 16 * 1024 @@ -113,7 +117,8 @@ def test_http_500_on_extract_sets_error_and_reraises(fake): span = traced.one() assert span.name == "parallel.extract" assert span.status.status_code is StatusCode.ERROR - assert span.status.description.startswith("InternalServerError: Error code: 500") + prefix = "Error code: 500" if NATIVE_ERROR_POLICY else "InternalServerError: Error code: 500" + assert span.status.description.startswith(prefix) assert _exception_event(span)["exception.type"] == "parallel.InternalServerError" diff --git a/python/frameworks/replicate/tests/test_no_stacktrace.py b/python/frameworks/replicate/tests/test_no_stacktrace.py index f8959f572..f3a89493f 100644 --- a/python/frameworks/replicate/tests/test_no_stacktrace.py +++ b/python/frameworks/replicate/tests/test_no_stacktrace.py @@ -31,6 +31,9 @@ make_client, ) +# Published core 1.1 retains the legacy descriptions; current core owns error privacy. +NATIVE_ERROR_POLICY = hasattr(TraceConfig(), "hide_error_details") + LOCAL = (__file__, os.path.dirname(os.path.dirname(replicate.__file__)), "Traceback", "site-packages") @@ -56,8 +59,13 @@ def test_a_failed_run_exports_type_and_message_but_no_stacktrace(config): assert span.status.status_code is StatusCode.ERROR (event,) = exception_events(span) assert event.attributes["exception.type"] == "replicate.exceptions.ModelError" - expected = "__REDACTED__" if config.hide_outputs else MODEL_ERROR - assert event.attributes["exception.message"] == expected + if NATIVE_ERROR_POLICY and config.hide_outputs: + assert span.status.description is None + assert "exception.message" not in event.attributes + assert MODEL_ERROR not in traced.wire() + else: + expected = "__REDACTED__" if config.hide_outputs else MODEL_ERROR + assert event.attributes["exception.message"] == expected _assert_no_local_detail(traced) diff --git a/python/frameworks/replicate/tests/test_run.py b/python/frameworks/replicate/tests/test_run.py index d2e9c15f8..37cd0d96d 100644 --- a/python/frameworks/replicate/tests/test_run.py +++ b/python/frameworks/replicate/tests/test_run.py @@ -38,6 +38,9 @@ make_client, ) +# Published core 1.1 retains the legacy descriptions; current core owns error privacy. +NATIVE_ERROR_POLICY = hasattr(TraceConfig(), "hide_error_details") + def test_run_text_model_is_one_llm_span(): with FakeReplicate() as fake, instrumented() as traced: @@ -151,11 +154,16 @@ def test_failed_error_string_is_redacted_when_outputs_are_hidden(): client.run(FAIL_MODEL, input={}) span = traced.one() - # AC-05: still an error, the error string is redacted rather than dropped. + # Still an error: current core drops details, legacy core keeps a placeholder. assert span.status.status_code is StatusCode.ERROR - assert span.status.description == "__REDACTED__" + assert span.status.description == (None if NATIVE_ERROR_POLICY else "__REDACTED__") (event,) = exception_events(span) - assert event.attributes["exception.message"] == "__REDACTED__" + assert event.attributes["exception.type"].endswith("ModelError") + assert "exception.stacktrace" not in event.attributes + if NATIVE_ERROR_POLICY: + assert "exception.message" not in event.attributes + else: + assert event.attributes["exception.message"] == "__REDACTED__" assert MODEL_ERROR not in traced.wire() diff --git a/python/frameworks/tavily/tests/test_contract_harness.py b/python/frameworks/tavily/tests/test_contract_harness.py index 57779223e..e36b72d88 100644 --- a/python/frameworks/tavily/tests/test_contract_harness.py +++ b/python/frameworks/tavily/tests/test_contract_harness.py @@ -19,6 +19,7 @@ from typing import Any, Dict, List, Tuple import pytest +from fi_instrumentation import TraceConfig pytest.importorskip("tavily", reason="tavily-python must be installed to test its instrumentor") pytest.importorskip("opentelemetry.proto", reason="the harness decodes OTLP protobuf") @@ -29,6 +30,9 @@ from _tavily_fake import CONTENT_MARKERS, FAIL_401, TAVILY_KEY, FakeTavily # noqa: E402 from harness import Receiver, _flatten_attributes # noqa: E402 +# Published core 1.1 retains the legacy descriptions; current core owns error privacy. +NATIVE_ERROR_POLICY = hasattr(TraceConfig(), "hide_error_details") + FI_API_KEY = "placeholder-fi-api-key" FI_SECRET_KEY = "placeholder-fi-secret-key" PROJECT = "tavily-contract" @@ -132,9 +136,12 @@ def test_real_client_calls_reach_the_collector_contract(journey): assert "tavily.result_count" not in failed assert search_failed["status"]["code"] == "STATUS_CODE_ERROR" assert search_failed["status"]["message"] == ( - "InvalidAPIKeyError: Unauthorized: invalid API key." + "Unauthorized: invalid API key." if NATIVE_ERROR_POLICY + else "InvalidAPIKeyError: Unauthorized: invalid API key." ) assert [event["name"] for event in search_failed.get("events", [])] == ["exception"] + event = _flatten_attributes(search_failed["events"][0]["attributes"]) + assert event["exception.type"].endswith("InvalidAPIKeyError") assert int(_flatten_attributes(async_search["attributes"])["tavily.result_count"]) == 1 diff --git a/python/frameworks/tavily/tests/test_errors.py b/python/frameworks/tavily/tests/test_errors.py index 99e1b597b..e1d315732 100644 --- a/python/frameworks/tavily/tests/test_errors.py +++ b/python/frameworks/tavily/tests/test_errors.py @@ -6,6 +6,7 @@ from typing import Any import pytest +from fi_instrumentation import TraceConfig pytest.importorskip("tavily", reason="tavily-python must be installed to test its instrumentor") @@ -21,6 +22,9 @@ FakeTavily, ) +# Published core 1.1 retains the legacy descriptions; current core owns error privacy. +NATIVE_ERROR_POLICY = hasattr(TraceConfig(), "hide_error_details") + @pytest.fixture() def fake(): @@ -50,7 +54,8 @@ def _exception_events(span: Any) -> list: def _assert_error_span(span: Any, error: BaseException) -> None: assert span.status.status_code is StatusCode.ERROR - assert span.status.description == "{0}: {1}".format(type(error).__name__, error) + expected = str(error) if NATIVE_ERROR_POLICY else "{0}: {1}".format(type(error).__name__, error) + assert span.status.description == expected (event,) = _exception_events(span) assert event.attributes["exception.type"].endswith(type(error).__name__) values = attrs(span) diff --git a/python/frameworks/tavily/tests/test_isolation.py b/python/frameworks/tavily/tests/test_isolation.py index 24ad4e713..1d38d872f 100644 --- a/python/frameworks/tavily/tests/test_isolation.py +++ b/python/frameworks/tavily/tests/test_isolation.py @@ -13,6 +13,7 @@ from typing import Any import pytest +from fi_instrumentation import TraceConfig pytest.importorskip("tavily", reason="tavily-python must be installed to test its instrumentor") @@ -23,6 +24,9 @@ from _support import attrs, instrumented # noqa: E402 from _tavily_fake import FAIL_401, ODD_RESULTS, TAVILY_KEY, FakeTavily # noqa: E402 +# Published core 1.1 retains the legacy descriptions; current core owns error privacy. +NATIVE_ERROR_POLICY = hasattr(TraceConfig(), "hide_error_details") + @pytest.fixture() def fake(): @@ -103,13 +107,15 @@ def __str__(self) -> str: raise _Boom("str") with instrumented() as traced: - with pytest.raises(TypeError, match="not JSON serializable"): + with pytest.raises(TypeError, match="not JSON serializable") as raised: _client(fake).search(Unprintable()) span = traced.one() assert "input.value" not in attrs(span) assert span.status.status_code is StatusCode.ERROR - assert span.status.description.startswith("TypeError") + expected = str(raised.value) if NATIVE_ERROR_POLICY else "TypeError: " + str(raised.value) + assert span.status.description == expected + assert span.events[0].attributes["exception.type"].endswith("TypeError") @pytest.mark.parametrize("mode", ["sync", "async"]) diff --git a/python/frameworks/tavily/tests/test_privacy.py b/python/frameworks/tavily/tests/test_privacy.py index 4e96ac8ba..bbaefc511 100644 --- a/python/frameworks/tavily/tests/test_privacy.py +++ b/python/frameworks/tavily/tests/test_privacy.py @@ -19,12 +19,16 @@ from typing import Any, Dict import pytest +from fi_instrumentation import TraceConfig pytest.importorskip("tavily", reason="tavily-python must be installed to test its instrumentor") from _support import attrs, instrumented # noqa: E402 from _tavily_fake import ECHO_400, TAVILY_KEY, FakeTavily # noqa: E402 +# Published core 1.1 retains the legacy descriptions; current core owns error privacy. +NATIVE_ERROR_POLICY = hasattr(TraceConfig(), "hide_error_details") + OTHER_KEY = "tvly-session-key-must-not-be-exported" @@ -190,10 +194,16 @@ async def call_async() -> None: assert str(info.value) == "Bad query: " + query # unchanged for the caller span = traced.one() assert attrs(span)["input.value"] == "__REDACTED__" - assert span.status.description == "BadRequestError: Bad query: __REDACTED__" (event,) = [event for event in span.events if event.name == "exception"] - assert event.attributes["exception.message"] == "Bad query: __REDACTED__" - assert "Bad query: __REDACTED__" in event.attributes["exception.stacktrace"] + assert event.attributes["exception.type"].endswith("BadRequestError") + if NATIVE_ERROR_POLICY: + assert span.status.description is None + assert "exception.message" not in event.attributes + assert "exception.stacktrace" not in event.attributes + else: + assert span.status.description == "BadRequestError: Bad query: __REDACTED__" + assert event.attributes["exception.message"] == "Bad query: __REDACTED__" + assert "Bad query: __REDACTED__" in event.attributes["exception.stacktrace"] wire = _wire(traced) assert PRIVATE not in wire assert "tvly-" not in wire @@ -221,14 +231,15 @@ async def call_async() -> None: span = traced.one() assert attrs(span)["input.value"] == query - assert span.status.description == "BadRequestError: Bad query: " + query + prefix = "Bad query: " if NATIVE_ERROR_POLICY else "BadRequestError: Bad query: " + assert span.status.description == prefix + query (event,) = [event for event in span.events if event.name == "exception"] assert event.attributes["exception.message"] == "Bad query: " + query assert "Bad query: " + query in event.attributes["exception.stacktrace"] -def test_hidden_inputs_leave_error_text_alone_for_an_empty_query(): - """An empty query has nothing to remove; the error text stays readable.""" +def test_hidden_inputs_apply_error_policy_even_for_an_empty_query(): + """Native privacy drops error text even when the input itself was empty.""" import requests from fi_instrumentation import TraceConfig from tavily import TavilyClient @@ -240,11 +251,18 @@ def test_hidden_inputs_leave_error_text_alone_for_an_empty_query(): client.search("") span = traced.one() - assert span.status.description.startswith("ConnectionError: ") - assert "__REDACTED__" not in span.status.description (event,) = [event for event in span.events if event.name == "exception"] - assert "__REDACTED__" not in event.attributes["exception.message"] - assert event.attributes["exception.message"] == str(info.value)[:1024] + assert event.attributes["exception.type"].endswith("ConnectionError") + if NATIVE_ERROR_POLICY: + assert span.status.description is None + assert "exception.message" not in event.attributes + assert "exception.stacktrace" not in event.attributes + assert str(info.value) not in traced.wire() + else: + assert span.status.description.startswith("ConnectionError: ") + assert "__REDACTED__" not in span.status.description + assert "__REDACTED__" not in event.attributes["exception.message"] + assert event.attributes["exception.message"] == str(info.value)[:1024] def test_query_is_capped_at_1024_utf8_bytes_on_a_character_boundary(fake): @@ -324,8 +342,14 @@ async def call_async() -> None: assert info.value.__traceback__ is not None span = traced.one() (event,) = [event for event in span.events if event.name == "exception"] - stacktrace = event.attributes["exception.stacktrace"] - assert stacktrace.startswith("BadRequestError") + assert event.attributes["exception.type"].endswith("BadRequestError") + if hide and NATIVE_ERROR_POLICY: + assert span.status.description is None + assert "exception.message" not in event.attributes + assert "exception.stacktrace" not in event.attributes + else: + stacktrace = event.attributes["exception.stacktrace"] + assert stacktrace.startswith("BadRequestError") wire = traced.wire() for marker in FRAME_MARKERS + (os.path.expanduser("~"), getpass.getuser(), __file__): assert marker not in wire, marker diff --git a/python/frameworks/typesafe/tests/test_errors.py b/python/frameworks/typesafe/tests/test_errors.py index c0e5de743..5d2d302f3 100644 --- a/python/frameworks/typesafe/tests/test_errors.py +++ b/python/frameworks/typesafe/tests/test_errors.py @@ -23,6 +23,9 @@ questions, ) +# Published core 1.1 retains the legacy descriptions; current core owns error privacy. +NATIVE_ERROR_POLICY = hasattr(TraceConfig(), "hide_error_details") + def _fail(model, config=None, **kwargs): from typesafe_sdk import TypeSafeError @@ -66,7 +69,11 @@ def test_422_message_shown_by_default_hidden_with_the_flags(): assert STATE_MARKER in span.status.description # the API echoed the state _, _, hidden = _fail("fail-422", TraceConfig(hide_inputs=True, hide_outputs=True)) assert STATE_MARKER not in everything(hidden) - assert hidden.status.description == "TypeSafeUnprocessableEntityError (HTTP 422)" + expected = None if NATIVE_ERROR_POLICY else "TypeSafeUnprocessableEntityError (HTTP 422)" + assert hidden.status.description == expected + assert hidden.status.status_code is StatusCode.ERROR + assert hidden.events[0].attributes["exception.type"].endswith("TypeSafeUnprocessableEntityError") + assert "exception.stacktrace" not in hidden.events[0].attributes assert "exception.message" not in hidden.events[0].attributes diff --git a/python/frameworks/voyage/tests/test_async.py b/python/frameworks/voyage/tests/test_async.py index 3964a9b24..2e1543698 100644 --- a/python/frameworks/voyage/tests/test_async.py +++ b/python/frameworks/voyage/tests/test_async.py @@ -6,6 +6,7 @@ from typing import Any, Awaitable, Callable import pytest +from fi_instrumentation import TraceConfig pytest.importorskip("voyageai", reason="voyageai must be installed to test its instrumentor") @@ -30,6 +31,9 @@ status_code, ) +# Published core 1.1 retains the legacy descriptions; current core owns error privacy. +NATIVE_ERROR_POLICY = hasattr(TraceConfig(), "hide_error_details") + @pytest.fixture() def fake(): @@ -101,6 +105,8 @@ def test_async_errors_match_the_sync_error_span(fake): sync_span, async_span = traced.spans() for span in (sync_span, async_span): assert status_code(span) == "ERROR" - assert span.status.description.startswith("AuthenticationError") + message = span.events[0].attributes["exception.message"] + assert span.status.description == (message if NATIVE_ERROR_POLICY else "AuthenticationError: " + message) + assert span.events[0].attributes["exception.type"].endswith("AuthenticationError") assert event_names(span) == ["exception"] assert attrs(async_span) == attrs(sync_span) diff --git a/python/frameworks/voyage/tests/test_errors.py b/python/frameworks/voyage/tests/test_errors.py index 217a7aad8..e0f578f1d 100644 --- a/python/frameworks/voyage/tests/test_errors.py +++ b/python/frameworks/voyage/tests/test_errors.py @@ -27,6 +27,9 @@ status_code, ) +# Published core 1.1 retains the legacy descriptions; current core owns error privacy. +NATIVE_ERROR_POLICY = hasattr(TraceConfig(), "hide_error_details") + _RESULT_KEYS = ( "gen_ai.usage.input_tokens", "gen_ai.usage.total_tokens", @@ -51,7 +54,8 @@ def test_an_api_error_sets_error_status_and_one_exception_event(fake): for span in traced.spans(): assert status_code(span) == "ERROR" - assert span.status.description.startswith("AuthenticationError: ") + message = span.events[0].attributes["exception.message"] + assert span.status.description == (message if NATIVE_ERROR_POLICY else "AuthenticationError: " + message) assert event_names(span) == ["exception"] (event,) = span.events assert event.attributes["exception.type"].endswith("AuthenticationError") @@ -147,12 +151,14 @@ def test_the_key_is_redacted_before_pii_redaction(fake): def test_a_client_timeout_ends_the_span_as_an_error(fake): with instrumented() as traced: - with pytest.raises(voyageai.error.Timeout): + with pytest.raises(voyageai.error.Timeout) as raised: client(fake, timeout=0.3).embed(TEXTS, model="slow") span = traced.one() assert status_code(span) == "ERROR" - assert span.status.description.startswith("Timeout") + expected = str(raised.value) if NATIVE_ERROR_POLICY else "Timeout: " + str(raised.value) + assert span.status.description == expected + assert span.events[0].attributes["exception.type"].endswith("Timeout") assert "voyage.cancelled" not in attrs(span) From fe81b3de54819893d2706967682bae7fab31b8b7 Mon Sep 17 00:00:00 2001 From: nik13 Date: Tue, 27 Oct 2026 03:56:48 -0700 Subject: [PATCH 2/2] test(python): make privacy and exporter oracles fail-sensitive Capture AG2 and Classic assertions at export time, verify exact Classic deliveries against started span IDs, and require ERROR alongside hidden error privacy. Co-Authored-By: Claude Code --- .../ag2-classic/tests/test_setup.py | 68 ++++++++++++++++--- python/frameworks/ag2/tests/test_processor.py | 29 ++++++-- .../exa/tests/test_privacy_config.py | 3 + .../mcp-use/tests/test_mcp_use_agent.py | 7 ++ .../mcp-use/tests/test_mcp_use_callback.py | 12 +++- .../frameworks/tavily/tests/test_privacy.py | 6 ++ 6 files changed, 106 insertions(+), 19 deletions(-) diff --git a/python/frameworks/ag2-classic/tests/test_setup.py b/python/frameworks/ag2-classic/tests/test_setup.py index 8bf1fde2d..a181593c8 100644 --- a/python/frameworks/ag2-classic/tests/test_setup.py +++ b/python/frameworks/ag2-classic/tests/test_setup.py @@ -13,6 +13,7 @@ import signal import subprocess import sys +from collections import Counter from pathlib import Path import pytest @@ -174,6 +175,18 @@ def test_register_then_setup_keeps_one_provider_resource_and_headers(monkeypatch signal.signal(sig, handler) +class _SnapshotExporter(InMemorySpanExporter): + """Freeze each delivery before any later processor can change the span.""" + + def __init__(self): + super().__init__() + self.snapshots = [] + + def export(self, spans): + self.snapshots.extend(span.to_json() for span in spans) + return super().export(spans) + + def test_add_span_processor_after_register_and_setup_keeps_our_processor_first(monkeypatch, fake): """Every active exporter receives filtered spans after a later processor add. @@ -188,7 +201,7 @@ def test_add_span_processor_after_register_and_setup_keeps_our_processor_first(m monkeypatch.setenv("FI_SECRET_KEY", "placeholder-secret-key") monkeypatch.setenv("FI_BASE_URL", "http://127.0.0.1:9") saved = {sig: signal.getsignal(sig) for sig in (signal.SIGINT, signal.SIGTERM)} - original = InMemorySpanExporter() + original = _SnapshotExporter() provider = register( project_type=ProjectType.OBSERVE, project_name="ag2-classic-add", @@ -198,8 +211,18 @@ def test_add_span_processor_after_register_and_setup_keeps_our_processor_first(m set_global_tracer_provider=False, ) tracing = setup(tracer_provider=provider, capture_content=False) - memory = InMemorySpanExporter() - second = InMemorySpanExporter() + memory = _SnapshotExporter() + second = _SnapshotExporter() + # Record generated IDs independently of delivery. An exporter-derived set + # would miss both duplicated registrations and spans lost by every exporter. + started = [] + on_start = tracing.processor.on_start + + def record_start(span, parent_context=None): + started.append((format(span.context.span_id, "#018x"), span.name)) + return on_start(span, parent_context=parent_context) + + monkeypatch.setattr(tracing.processor, "on_start", record_start) try: provider.add_span_processor(SimpleSpanProcessor(memory)) provider.add_span_processor(SimpleSpanProcessor(second)) @@ -209,19 +232,44 @@ def test_add_span_processor_after_register_and_setup_keeps_our_processor_first(m two_agent_tool_chat(tracing, fake) assert provider.force_flush(timeout_millis=10_000) - expected_ids = {s.context.span_id for s in memory.get_finished_spans()} + expected_ids = [span_id for span_id, _name in started] + assert len(expected_ids) == len(set(expected_ids)) == 7 + expected_kinds = { + "chat " + FAKE_MODEL: "LLM", + "invoke_agent assistant": "AGENT", + "invoke_agent user": "AGENT", + "execute_tool get_weather": "TOOL", + "conversation user": "CHAIN", + } + assert Counter(name for _span_id, name in started) == { + "chat " + FAKE_MODEL: 2, + "invoke_agent assistant": 2, + "invoke_agent user": 1, + "execute_tool get_weather": 1, + "conversation user": 1, + } exporters = [memory, second] if hasattr(TraceConfig(), "hide_error_details"): exporters.append(original) # Current register() keeps its exporter. else: assert original.get_finished_spans() == () # Published 1.1 contract. + assert original.snapshots == [] for exporter in exporters: - spans = exporter.get_finished_spans() - assert {s.context.span_id for s in spans} == expected_ids - assert any(s.name == "conversation user" for s in spans) - _assert_no_content_and_no_promoted_usage_off_llm(spans) - conversation = next(s for s in spans if s.name == "conversation user").attributes - assert conversation["gen_ai.span.kind"] == "CHAIN" + spans = [json.loads(snapshot) for snapshot in exporter.snapshots] + delivered_ids = [span["context"]["span_id"] for span in spans] + assert len(delivered_ids) == len(expected_ids) + assert len(set(delivered_ids)) == len(delivered_ids) + assert Counter(delivered_ids) == Counter(expected_ids) + for span, snapshot in zip(spans, exporter.snapshots): + attributes = span["attributes"] + assert attributes.get("gen_ai.span.kind") == expected_kinds[span["name"]] + assert not set(CONTENT_KEYS).intersection(attributes), span["name"] + assert SECRET_PROMPT not in snapshot and TOOL_SECRET_CITY not in snapshot + if attributes["gen_ai.span.kind"] != "LLM": + assert not any( + key in PROMOTED_USAGE_KEYS or key.startswith("gen_ai.usage.") + for key in attributes + ), span["name"] finally: tracing.uninstrument() provider.shutdown() diff --git a/python/frameworks/ag2/tests/test_processor.py b/python/frameworks/ag2/tests/test_processor.py index e6e98450e..3007a6a36 100644 --- a/python/frameworks/ag2/tests/test_processor.py +++ b/python/frameworks/ag2/tests/test_processor.py @@ -2,6 +2,7 @@ from __future__ import annotations +import json import logging import pytest @@ -777,16 +778,28 @@ def _fi_provider(exporter): ) +class _SnapshotExporter(InMemorySpanExporter): + """Freeze each delivery before any later processor can change the span.""" + + def __init__(self): + super().__init__() + self.snapshots = [] + + def export(self, spans): + self.snapshots.extend(span.to_json() for span in spans) + return super().export(spans) + + def test_later_add_span_processor_on_fi_provider_keeps_ag2_processor_first(): """Current register() keeps its exporter on later adds (#281). Both it and the added exporter must receive exactly one normalized, private span. Published core 1.1 still replaces the original exporter on the first add. """ - original = InMemorySpanExporter() + original = _SnapshotExporter() provider = _fi_provider(original) try: assert install_span_processor(provider, config=TraceConfig(hide_inputs=True)) is True - added = InMemorySpanExporter() + added = _SnapshotExporter() provider.add_span_processor(SimpleSpanProcessor(added)) chain = provider._active_span_processor._span_processors @@ -803,14 +816,16 @@ def test_later_add_span_processor_on_fi_provider_keeps_ag2_processor_first(): exporters.append(original) else: assert original.get_finished_spans() == () + assert original.snapshots == [] ids = [] for exporter in exporters: - (span,) = exporter.get_finished_spans() - ids.append(span.context.span_id) - got = attrs(span) - assert got["gen_ai.span.kind"] == "LLM" + (snapshot,) = exporter.snapshots + span = json.loads(snapshot) + ids.append(span["context"]["span_id"]) + got = span["attributes"] + assert got.get("gen_ai.span.kind") == "LLM" assert got["gen_ai.usage.reasoning.output_tokens"] == 2 - assert "PRIVATE-LATER-PROCESSOR" not in span.to_json() + assert "PRIVATE-LATER-PROCESSOR" not in snapshot assert len(set(ids)) == 1 finally: provider.shutdown() diff --git a/python/frameworks/exa/tests/test_privacy_config.py b/python/frameworks/exa/tests/test_privacy_config.py index 2d48f1b03..08250beab 100644 --- a/python/frameworks/exa/tests/test_privacy_config.py +++ b/python/frameworks/exa/tests/test_privacy_config.py @@ -7,6 +7,7 @@ import pytest from opentelemetry import trace as trace_api from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.trace import StatusCode pytest.importorskip("exa_py", reason="exa-py must be installed to test its instrumentor") @@ -178,6 +179,7 @@ def test_hidden_inputs_apply_the_core_error_policy(fake, call): assert PRIVATE in str(caught.value) span = traced.one() + assert span.status.status_code is StatusCode.ERROR assert PRIVATE not in traced.wire() event = _exception_event(span) assert event["exception.type"] == "ValueError" @@ -309,6 +311,7 @@ def test_hidden_inputs_remove_requested_urls_from_error_text(fake): assert urls[0] in str(caught.value) span = traced.one() + assert span.status.status_code is StatusCode.ERROR assert "example.com" not in traced.wire() assert EXA_KEY not in traced.wire() if NATIVE_ERROR_POLICY: diff --git a/python/frameworks/mcp-use/tests/test_mcp_use_agent.py b/python/frameworks/mcp-use/tests/test_mcp_use_agent.py index 426fae433..7d5664cb0 100644 --- a/python/frameworks/mcp-use/tests/test_mcp_use_agent.py +++ b/python/frameworks/mcp-use/tests/test_mcp_use_agent.py @@ -203,6 +203,8 @@ def test_tool_error_text_is_recorded_by_default_and_dropped_with_hide_inputs(): run_agent([tool_call("fail", {"reason": ARG}), answer()], [handler]) spans = exporter.get_finished_spans() tool = only(spans, "execute_tool fail") + assert tool.status.status_code is StatusCode.ERROR + assert only(spans, AGENT).status.status_code is StatusCode.OK assert tool.status.description == "RuntimeError: Error executing tool fail: tool failed because " + ARG (event,) = tool.events assert "exception.stacktrace" not in event.attributes @@ -218,8 +220,11 @@ def test_tool_error_text_is_recorded_by_default_and_dropped_with_hide_inputs(): run_agent([tool_call("fail", {"reason": ARG}), answer()], [handler]) spans = exporter.get_finished_spans() tool = only(spans, "execute_tool fail") + assert tool.status.status_code is StatusCode.ERROR + assert only(spans, AGENT).status.status_code is StatusCode.OK assert tool.status.description == (None if NATIVE_ERROR_POLICY else "RuntimeError") (event,) = tool.events + assert event.name == "exception" assert dict(event.attributes) == {"exception.type": "RuntimeError"} assert ARG not in wire(spans) and "PROMPT-MARKER" not in wire(spans) @@ -302,8 +307,10 @@ def test_hide_outputs_keeps_earlier_turns_out_of_later_turns(method, external): # outputs) is dropped, the exception type kept. assert attrs(only(third, AGENT))["input.value"] == "THIRD-PROMPT" for span in (only(third, AGENT), only(third, LLM_SPAN)): + assert span.status.status_code is StatusCode.ERROR assert span.status.description == (None if NATIVE_ERROR_POLICY else "ValueError") (event,) = span.events + assert event.name == "exception" assert dict(event.attributes) == {"exception.type": "ValueError"} diff --git a/python/frameworks/mcp-use/tests/test_mcp_use_callback.py b/python/frameworks/mcp-use/tests/test_mcp_use_callback.py index c9777010a..7156f9ecd 100644 --- a/python/frameworks/mcp-use/tests/test_mcp_use_callback.py +++ b/python/frameworks/mcp-use/tests/test_mcp_use_callback.py @@ -360,9 +360,14 @@ def test_hide_outputs_drops_earlier_turns_replayed_as_input(): assert llm_span[call + "function.name"] == "add" assert call + "function.arguments" not in llm_span assert llm_span["input.value"] == PROMPT + assert sorted(span.name for span in spans) == sorted([AGENT, "chat"]) for span in spans: - # Error text is content: dropped with either hide flag. + # Error text is content: dropped with either hide flag, not its status. + assert span.status.status_code is StatusCode.ERROR assert span.status.description == (None if NATIVE_ERROR_POLICY else "ValueError") + (event,) = span.events + assert event.name == "exception" + assert dict(event.attributes) == {"exception.type": "ValueError"} def test_hide_outputs_with_a_replayed_reply_last_redacts_the_llm_input_value(): @@ -982,8 +987,11 @@ def test_hide_flags_fail_closed_when_a_run_remembers_too_much(monkeypatch): for marker in CONTENT_MARKERS + (ANSWER,): assert marker not in text, marker tool = only(spans, "execute_tool fail") + assert tool.status.status_code is StatusCode.ERROR assert tool.status.description == (None if NATIVE_ERROR_POLICY else "ValueError") - assert dict(tool.events[0].attributes) == {"exception.type": "ValueError"} + (event,) = tool.events + assert event.name == "exception" + assert dict(event.attributes) == {"exception.type": "ValueError"} def test_an_internal_failure_in_any_callback_never_reaches_langchain(monkeypatch): diff --git a/python/frameworks/tavily/tests/test_privacy.py b/python/frameworks/tavily/tests/test_privacy.py index bbaefc511..c7da4ad65 100644 --- a/python/frameworks/tavily/tests/test_privacy.py +++ b/python/frameworks/tavily/tests/test_privacy.py @@ -20,6 +20,7 @@ import pytest from fi_instrumentation import TraceConfig +from opentelemetry.trace import StatusCode pytest.importorskip("tavily", reason="tavily-python must be installed to test its instrumentor") @@ -193,6 +194,7 @@ async def call_async() -> None: assert str(info.value) == "Bad query: " + query # unchanged for the caller span = traced.one() + assert span.status.status_code is StatusCode.ERROR assert attrs(span)["input.value"] == "__REDACTED__" (event,) = [event for event in span.events if event.name == "exception"] assert event.attributes["exception.type"].endswith("BadRequestError") @@ -251,6 +253,7 @@ def test_hidden_inputs_apply_error_policy_even_for_an_empty_query(): client.search("") span = traced.one() + assert span.status.status_code is StatusCode.ERROR (event,) = [event for event in span.events if event.name == "exception"] assert event.attributes["exception.type"].endswith("ConnectionError") if NATIVE_ERROR_POLICY: @@ -341,8 +344,11 @@ async def call_async() -> None: # The caller still gets the real traceback. assert info.value.__traceback__ is not None span = traced.one() + assert span.status.status_code is StatusCode.ERROR (event,) = [event for event in span.events if event.name == "exception"] assert event.attributes["exception.type"].endswith("BadRequestError") + if hide: + assert PRIVATE not in traced.wire() if hide and NATIVE_ERROR_POLICY: assert span.status.description is None assert "exception.message" not in event.attributes