From 46ea8f167b15aa410026a6d109b0e06ff9be4002 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek <8347993+nik13@users.noreply.github.com> Date: Sat, 31 Oct 2026 06:45:37 -0700 Subject: [PATCH] fix(exa): capture bounded response previews and reported costs Keep Exa lifecycle instrumentation and public entry points intact. Make response previews opt-in, honor output privacy, preserve stream completion semantics, and map only vendor-reported USD charges. Co-Authored-By: Claude Code --- python/frameworks/exa/README.md | 82 +++- .../exa/tests/test_response_capture.py | 407 ++++++++++++++++++ python/frameworks/exa/traceai_exa/__init__.py | 29 +- .../frameworks/exa/traceai_exa/_response.py | 166 +++++++ .../frameworks/exa/traceai_exa/_wrappers.py | 65 ++- 5 files changed, 716 insertions(+), 33 deletions(-) create mode 100644 python/frameworks/exa/tests/test_response_capture.py create mode 100644 python/frameworks/exa/traceai_exa/_response.py diff --git a/python/frameworks/exa/README.md b/python/frameworks/exa/README.md index c8ee1896b..7916ff9c9 100644 --- a/python/frameworks/exa/README.md +++ b/python/frameworks/exa/README.md @@ -13,14 +13,18 @@ pip install traceAI-exa It accepts `exa-py>=2.25.0,<3` and is tested with `exa-py` 2.25.0 on Python 3.10, 3.11, 3.12 and 3.13. The test suite also passes on `exa-py` 2.26.0. -Live vendor verification (2026-10-17): a real run against `api.exa.ai` with -`exa-py` 2.25.0 (`search`, `get_contents`, `answer`, both streams, `AsyncExa.search` -and an invalid-key 401 recorded as ERROR) exported the expected spans to Future -AGI, and `FI_HIDE_INPUTS=true` kept the query out of every stored column. +## Configure and trace -## Usage +Set your Future AGI and Exa credentials in your application environment: + +```bash +export FI_API_KEY="your-futureagi-api-key" +export FI_SECRET_KEY="your-futureagi-secret-key" +export EXA_API_KEY="your-exa-api-key" +``` ```python +import os from exa_py import Exa from fi_instrumentation import register from fi_instrumentation.fi_types import ProjectType @@ -29,10 +33,16 @@ from traceai_exa import ExaInstrumentor tracer_provider = register(project_name="exa-search", project_type=ProjectType.OBSERVE) ExaInstrumentor().instrument(tracer_provider=tracer_provider) -client = Exa(api_key="your-exa-api-key") +client = Exa(api_key=os.environ["EXA_API_KEY"]) client.search("recent retrieval research", num_results=5) +tracer_provider.force_flush() ``` +Open the `exa-search` project in Future AGI's Observe view and select the +`exa.search` span. By default, it shows the query, result count, duration, +status, and cost when Exa reports one. Enable response previews below to +inspect retrieved content. + ## Instrumented calls `Exa` and `AsyncExa` calls to `search`, `get_contents`, `answer`, and the @@ -60,11 +70,57 @@ under it. | `exa.result_count` | successful calls | `results` for `search` and `get_contents`, `citations` for `answer`, and the citations summed over every chunk of a fully read stream. Absent when the count is unknown (an error or a cancellation), never 0 for unknown. | | `exa.cancelled` | cancelled calls and streams | `true`; see [Errors and cancellation](#errors-and-cancellation). | -The spans never carry response content: no result titles, text, highlights or -summaries, no answer or stream text, no cost, token count or model name. There -is no setting that adds them. The Exa API key is never recorded; it is -replaced by `[redacted]` wherever it appears verbatim in a recorded query, URL -or error text. +Response content is off by default. The instrumentor never invents a model +name or token usage. The client's Exa key is replaced by `[redacted]` wherever +it appears verbatim in recorded text. + +### Response previews + +Opt in to content capture when you first instrument the client: + +```python +ExaInstrumentor().instrument( + tracer_provider=tracer_provider, + capture_outputs=True, + capture_urls=True, # Optional: also include returned document URL fields. +) +``` + +`output.value` is a JSON preview (`output.mime_type=application/json`): + +- `search`, `search_and_contents`, and `get_contents`: `results` containing + available titles, text, summaries, and highlights. +- `answer`: answer text and `citations` containing available titles and text. + A structured answer is serialized into a bounded text preview. +- Fully consumed streams: accumulated text in `answer` and returned `citations`. + A failed or cancelled stream does not export partial output. + +Previews include at most 20 documents or citations, 10 highlights per document, +1 KiB of UTF-8 per document field or highlight, and 8 KiB of answer text. The +entire JSON preview is capped at 16 KiB; trailing documents or answer text may +be omitted to fit. `exa.output_truncated=true` marks a shortened preview. +`exa.result_count` still describes the full response, and your returned objects +and stream chunks are unchanged. Structured answers also have depth and item +limits; previews are not a replacement for storing application results. + +Returned URL fields require both `capture_outputs=True` and `capture_urls=True`. +Document IDs are not copied because Exa IDs can themselves contain URLs. +`capture_urls` controls URL fields, not links embedded in captured text. + +`FI_HIDE_OUTPUTS=true` or `TraceConfig(hide_outputs=True)` overrides this opt-in +and prevents response previews. `hide_output_text=True` also disables previews. +`hide_inputs=True` suppresses URL fields and removes verbatim request echoes +from captured output. `pii_redaction=True` applies before text is serialized. + +### Reported cost + +When Exa returns `cost_dollars.total`, `gen_ai.cost.total` records that amount +in USD, independently of content capture. Available breakdowns use +`exa.cost.search.neural`, `exa.cost.search.keyword`, `exa.cost.contents.text`, +and `exa.cost.contents.summary`. These are vendor request charges, not inferred +LLM input/output token costs. Only finite, non-negative numbers are recorded; +a missing cost stays absent, while a reported zero stays zero. Streaming chunks +do not expose cost metadata, so stream cost is absent. ### get_contents URLs @@ -137,7 +193,9 @@ example `aclose = getattr(stream, "aclose", None)`. A missing API key raises `ValueError` in `Exa()` or `AsyncExa()` itself, before any traced method runs, so no span is recorded for it. -`find_similar` and `find_similar_and_contents` are not traced. +`find_similar` and `find_similar_and_contents` are deprecated upstream and +are not traced. Use `client.search("pages similar to " + url)` instead; pass +`contents={"text": True}` when you need page text. Do not use this package together with another instrumentor that wraps the same Exa client methods, or duplicate spans may result. diff --git a/python/frameworks/exa/tests/test_response_capture.py b/python/frameworks/exa/tests/test_response_capture.py new file mode 100644 index 000000000..3e587984f --- /dev/null +++ b/python/frameworks/exa/tests/test_response_capture.py @@ -0,0 +1,407 @@ +"""Response visibility through the real SDK, with no vendor connections.""" + +from __future__ import annotations + +import asyncio +import json +from types import SimpleNamespace + +import pytest +from _support import EXA_KEY, FakeExa, attrs, instrumented +from exa_py import AsyncExa, Exa +from fi_instrumentation import TraceConfig +from opentelemetry.exporter.otlp.proto.common.trace_encoder import encode_spans + +TEXT = "retrieved document content" +ANSWER = "answer backed by citations" +URL = "https://example.com/private?token=opaque" + + +@pytest.fixture() +def response(monkeypatch): + document = { + "id": URL, + "url": URL, + "title": "source title", + "text": TEXT, + "summary": "document summary", + "highlights": ["relevant passage"], + } + payload = { + "results": [document], + "answer": ANSWER, + "citations": [document], + "costDollars": { + "total": 0.012, + "search": {"neural": 0.005}, + "contents": {"text": 0.007}, + }, + } + + def request(*args, **kwargs): + return payload + + async def async_request(*args, **kwargs): + return payload + + monkeypatch.setattr(Exa, "request", request) + monkeypatch.setattr(AsyncExa, "async_request", async_request) + return payload + + +def call(method, asynchronous=False): + argument = [URL] if method == "get_contents" else "question" + if asynchronous: + + async def run(): + client = AsyncExa(api_key=EXA_KEY) + try: + return await getattr(client, method)(argument) + finally: + await client.client.aclose() + + return asyncio.run(run()) + return getattr(Exa(api_key=EXA_KEY), method)(argument) + + +@pytest.mark.parametrize( + "method", ["search", "search_and_contents", "get_contents", "answer"] +) +@pytest.mark.parametrize("asynchronous", [False, True]) +def test_response_content_and_vendor_cost(response, method, asynchronous): + with instrumented(capture_outputs=True, capture_urls=True) as traced: + result = call(method, asynchronous) + values = attrs(traced.one()) + output = json.loads(values["output.value"]) + documents = output["citations" if method == "answer" else "results"] + assert documents[0]["text"] == TEXT + assert documents[0]["url"] == URL + assert documents[0]["title"] == "source title" + if method == "answer": + assert output["answer"] == ANSWER + assert result.answer == ANSWER + else: + assert documents[0]["highlights"] == ["relevant passage"] + assert documents[0]["summary"] == "document summary" + assert values["output.mime_type"] == "application/json" + assert values["gen_ai.cost.total"] == 0.012 + assert values["exa.cost.search.neural"] == 0.005 + assert values["exa.cost.contents.text"] == 0.007 + assert "gen_ai.cost.input" not in values + assert not any("token" in key or "model" in key for key in values) + + +@pytest.mark.parametrize( + "options", [{}, {"capture_urls": True}, {"config": TraceConfig(hide_outputs=False)}] +) +def test_content_capture_requires_explicit_opt_in(response, options): + with instrumented(**options) as traced: + call("answer") + wire = encode_spans(traced.spans()).SerializeToString() + assert ANSWER.encode() not in wire + assert TEXT.encode() not in wire + assert "output.value" not in attrs(traced.one()) + assert attrs(traced.one())["gen_ai.cost.total"] == 0.012 + + +@pytest.mark.parametrize("flag", ["hide_outputs", "hide_output_text"]) +def test_hidden_output_overrides_opt_in_on_otlp_wire(response, flag): + with instrumented( + capture_outputs=True, capture_urls=True, config=TraceConfig(**{flag: True}) + ) as traced: + call("answer") + wire = encode_spans(traced.spans()).SerializeToString() + assert ANSWER.encode() not in wire + assert TEXT.encode() not in wire + assert URL.encode() not in wire + assert attrs(traced.one())["exa.result_count"] == 1 + assert attrs(traced.one())["gen_ai.cost.total"] == 0.012 + + +def test_environment_output_privacy(response, monkeypatch): + monkeypatch.setenv("FI_HIDE_OUTPUTS", "true") + with instrumented(capture_outputs=True) as traced: + call("answer") + assert ANSWER not in traced.wire() + + +def test_returned_urls_require_separate_opt_in(response): + with instrumented(capture_outputs=True) as traced: + call("answer") + output = json.loads(attrs(traced.one())["output.value"]) + assert "url" not in output["citations"][0] + assert "id" not in output["citations"][0] # Exa IDs can themselves be URLs. + assert URL not in traced.wire() + + +def test_content_is_scrubbed_before_serialization(response): + response["answer"] = f"{EXA_KEY} question jane.doe@example.com" + response["citations"][0]["text"] = response["answer"] + with instrumented( + capture_outputs=True, + capture_urls=True, + config=TraceConfig(hide_inputs=True, pii_redaction=True), + ) as traced: + call("answer") + wire = encode_spans(traced.spans()).SerializeToString() + for secret in [EXA_KEY, "question", "jane.doe@example.com"]: + assert secret.encode() not in wire + output = json.loads(attrs(traced.one())["output.value"]) + assert "" in output["answer"] + assert "url" not in output["citations"][0] + + +def test_capture_is_bounded_and_does_not_mutate_results(response): + response["results"] *= 100 + response["results"][0]["text"] = "界" * 10000 + with instrumented(capture_outputs=True, capture_urls=True) as traced: + result = call("search") + values = attrs(traced.one()) + assert len(values["output.value"].encode()) <= 16384 + documents = json.loads(values["output.value"])["results"] + assert 0 < len(documents) <= 20 + assert len(documents[0]["text"].encode()) <= 1024 + assert values["exa.output_truncated"] is True + assert values["exa.result_count"] == 100 + assert len(result.results) == 100 + assert result.results[0].text == "界" * 10000 + + +@pytest.mark.parametrize("cost", [None, True, -1, float("nan"), float("inf"), "0.1"]) +def test_no_fabricated_or_invalid_cost(response, cost): + response["costDollars"] = {"total": cost} + with instrumented() as traced: + call("search") + assert "gen_ai.cost.total" not in attrs(traced.one()) + + +def test_zero_cost_is_preserved(response): + response["costDollars"] = {"total": 0} + with instrumented() as traced: + call("search") + assert attrs(traced.one())["gen_ai.cost.total"] == 0 + + +@pytest.mark.parametrize("method", ["stream_answer", "stream_search"]) +@pytest.mark.parametrize("asynchronous", [False, True]) +def test_stream_capture_preserves_lifecycle_and_citations(method, asynchronous): + with ( + FakeExa() as fake, + instrumented(capture_outputs=True, capture_urls=True) as traced, + ): + if asynchronous: + + async def run(): + client = AsyncExa(api_key=EXA_KEY, api_base=fake.origin) + try: + stream = await getattr(client, method)("question") + chunks = [chunk async for chunk in stream] + await stream.aclose() + return chunks + finally: + await client.client.aclose() + + chunks = asyncio.run(run()) + else: + stream = getattr(Exa(api_key=EXA_KEY, base_url=fake.origin), method)( + "question" + ) + chunks = list(stream) + stream.close() + values = attrs(traced.one()) + output = json.loads(values["output.value"]) + assert output["answer"] == "".join(chunk.content or "" for chunk in chunks) + assert len(output["citations"]) == values["exa.result_count"] == 3 + assert "gen_ai.cost.total" not in values # StreamChunk exposes no cost. + + +@pytest.mark.parametrize("flag", ["hide_outputs", "hide_output_text"]) +def test_stream_output_privacy(flag): + with ( + FakeExa() as fake, + instrumented( + capture_outputs=True, config=TraceConfig(**{flag: True}) + ) as traced, + ): + chunks = list( + Exa(api_key=EXA_KEY, base_url=fake.origin).stream_answer("question") + ) + wire = encode_spans(traced.spans()).SerializeToString() + for chunk in chunks: + if chunk.content: + assert chunk.content.encode() not in wire + + +def test_cancelled_stream_does_not_export_partial_output(): + with FakeExa() as fake, instrumented(capture_outputs=True) as traced: + stream = Exa(api_key=EXA_KEY, base_url=fake.origin).stream_answer("question") + next(stream) + stream.close() + assert "output.value" not in attrs(traced.one()) + assert attrs(traced.one())["exa.cancelled"] is True + + +def test_unexpected_response_property_cannot_break_call(monkeypatch, caplog): + class Response: + results = () + + @property + def answer(self): + raise RuntimeError(EXA_KEY) + + response = Response() + monkeypatch.setattr(Exa, "answer", lambda *args, **kwargs: response) + with ( + caplog.at_level("DEBUG", logger="traceai_exa"), + instrumented(capture_outputs=True) as traced, + ): + assert call("answer") is response + assert traced.one().status.status_code.name == "OK" + assert "Exa response inspection failed" in caplog.text + assert EXA_KEY not in caplog.text + + +def test_stream_redacts_keys_split_across_chunks(monkeypatch): + chunks = [ + SimpleNamespace(content=part, citations=None) + for part in [EXA_KEY[:9], EXA_KEY[9:], " jane.doe@", "example.com"] + ] + monkeypatch.setattr(Exa, "stream_answer", lambda *args, **kwargs: iter(chunks)) + with instrumented( + capture_outputs=True, config=TraceConfig(pii_redaction=True) + ) as traced: + assert list(call("stream_answer")) == chunks + output = json.loads(attrs(traced.one())["output.value"]) + assert EXA_KEY not in output["answer"] + assert "jane.doe@example.com" not in output["answer"] + assert "" in output["answer"] + + +def test_structured_answer_is_a_bounded_text_preview(response): + response["answer"] = {"facts": [f"{EXA_KEY} fact"] * 1000} + with instrumented(capture_outputs=True) as traced: + returned = call("answer") + values = attrs(traced.one()) + answer = json.loads(values["output.value"])["answer"] + assert "fact" in answer + assert EXA_KEY not in answer + assert len(answer.encode()) <= 8192 + assert len(values["output.value"].encode()) <= 16384 + assert values["exa.output_truncated"] is True + assert len(returned.answer["facts"]) == 1000 + + +def test_stream_budget_and_document_limit(monkeypatch): + document = SimpleNamespace(title="title", text="text" * 1024, url=URL) + chunk = SimpleNamespace(content="界" * 10000, citations=[document] * 30) + monkeypatch.setattr( + Exa, "stream_answer", lambda *args, **kwargs: iter([chunk] * 50) + ) + with instrumented(capture_outputs=True) as traced: + assert len(list(call("stream_answer"))) == 50 + values = attrs(traced.one()) + output = json.loads(values["output.value"]) + assert len(values["output.value"].encode()) <= 16384 + assert len(output["answer"].encode()) <= 8192 + assert len(output["citations"]) <= 20 + assert values["exa.result_count"] == 1500 + assert values["exa.output_truncated"] is True + + +@pytest.mark.parametrize( + "options, visible", + [ + ({"capture_outputs": True}, True), + ({"capture_outputs": True, "config": TraceConfig(hide_outputs=True)}, False), + ( + {"capture_outputs": True, "config": TraceConfig(hide_output_text=True)}, + False, + ), + ({}, False), + ], +) +def test_response_reaches_real_otlp_http_receiver(response, options, visible): + import sys + from pathlib import Path + + from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter + from opentelemetry.sdk.trace.export import SimpleSpanProcessor + + sys.path.insert(0, str(Path(__file__).resolve().parents[3] / "tests")) + from harness import Receiver, _flatten_attributes + + response["answer"] = f"{ANSWER} {EXA_KEY}" + with Receiver() as receiver, instrumented(**options) as traced: + traced.provider.add_span_processor( + SimpleSpanProcessor( + OTLPSpanExporter(endpoint=receiver.endpoint, headers={}, timeout=2) + ) + ) + call("answer") + traced.provider.force_flush() + spans = receiver.spans() + assert len(spans) == 1 + values = _flatten_attributes(spans[0]["attributes"]) + assert values["gen_ai.cost.total"] == 0.012 + assert int(values["exa.result_count"]) == 1 + wire = json.dumps(spans) + assert (ANSWER in wire) is visible + assert EXA_KEY not in wire + assert URL not in wire + if visible: + output = json.loads(values["output.value"]) + assert output["citations"][0]["text"] == TEXT + traced.provider.shutdown() + + +def test_json_escaping_respects_the_total_byte_budget(response): + response["answer"] = "\x00" * 8192 + with instrumented(capture_outputs=True) as traced: + call("answer") + values = attrs(traced.one()) + assert len(values["output.value"].encode()) <= 16384 + assert json.loads(values["output.value"])["answer"] + assert values["exa.output_truncated"] is True + + +def test_stream_inspection_failure_does_not_drop_user_chunks(monkeypatch, caplog): + class Chunk: + citations = () + + @property + def content(self): + raise RuntimeError(EXA_KEY) + + chunks = [Chunk(), SimpleNamespace(content="next chunk", citations=[])] + monkeypatch.setattr(Exa, "stream_answer", lambda *args, **kwargs: iter(chunks)) + with ( + caplog.at_level("DEBUG", logger="traceai_exa"), + instrumented(capture_outputs=True) as traced, + ): + assert list(call("stream_answer")) == chunks + assert traced.one().status.status_code.name == "OK" + assert "output.value" not in attrs(traced.one()) + assert "Exa stream inspection failed" in caplog.text + assert EXA_KEY not in caplog.text + + +def test_bad_cost_metadata_does_not_disable_content_capture(monkeypatch, caplog): + class Response: + results = () + + @property + def cost_dollars(self): + raise RuntimeError(EXA_KEY) + + response = Response() + monkeypatch.setattr(Exa, "search", lambda *args, **kwargs: response) + with ( + caplog.at_level("DEBUG", logger="traceai_exa"), + instrumented(capture_outputs=True) as traced, + ): + assert call("search") is response + values = attrs(traced.one()) + assert json.loads(values["output.value"]) == {"results": []} + assert "gen_ai.cost.total" not in values + assert "Exa cost inspection failed" in caplog.text + assert EXA_KEY not in caplog.text diff --git a/python/frameworks/exa/traceai_exa/__init__.py b/python/frameworks/exa/traceai_exa/__init__.py index 02743a9cb..74f1a6201 100644 --- a/python/frameworks/exa/traceai_exa/__init__.py +++ b/python/frameworks/exa/traceai_exa/__init__.py @@ -41,7 +41,8 @@ class ExaInstrumentor(BaseInstrumentor): """Instrument Exa's search, contents, answer, and streaming APIs. - ``instrument()`` accepts ``tracer_provider``, ``capture_urls`` and + ``instrument()`` accepts ``tracer_provider``, ``capture_urls``, + ``capture_outputs`` (bounded response previews, off by default) and ``config`` (a ``fi_instrumentation.TraceConfig``; by default one built from the ``FI_*`` environment variables, such as ``FI_HIDE_INPUTS``). Spans come from an ``FITracer``, so the config's masking and PII @@ -74,30 +75,32 @@ def _instrument(self, **kwargs: Any) -> None: capture_urls = bool(kwargs.get("capture_urls", False)) hide_inputs = bool(config.hide_inputs) pii = bool(config.pii_redaction) + capture_outputs = bool(kwargs.get("capture_outputs", False)) and not ( + config.hide_outputs or config.hide_output_text + ) + options = { + "capture_urls": capture_urls, + "capture_outputs": capture_outputs, + "hide_inputs": hide_inputs, + "pii": pii, + } api_module = import_module(_MODULE) self._original_methods: Dict[Tuple[str, str], Any] = {} for client_name in ("Exa", "AsyncExa"): is_async = client_name == "AsyncExa" for method_name, span_name in _OPERATIONS.items(): - options = { - "contents": method_name == "get_contents", - "capture_urls": capture_urls, - "hide_inputs": hide_inputs, - "pii": pii, - } - wrapper = ( - AsyncOperationWrapper(tracer, span_name, **options) - if is_async - else OperationWrapper(tracer, span_name, **options) + wrapper_type = AsyncOperationWrapper if is_async else OperationWrapper + wrapper = wrapper_type( + tracer, span_name, contents=method_name == "get_contents", **options ) self._wrap_method(api_module, client_name, method_name, wrapper) for method_name, span_name in _STREAM_OPERATIONS.items(): wrapper = ( - AsyncStreamWrapper(tracer, span_name, hide_inputs=hide_inputs, pii=pii) + AsyncStreamWrapper(tracer, span_name, **options) if is_async - else StreamWrapper(tracer, span_name, hide_inputs=hide_inputs, pii=pii) + else StreamWrapper(tracer, span_name, **options) ) self._wrap_method(api_module, client_name, method_name, wrapper) diff --git a/python/frameworks/exa/traceai_exa/_response.py b/python/frameworks/exa/traceai_exa/_response.py new file mode 100644 index 000000000..37a852515 --- /dev/null +++ b/python/frameworks/exa/traceai_exa/_response.py @@ -0,0 +1,166 @@ +"""Bounded, opt-in Exa response previews and vendor-reported USD costs.""" + +from __future__ import annotations + +import json +import logging +import math +from collections.abc import Callable, Mapping +from itertools import islice +from typing import Any + +logger = logging.getLogger(__name__) + +_MAX_DOCUMENTS = 20 +_MAX_HIGHLIGHTS = 10 +_MAX_FIELD_BYTES = 1024 +_MAX_ANSWER_BYTES = 8192 +_MAX_OUTPUT_BYTES = 16384 + + +def _field(value: Any, name: str) -> Any: + return value.get(name) if isinstance(value, Mapping) else getattr(value, name, None) + + +def _json(value: Any) -> str: + return json.dumps(value, ensure_ascii=False, separators=(",", ":"), allow_nan=False) + + +def cost_attributes(response: Any) -> dict[str, float]: + """Copy only finite, non-negative USD amounts; never estimate token costs.""" + attributes = {} + try: + cost = _field(response, "cost_dollars") + candidates = {"gen_ai.cost.total": _field(cost, "total")} + for group, names in ( + ("search", ("neural", "keyword")), + ("contents", ("text", "summary")), + ): + breakdown = _field(cost, group) + for name in names: + candidates[f"exa.cost.{group}.{name}"] = _field(breakdown, name) + for key, value in candidates.items(): + if type(value) in (int, float) and value >= 0 and math.isfinite(value): + attributes[key] = float(value) + except Exception: # noqa: BLE001 - vendor metadata cannot break user calls + # Do not log exception text or stack traces: vendor fields may hold secrets. + logger.debug("Exa cost inspection failed; unavailable costs omitted") + return attributes + + +class ResponseCapture: + """Retain a small allowlisted preview, never the full vendor response. + + Streams retain at most 8192 text characters until completion, so redaction + also catches secrets/PII split between chunks. Documents are bounded as + they arrive. Partial outputs are not exported on error or cancellation. + """ + + def __init__(self, scrub: Callable[[str], str], capture_urls: bool) -> None: + self._scrub = scrub + self._capture_urls = capture_urls + self._data: dict = {} + self._stream_text = "" + self.truncated = False + + def _text(self, value: str, limit: int = _MAX_FIELD_BYTES) -> str: + value = self._scrub(value) + # Slice characters first to avoid encoding arbitrarily large strings. + encoded = value[:limit].encode("utf-8", "replace") + result = encoded[:limit].decode("utf-8", "ignore") + self.truncated |= len(result) < len(value) + return result + + def _structured(self, value: Any, remaining: list[int], depth: int = 0) -> Any: + """Bound structured answers before serializing them into a text preview.""" + if depth > 4 or remaining[0] <= 0: + self.truncated = True + return None + remaining[0] -= 1 + if isinstance(value, str): + return self._text(value) + if value is None or type(value) in (bool, int): + return value + if isinstance(value, float): + return value if math.isfinite(value) else None + if isinstance(value, (dict, list)): + self.truncated |= len(value) > 20 + if isinstance(value, dict): + return { + self._text(key): self._structured(item, remaining, depth + 1) + for key, item in islice(value.items(), 20) + if isinstance(key, str) + } + return [self._structured(item, remaining, depth + 1) for item in value[:20]] + return None # Never stringify arbitrary response objects. + + def _documents(self, key: str, documents: Any) -> None: + if not isinstance(documents, (list, tuple)): + return + captured = self._data.setdefault(key, []) + available = _MAX_DOCUMENTS - len(captured) + self.truncated |= len(documents) > available + for document in documents[:available]: + item: dict[str, str | list[str]] = {} + fields = ("title", "text", "summary") + ( + ("url",) if self._capture_urls else () + ) + for name in fields: + value = _field(document, name) + if isinstance(value, str): + item[name] = self._text(value) + highlights = _field(document, "highlights") + if isinstance(highlights, (list, tuple)): + self.truncated |= len(highlights) > _MAX_HIGHLIGHTS + item["highlights"] = [ + self._text(value) + for value in highlights[:_MAX_HIGHLIGHTS] + if isinstance(value, str) + ] + captured.append(item) + if len(_json(self._data).encode("utf-8")) > _MAX_OUTPUT_BYTES: + captured.pop() + self.truncated = True + break + + def response(self, response: Any) -> None: + answer = _field(response, "answer") + if isinstance(answer, dict): + answer = _json(self._structured(answer, [128])) + if isinstance(answer, str): + self._data["answer"] = self._text(answer, _MAX_ANSWER_BYTES) + for key in ("results", "citations"): + self._documents(key, _field(response, key)) + + def chunk(self, chunk: Any) -> None: + text = _field(chunk, "content") + if isinstance(text, str): + available = _MAX_ANSWER_BYTES - len(self._stream_text) + self.truncated |= len(text) > available + self._stream_text += text[:available] + self._documents("citations", _field(chunk, "citations")) + + def attributes(self, *, stream: bool = False) -> dict[str, str | bool]: + if stream: + self._data["answer"] = self._text(self._stream_text, _MAX_ANSWER_BYTES) + if not self._data: + return {} + # JSON escaping and stream text added after citations can exceed the + # overall budget. Drop trailing documents, then shorten the answer. + output = _json(self._data) + while len(output.encode("utf-8")) > _MAX_OUTPUT_BYTES: + self.truncated = True + documents = self._data.get("results") or self._data.get("citations") + if documents: + documents.pop() + else: + answer = self._data["answer"] + self._data["answer"] = answer[: len(answer) // 2] + output = _json(self._data) + attributes: dict[str, str | bool] = { + "output.value": output, + "output.mime_type": "application/json", + } + if self.truncated: + attributes["exa.output_truncated"] = True + return attributes diff --git a/python/frameworks/exa/traceai_exa/_wrappers.py b/python/frameworks/exa/traceai_exa/_wrappers.py index 6a3e64419..0bc0c3f05 100644 --- a/python/frameworks/exa/traceai_exa/_wrappers.py +++ b/python/frameworks/exa/traceai_exa/_wrappers.py @@ -4,6 +4,7 @@ import asyncio import inspect +import logging import weakref from typing import Any, Callable, Mapping, Optional, Sequence, Tuple @@ -13,6 +14,10 @@ from opentelemetry import trace as trace_api from opentelemetry.trace import Span, Status, StatusCode, Tracer +from traceai_exa._response import ResponseCapture, cost_attributes + +logger = logging.getLogger(__name__) + _SPAN_KIND = "gen_ai.span.kind" _INPUT_VALUE = "input.value" _RETRIEVAL_QUERY = "gen_ai.retrieval.query" @@ -182,6 +187,7 @@ def __init__( *, contents: bool = False, capture_urls: bool = False, + capture_outputs: bool = False, hide_inputs: bool = False, pii: bool = False, ) -> None: @@ -189,6 +195,7 @@ def __init__( self._span_name = span_name self._contents = contents self._capture_urls = capture_urls + self._capture_outputs = capture_outputs self._hide_inputs = hide_inputs self._pii = pii @@ -216,6 +223,22 @@ def _start_span( except Exception: # a tracer failure must never reach the caller return None, None + def _capture(self, scrub: _Scrub | None) -> ResponseCapture | None: + if not self._capture_outputs or scrub is None: + return None + return ResponseCapture(scrub, self._capture_urls and not self._hide_inputs) + + def _record_response(self, span: Span, result: Any, scrub: _Scrub | None) -> None: + try: + span.set_attributes(cost_attributes(result)) + capture = self._capture(scrub) + if capture is not None: + capture.response(result) + span.set_attributes(capture.attributes()) + except Exception: # noqa: BLE001 - telemetry must not break user calls + # Fail open without logging sensitive vendor text or local stack frames. + logger.debug("Exa response inspection failed; output omitted") + @staticmethod def _finish_ok(span: Span, document_count: Optional[int]) -> None: try: @@ -309,6 +332,7 @@ def __call__( except BaseException as error: self._finish_error(span, error, scrub) raise + self._record_response(span, result, scrub) self._finish_ok(span, _document_count(result)) return result @@ -335,6 +359,7 @@ async def __call__( except BaseException as error: self._finish_error(span, error, scrub) raise + self._record_response(span, result, scrub) self._finish_ok(span, _document_count(result)) return result @@ -357,9 +382,12 @@ def end_open_streams() -> None: class _StreamState: """Ends one stream span exactly once; counts citations as chunks arrive.""" - def __init__(self, span: Span, scrub: Optional[_Scrub]) -> None: + def __init__( + self, span: Span, scrub: Optional[_Scrub], capture: ResponseCapture | None = None + ) -> None: self.span = span self.scrub = scrub + self.capture = capture self.finished = False # Citations accumulate over chunks; the total is known only at the end. self.citations = 0 @@ -367,6 +395,12 @@ def __init__(self, span: Span, scrub: Optional[_Scrub]) -> None: def observe(self, chunk: Any) -> None: self.citations += _chunk_citations(chunk) + if not self.finished and self.capture is not None: + try: + self.capture.chunk(chunk) + except Exception: # noqa: BLE001 - telemetry must not break iteration + self.capture = None # Withhold an incomplete preview, not the user's chunk. + logger.debug("Exa stream inspection failed; output omitted") def _claim(self) -> bool: if self.finished: @@ -377,14 +411,23 @@ def _claim(self) -> bool: def ok(self) -> None: if self._claim(): - _BaseWrapper._finish_ok(self.span, self.citations) + try: + if self.capture is not None: + self.span.set_attributes(self.capture.attributes(stream=True)) + except Exception: # noqa: BLE001 - span finalization must still run + logger.debug("Exa stream output recording failed; output omitted") + finally: + self.capture = None + _BaseWrapper._finish_ok(self.span, self.citations) def error(self, error: BaseException) -> None: if self._claim(): + self.capture = None _BaseWrapper._finish_error(self.span, error, self.scrub) def cancelled(self) -> None: if self._claim(): + self.capture = None _BaseWrapper._finish_cancelled(self.span) @@ -402,9 +445,12 @@ class _TracedStream(wrapt.ObjectProxy): # type: ignore[misc] An ObjectProxy, so ``isinstance`` against the vendor class still holds. """ - def __init__(self, response: Any, span: Span, scrub: Optional[_Scrub]) -> None: + def __init__( + self, response: Any, span: Span, scrub: Optional[_Scrub], + capture: ResponseCapture | None = None, + ) -> None: super().__init__(response) - self._self_state = _StreamState(span, scrub) + self._self_state = _StreamState(span, scrub, capture) self._self_iterator = None def __iter__(self) -> "_TracedStream": @@ -460,15 +506,18 @@ def __call__( except BaseException as error: self._finish_error(span, error, scrub) raise - return _TracedStream(response, span, scrub) + return _TracedStream(response, span, scrub, self._capture(scrub)) class _TracedAsyncStream(wrapt.ObjectProxy): # type: ignore[misc] """An AsyncStreamSearchResponse/AsyncStreamAnswerResponse that ends its span.""" - def __init__(self, response: Any, span: Span, scrub: Optional[_Scrub]) -> None: + def __init__( + self, response: Any, span: Span, scrub: Optional[_Scrub], + capture: ResponseCapture | None = None, + ) -> None: super().__init__(response) - self._self_state = _StreamState(span, scrub) + self._self_state = _StreamState(span, scrub, capture) self._self_iterator = None def __aiter__(self) -> "_TracedAsyncStream": @@ -549,4 +598,4 @@ async def __call__( except BaseException as error: self._finish_error(span, error, scrub) raise - return _TracedAsyncStream(response, span, scrub) + return _TracedAsyncStream(response, span, scrub, self._capture(scrub))