diff --git a/python/frameworks/exa/CHANGELOG.md b/python/frameworks/exa/CHANGELOG.md new file mode 100644 index 000000000..a39d6fa4a --- /dev/null +++ b/python/frameworks/exa/CHANGELOG.md @@ -0,0 +1,26 @@ +# Changelog + +All notable changes to `traceAI-exa` will be documented here. + +## [0.1.0] - Unreleased + +### Added +- `ExaInstrumentor` for the `exa-py` client (`exa-py>=2.25.0,<3`): `search`, `get_contents`, `answer`, + the deprecated `search_and_contents` alias, their async twins, and `stream_search` / `stream_answer`. +- Spans record the redacted query (capped at 1024 UTF-8 bytes) and document or citation counts; response + bodies and the API key are never recorded. +- `get_contents` records `exa.url_count`; `instrument(capture_urls=True)` adds up to 20 redacted URLs in `exa.urls`. +- `TraceConfig` / `FI_*` environment support (`FI_HIDE_INPUTS`, PII redaction, `using_session` and friends). +- Key-safe error text capped at 1 KB, cancelled-call handling, and export of streams still open at + interpreter exit. +- Spans use the repo's keys: `gen_ai.span.kind` (`RETRIEVER` for search, answer and streams; `TOOL` for + `get_contents`), the query in `gen_ai.retrieval.query`, and counts in `exa.result_count` / `exa.url_count`. +- Only methods defined on each client class are wrapped, so `AsyncExa` can never wrap an inherited `Exa` + method twice; a tracer that fails to start a span leaves the call untraced instead of raising. +- The exception event has `exception.type` and `exception.message` only; no stack trace (local paths) is exported. +- `pii_redaction` also applies to the error status and the exception event. + +### Verified +- 2026-10-17: live run against api.exa.ai with exa-py 2.25.0; response shapes matched, no code change needed. +- 2026-10-21: the test suite also passes on exa-py 2.26.0, which adds its own async `aclose()`; a traced + stream's `aclose()` delegates to it (no live run on 2.26.0). diff --git a/python/frameworks/exa/README.md b/python/frameworks/exa/README.md new file mode 100644 index 000000000..c8ee1896b --- /dev/null +++ b/python/frameworks/exa/README.md @@ -0,0 +1,143 @@ +# traceAI-exa + +OpenTelemetry instrumentation for the [`exa-py`](https://github.com/exa-labs/exa-py) client. + +This package wraps Exa client calls; it is not an Exa exporter. Exa does not publish an exporter. + +## Installation + +```bash +pip install traceAI-exa +``` + +It accepts `exa-py>=2.25.0,<3` and is tested with `exa-py` 2.25.0 on Python +3.10, 3.11, 3.12 and 3.13. The test suite also passes on `exa-py` 2.26.0. + +Live vendor verification (2026-10-17): a real run against `api.exa.ai` with +`exa-py` 2.25.0 (`search`, `get_contents`, `answer`, both streams, `AsyncExa.search` +and an invalid-key 401 recorded as ERROR) exported the expected spans to Future +AGI, and `FI_HIDE_INPUTS=true` kept the query out of every stored column. + +## Usage + +```python +from exa_py import Exa +from fi_instrumentation import register +from fi_instrumentation.fi_types import ProjectType +from traceai_exa import ExaInstrumentor + +tracer_provider = register(project_name="exa-search", project_type=ProjectType.OBSERVE) +ExaInstrumentor().instrument(tracer_provider=tracer_provider) + +client = Exa(api_key="your-exa-api-key") +client.search("recent retrieval research", num_results=5) +``` + +## Instrumented calls + +`Exa` and `AsyncExa` calls to `search`, `get_contents`, `answer`, and the +deprecated `search_and_contents` alias (traced as `exa.search`) each produce +one span. `stream_search` and `stream_answer` produce one span that stays open +until the stream is fully read, closed, or garbage-collected; the returned +object is still an instance of the vendor's stream class. A stream still open +when the interpreter exits is ended as cancelled before `register()`'s exit +hook shuts the exporter down (when `instrument()` runs after `register()`). + +`gen_ai.span.kind` is `RETRIEVER` for `search`, `answer` and the two streams, +and `TOOL` for `get_contents`, which fetches the pages you name rather than +searching. Each span is a child of the active span +when there is one, and a root span otherwise. The span is current while +exa-py sends the request, so a span from an HTTP client instrumentation nests +under it. + +## Span attributes + +| Attribute | Calls | Value | +|---|---|---| +| `gen_ai.retrieval.query`, `input.value` | `search`, `answer`, streams | The query, with the client's Exa API key replaced by `[redacted]`, cut to 1 KB of UTF-8 on a character boundary. | +| `exa.url_count` | `get_contents` | Number of requested URLs (a single URL, a list of URLs, or a list of results). | +| `exa.urls` | `get_contents` with `capture_urls=True` | See [get_contents URLs](#get_contents-urls). | +| `exa.result_count` | successful calls | `results` for `search` and `get_contents`, `citations` for `answer`, and the citations summed over every chunk of a fully read stream. Absent when the count is unknown (an error or a cancellation), never 0 for unknown. | +| `exa.cancelled` | cancelled calls and streams | `true`; see [Errors and cancellation](#errors-and-cancellation). | + +The spans never carry response content: no result titles, text, highlights or +summaries, no answer or stream text, no cost, token count or model name. There +is no setting that adds them. The Exa API key is never recorded; it is +replaced by `[redacted]` wherever it appears verbatim in a recorded query, URL +or error text. + +### get_contents URLs + +`get_contents` spans record the number of requested URLs, not the URLs, +because URLs can carry tokens or personal data. To record the URLs as well, +opt in when instrumenting: + +```python +ExaInstrumentor().instrument(tracer_provider=tracer_provider, capture_urls=True) +``` + +With `capture_urls=True`, `exa.urls` holds at most the first 20 +requested URLs, each with the Exa API key replaced by `[redacted]` and cut to +1 KB. Query strings are otherwise kept as given. Hidden inputs (below) turn +URL capture off. + +## Trace config and context + +Spans are created through `fi_instrumentation`'s `FITracer`, as in the other +traceAI instrumentors. Pass a `TraceConfig` to `instrument()`, or set the +matching `FI_*` environment variables; without `config`, the environment is +read when `instrument()` runs. + +```python +from fi_instrumentation import TraceConfig + +ExaInstrumentor().instrument( + tracer_provider=tracer_provider, config=TraceConfig(hide_inputs=True) +) +``` + +With `hide_inputs=True` (or `FI_HIDE_INPUTS=true`), `input.value` is +`__REDACTED__`, `gen_ai.retrieval.query` and `exa.urls` are not recorded, +and the query or the requested URLs are replaced by `__REDACTED__` in error +text. The URL and result counts stay. `pii_redaction` applies to the recorded +attributes and to the error text described below, and +`using_session`, `using_user`, `using_metadata` and `using_tags` attributes are +added to the Exa spans. + +## Errors and cancellation + +An error raised by exa-py (for example `ValueError` for an HTTP 4xx/5xx +response) sets the span status to ERROR and records an exception event. The +exception is re-raised unchanged. + +exa-py puts the server's response body in its error message, and a body can +repeat the request. On the span, the status description +(`: `) and the event's `exception.message` therefore +have the Exa key replaced by `[redacted]` (and, with hidden inputs, the query +or the requested URLs by `__REDACTED__`; with `pii_redaction`, PII by tokens such +as ``) and the message cut to 1 KB of UTF-8. +If the exception's own `str()` fails, the span records only its type. The event carries +`exception.type` and `exception.message` only: no `exception.stacktrace` is +exported, because frames hold local file paths, user names and host names. +Chained exceptions' messages are not copied. Only verbatim +copies are found: an escaped, normalised or truncated echo of the key, query or +URLs (for example a JSON-escaped `\/` or a percent-encoded URL) is not. + +Closing a stream before its last chunk (`close()` or `aclose()`), dropping it +mid-iteration, or cancelling an `AsyncExa` call or stream ends the span with +status ERROR, description `cancelled`, and `exa.cancelled` = `true`. + +A traced `AsyncExa` stream always has `aclose()`, which ends the span and closes +the HTTP response. exa-py's own async streams have `aclose()` from 2.26.0 only; +2.25.0's have none. On both versions their `close()` raises `RuntimeError` +(traced or not; the span is ended first). Code that must also run on exa-py +2.25.0 without this instrumentor can call `aclose()` only when it exists, for +example `aclose = getattr(stream, "aclose", None)`. + +A missing API key raises `ValueError` in `Exa()` or `AsyncExa()` itself, +before any traced method runs, so no span is recorded for it. + +`find_similar` and `find_similar_and_contents` are not traced. + +Do not use this package together with another instrumentor that wraps the same +Exa client methods, or duplicate spans may result. diff --git a/python/frameworks/exa/examples/search.py b/python/frameworks/exa/examples/search.py new file mode 100644 index 000000000..9f00e7453 --- /dev/null +++ b/python/frameworks/exa/examples/search.py @@ -0,0 +1,30 @@ +"""Run one traced Exa search against the configured Exa endpoint.""" + +import os + +from exa_py import Exa +from fi_instrumentation import register +from fi_instrumentation.fi_types import ProjectType +from traceai_exa import ExaInstrumentor + + +def main() -> None: + tracer_provider = register( + project_name="exa-example", + project_type=ProjectType.OBSERVE, + batch=False, + verbose=False, + ) + ExaInstrumentor().instrument(tracer_provider=tracer_provider) + + client_options = {"api_key": "exa-dummy-key"} + base_url = os.getenv("EXA_BASE_URL") + if base_url: + client_options["base_url"] = base_url + client = Exa(**client_options) + client.search("TraceAI Exa instrumentation example", num_results=1) + tracer_provider.force_flush() + + +if __name__ == "__main__": + main() diff --git a/python/frameworks/exa/pyproject.toml b/python/frameworks/exa/pyproject.toml new file mode 100644 index 000000000..44015d404 --- /dev/null +++ b/python/frameworks/exa/pyproject.toml @@ -0,0 +1,34 @@ +[tool.poetry] +name = "traceAI-exa" +version = "0.1.0" +description = "OpenTelemetry instrumentation for Exa" +authors = ["Future AGI "] +readme = "README.md" +packages = [ + { include = "traceai_exa" } +] +classifiers = [ + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "License :: OSI Approved :: Apache Software License", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Topic :: Software Development :: Libraries :: Python Modules", +] + +[tool.poetry.dependencies] +python = ">=3.10,<3.14" +fi-instrumentation-otel = ">=1.1.0" +exa-py = ">=2.25.0,<3" +wrapt = ">=1.0.0" + +[tool.poetry.group.dev.dependencies] +pytest = ">=7.0.0" +pytest-asyncio = ">=0.21.0" + +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" diff --git a/python/frameworks/exa/tests/_support.py b/python/frameworks/exa/tests/_support.py new file mode 100644 index 000000000..4fc3c7d6c --- /dev/null +++ b/python/frameworks/exa/tests/_support.py @@ -0,0 +1,214 @@ +"""Shared support for the real-SDK tests: a loopback Exa API and a tracer. + +``FakeExa`` speaks just enough of api.exa.ai for exa-py 2.25.0: JSON for +``/search``, ``/contents`` and ``/answer``, and server-sent events when the +request body has ``"stream": true`` (``stream_search`` / ``stream_answer``). +A query of ``fail-401`` gets HTTP 401 on every route; a query of ``slow`` is +held open until the fake closes. Nothing here reaches the network beyond +127.0.0.1, and every key is a placeholder. +""" + +from __future__ import annotations + +import contextlib +import json +import threading +from dataclasses import dataclass +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from typing import Any, Dict, Iterator, List, Tuple + +from opentelemetry.sdk.resources import Resource +from opentelemetry.sdk.trace import ReadableSpan, TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter + +EXA_KEY = "placeholder-exa-key-must-not-be-exported" +FAIL_QUERY = "fail-401" +# A query starting with this (or a get_contents URL containing it) gets HTTP +# 400 whose body echoes the request, padded past 1 KB, as some APIs and +# proxies do; exa-py copies the body into its ValueError. +ECHO_QUERY = "echo-400" +ECHO_PADDING = "p" * 3000 +# The fake holds a "slow" request (and a "slow" stream after its first chunk) +# until the test ends, so a caller can cancel mid-call. +SLOW_QUERY = "slow" +# Response content the fake returns. None of it may reach a span. +RESULT_TITLE = "RESULT-TITLE-MUST-NOT-BE-EXPORTED" +RESULT_TEXT = "RESULT-TEXT-MUST-NOT-BE-EXPORTED" +ANSWER_TEXT = "ANSWER-TEXT-MUST-NOT-BE-EXPORTED" +STREAM_TEXT = "STREAM-TEXT-MUST-NOT-BE-EXPORTED" +CONTENT_MARKERS = (RESULT_TITLE, RESULT_TEXT, ANSWER_TEXT, STREAM_TEXT) + +SEARCH_RESULTS = 2 +ANSWER_CITATIONS = 1 +# A stream carries its citations in two separate chunks (2, then 1), so a +# count taken from only the first or the last chunk is wrong. +STREAM_CITATION_CHUNKS = (2, 1) +STREAM_CITATIONS = sum(STREAM_CITATION_CHUNKS) + + +def _document(index: int) -> Dict[str, Any]: + return { + "id": "doc-{0}".format(index), + "url": "https://example.com/{0}".format(index), + "title": RESULT_TITLE, + "text": RESULT_TEXT, + } + + +def _stream_events() -> List[Dict[str, Any]]: + events: List[Dict[str, Any]] = [] + index = 0 + for count in STREAM_CITATION_CHUNKS: + events.append({"choices": [{"delta": {"content": STREAM_TEXT}}]}) + events.append({"citations": [_document(index + n) for n in range(count)]}) + index += count + events.append({"choices": [{"delta": {"content": STREAM_TEXT}}]}) + return events + + +# Chunks a fully consumed stream yields (exa-py drops the [DONE] sentinel). +STREAM_CHUNKS = len(_stream_events()) + + +class _QuietServer(ThreadingHTTPServer): + def handle_error(self, request: Any, client_address: Any) -> None: + # A cancelled client leaves a broken pipe behind; that is expected. + return + + +class FakeExa: + """Loopback stand-in for api.exa.ai.""" + + def __init__(self) -> None: + self.calls: List[Tuple[str, str, Dict[str, Any]]] = [] + self._release = threading.Event() + owner = self + + class Handler(BaseHTTPRequestHandler): + def do_POST(self) -> None: # noqa: N802 - BaseHTTPRequestHandler API + length = int(self.headers.get("Content-Length", "0") or 0) + body = json.loads(self.rfile.read(length) or b"{}") + path = self.path.split("?")[0] + owner.calls.append(("POST", path, body)) + slow = body.get("query") == SLOW_QUERY + if body.get("query") == FAIL_QUERY: + self._send(401, "application/json", {"error": "invalid api key"}) + elif str(body.get("query", "")).startswith(ECHO_QUERY): + error = "Bad query: {0} {1}".format(body["query"], ECHO_PADDING) + self._send(400, "application/json", {"error": error}) + elif body.get("stream"): + events = [ + "data: {0}\n\n".format(json.dumps(event)) for event in _stream_events() + ] + events.append("data: [DONE]\n\n") + # A slow stream sends its first chunk, then stalls. + self._send( + 200, + "text/event-stream", + "".join(events), + stall_after=len(events[0].encode("utf-8")) if slow else None, + ) + elif slow: + owner._release.wait(30) + self._send(200, "application/json", {"results": []}) + elif path == "/search": + count = int(body.get("numResults") or SEARCH_RESULTS) + results = [_document(i) for i in range(count)] + self._send(200, "application/json", {"requestId": "req-1", "results": results}) + elif path == "/contents" and any( + ECHO_QUERY in str(url) for url in body.get("urls", []) + ): + error = "Bad urls: {0} {1}".format(", ".join(body["urls"]), ECHO_PADDING) + self._send(400, "application/json", {"error": error}) + elif path == "/contents": + results = [ + dict(_document(i), url=url) for i, url in enumerate(body.get("urls", [])) + ] + self._send(200, "application/json", {"results": results, "statuses": []}) + elif path == "/answer": + citations = [_document(i) for i in range(ANSWER_CITATIONS)] + self._send( + 200, "application/json", {"answer": ANSWER_TEXT, "citations": citations} + ) + else: + self._send(404, "application/json", {"error": "not found"}) + + def _send( + self, status: int, content_type: str, payload: Any, stall_after: Any = None + ) -> None: + text = payload if isinstance(payload, str) else json.dumps(payload) + data = text.encode("utf-8") + self.send_response(status) + self.send_header("Content-Type", content_type) + self.send_header("Content-Length", str(len(data))) + self.end_headers() + if stall_after is not None: + self.wfile.write(data[:stall_after]) + self.wfile.flush() + owner._release.wait(30) + data = data[stall_after:] + self.wfile.write(data) + + def log_message(self, *_: Any) -> None: + return + + self._server = _QuietServer(("127.0.0.1", 0), Handler) + self.origin = "http://127.0.0.1:{0}".format(self._server.server_port) + self._thread = threading.Thread( + target=self._server.serve_forever, kwargs={"poll_interval": 0.05}, daemon=True + ) + self._thread.start() + + def paths(self) -> List[str]: + return [path for _, path, _ in self.calls] + + def close(self) -> None: + self._release.set() + self._server.shutdown() + self._server.server_close() + self._thread.join() + + def __enter__(self) -> "FakeExa": + return self + + def __exit__(self, *_: Any) -> None: + self.close() + + +@dataclass +class Traced: + exporter: InMemorySpanExporter + provider: TracerProvider + + def spans(self) -> List[ReadableSpan]: + return list(self.exporter.get_finished_spans()) + + def one(self) -> ReadableSpan: + spans = self.spans() + assert len(spans) == 1, [span.name for span in spans] + return spans[0] + + def wire(self) -> str: + """Everything an exporter could send: attributes, events, status.""" + return "".join(span.to_json() for span in self.spans()) + + +@contextlib.contextmanager +def instrumented(**options: Any) -> Iterator[Traced]: + """Instrument Exa against a fresh in-memory provider; uninstrument after.""" + from traceai_exa import ExaInstrumentor + + exporter = InMemorySpanExporter() + provider = TracerProvider(resource=Resource.create({"service.name": "test-exa"})) + provider.add_span_processor(SimpleSpanProcessor(exporter)) + instrumentor = ExaInstrumentor() + instrumentor.instrument(tracer_provider=provider, **options) + try: + yield Traced(exporter, provider) + finally: + instrumentor.uninstrument() + + +def attrs(span: ReadableSpan) -> Dict[str, Any]: + return dict(span.attributes or {}) diff --git a/python/frameworks/exa/tests/conftest.py b/python/frameworks/exa/tests/conftest.py new file mode 100644 index 000000000..fcdf1b13e --- /dev/null +++ b/python/frameworks/exa/tests/conftest.py @@ -0,0 +1,27 @@ +"""Pytest configuration and fixtures for Exa instrumentation tests.""" + +from typing import Generator + +import pytest +from opentelemetry import trace as trace_api +from opentelemetry.sdk import trace as trace_sdk +from opentelemetry.sdk.resources import Resource +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter + + +@pytest.fixture +def in_memory_span_exporter() -> InMemorySpanExporter: + """Create an in-memory span exporter for testing.""" + return InMemorySpanExporter() + + +@pytest.fixture +def tracer_provider( + in_memory_span_exporter: InMemorySpanExporter, +) -> trace_api.TracerProvider: + """Create a tracer provider with an in-memory exporter.""" + resource = Resource(attributes={"service.name": "test-exa"}) + provider = trace_sdk.TracerProvider(resource=resource) + provider.add_span_processor(SimpleSpanProcessor(in_memory_span_exporter)) + return provider diff --git a/python/frameworks/exa/tests/test_async_parity.py b/python/frameworks/exa/tests/test_async_parity.py new file mode 100644 index 000000000..4cc0a4791 --- /dev/null +++ b/python/frameworks/exa/tests/test_async_parity.py @@ -0,0 +1,93 @@ +"""AsyncExa parity: the async twins give the same spans as Exa (AC-05).""" + +from __future__ import annotations + +import asyncio +import warnings + +import pytest +from opentelemetry.trace import StatusCode + +pytest.importorskip("exa_py", reason="exa-py must be installed to test its instrumentor") + +from _support import CONTENT_MARKERS, EXA_KEY, FAIL_QUERY, FakeExa, attrs, instrumented # noqa: E402 +from exa_py import AsyncExa, Exa # noqa: E402 + +URLS = ["https://example.com/0", "https://example.com/1"] + + +@pytest.fixture() +def fake(): + with FakeExa() as server: + yield server + + +def _summary(traced): + return [ + (span.name, span.status.status_code, sorted(attrs(span).items())) + for span in traced.spans() + ] + + +def _sync_calls(client: Exa) -> None: + client.search("parity query", num_results=2) + client.get_contents(URLS) + client.answer("parity query") + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + client.search_and_contents("parity query") + + +async def _async_calls(client: AsyncExa) -> None: + try: + await client.search("parity query", num_results=2) + await client.get_contents(URLS) + await client.answer("parity query") + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + await client.search_and_contents("parity query") + finally: + await client.client.aclose() + + +def test_async_client_produces_the_same_spans_as_the_sync_client(fake): + with instrumented() as sync_traced: + _sync_calls(Exa(api_key=EXA_KEY, base_url=fake.origin)) + with instrumented() as async_traced: + asyncio.run(_async_calls(AsyncExa(api_key=EXA_KEY, api_base=fake.origin))) + + expected = _summary(sync_traced) + assert [name for name, _, _ in expected] == [ + "exa.search", + "exa.get_contents", + "exa.answer", + "exa.search", + ] + assert all(status is StatusCode.OK for _, status, _ in expected) + assert _summary(async_traced) == expected + # Both clients made real HTTP calls to the fake. + assert fake.paths() == ["/search", "/contents", "/answer", "/search"] * 2 + for traced in (sync_traced, async_traced): + wire = traced.wire() + assert EXA_KEY not in wire + for marker in CONTENT_MARKERS: + assert marker not in wire + + +def test_async_vendor_error_is_recorded_and_reraised(fake): + async def call() -> None: + client = AsyncExa(api_key=EXA_KEY, api_base=fake.origin) + try: + with pytest.raises(ValueError, match="401"): + await client.search(FAIL_QUERY) + finally: + await client.client.aclose() + + with instrumented() as traced: + asyncio.run(call()) + + span = traced.one() + assert span.status.status_code is StatusCode.ERROR + assert [event.name for event in span.events] == ["exception"] + assert "exa.cancelled" not in attrs(span) + assert "exa.result_count" not in attrs(span) diff --git a/python/frameworks/exa/tests/test_consistency.py b/python/frameworks/exa/tests/test_consistency.py new file mode 100644 index 000000000..896a5c667 --- /dev/null +++ b/python/frameworks/exa/tests/test_consistency.py @@ -0,0 +1,106 @@ +"""Family consistency: span kinds and keys, class-local wrapping, isolation, PII in error text.""" + +from __future__ import annotations + +import types + +import pytest +from opentelemetry.sdk.trace import TracerProvider + +pytest.importorskip("exa_py", reason="exa-py must be installed to test its instrumentor") + +from _support import ECHO_QUERY, EXA_KEY, FakeExa, attrs, instrumented # noqa: E402 +from exa_py import Exa # noqa: E402 +from fi_instrumentation import TraceConfig # noqa: E402 + +from traceai_exa import ExaInstrumentor # noqa: E402 + +KIND = "gen_ai.span.kind" + + +@pytest.fixture() +def fake(): + with FakeExa() as server: + yield server + + +def _client(fake: FakeExa) -> Exa: + return Exa(api_key=EXA_KEY, base_url=fake.origin) + + +def test_search_and_answer_are_retrievers_and_get_contents_is_a_tool(fake): + with instrumented() as traced: + client = _client(fake) + client.search("q") + client.answer("q") + client.get_contents(["https://example.com/a"]) + + kinds = {span.name: attrs(span)[KIND] for span in traced.spans()} + assert kinds == {"exa.search": "RETRIEVER", "exa.answer": "RETRIEVER", "exa.get_contents": "TOOL"} + assert all("fi.span.kind" not in attrs(span) for span in traced.spans()) + + +def test_search_query_uses_the_repo_retrieval_key(fake): + with instrumented() as traced: + _client(fake).search("open telemetry") + + values = attrs(traced.one()) + assert values["gen_ai.retrieval.query"] == "open telemetry" + assert values["input.value"] == "open telemetry" + assert values["exa.result_count"] == 2 + assert not [key for key in values if key.startswith("fi.retrieval.")] + + +def test_a_method_only_inherited_by_a_class_is_not_wrapped_on_it(): + class Parent: + def search(self, query): + return query + + class Child(Parent): + pass + + module = types.ModuleType("fake_exa_api") + module.Parent, module.Child = Parent, Child + instrumentor = ExaInstrumentor() + instrumentor._original_methods = {} + instrumentor._wrap_method(module, "Child", "search", lambda w, i, a, k: w(*a, **k)) + + assert "search" not in vars(Child) + assert instrumentor._original_methods == {} + + +class _BrokenTracerProvider(TracerProvider): + def get_tracer(self, *args, **kwargs): + tracer = super().get_tracer(*args, **kwargs) + + def start_span(*_args, **_kwargs): + raise RuntimeError("tracer is broken") + + tracer.start_span = start_span + return tracer + + +def test_a_failing_start_span_never_reaches_the_caller(fake): + instrumentor = ExaInstrumentor() + instrumentor.instrument(tracer_provider=_BrokenTracerProvider()) + try: + client = _client(fake) + result = client.search("q") + assert len(result.results) == 2 + assert [chunk for chunk in client.stream_answer("q")] + finally: + instrumentor.uninstrument() + + +def test_pii_redaction_also_covers_error_text(fake): + query = "{0} mail jane.doe@example.com".format(ECHO_QUERY) + with instrumented(config=TraceConfig(pii_redaction=True)) as traced: + with pytest.raises(ValueError): + _client(fake).search(query) + + span = traced.one() + (event,) = [event for event in span.events if event.name == "exception"] + assert "jane.doe@example.com" not in span.status.description + assert "jane.doe@example.com" not in event.attributes["exception.message"] + assert "exception.stacktrace" not in event.attributes + assert "" in event.attributes["exception.message"] diff --git a/python/frameworks/exa/tests/test_context.py b/python/frameworks/exa/tests/test_context.py new file mode 100644 index 000000000..82eb42bb8 --- /dev/null +++ b/python/frameworks/exa/tests/test_context.py @@ -0,0 +1,110 @@ +"""Trace context: the Exa span is current during the vendor call and nests.""" + +from __future__ import annotations + +import asyncio + +import pytest +from opentelemetry import trace as trace_api + +pytest.importorskip("exa_py", reason="exa-py must be installed to test its instrumentor") + +from _support import EXA_KEY, FakeExa, instrumented # noqa: E402 +from exa_py import AsyncExa, Exa # noqa: E402 + + +@pytest.fixture() +def fake(): + with FakeExa() as server: + yield server + + +def _current_span_id() -> int: + return trace_api.get_current_span().get_span_context().span_id + + +@pytest.fixture() +def seen_by_http(monkeypatch): + """Record the current span id where exa-py sends HTTP (Exa.request / async_request). + + An HTTP client instrumentor starts its span at that point, so the id seen + there is the parent an HTTP child span would get. + """ + seen = [] + sync_request = Exa.request + async_request = AsyncExa.async_request + + def request(self, *args, **kwargs): + seen.append(_current_span_id()) + return sync_request(self, *args, **kwargs) + + async def async_request_(self, *args, **kwargs): + seen.append(_current_span_id()) + return await async_request(self, *args, **kwargs) + + monkeypatch.setattr(Exa, "request", request) + monkeypatch.setattr(AsyncExa, "async_request", async_request_) + return seen + + +def test_sync_vendor_http_runs_inside_the_exa_span(fake, seen_by_http): + with instrumented() as traced: + client = Exa(api_key=EXA_KEY, base_url=fake.origin) + client.search("q") + client.answer("q") + client.get_contents("https://example.com/0") + list(client.stream_search("q")) + list(client.stream_answer("q")) + + span_ids = [span.context.span_id for span in traced.spans()] + assert len(span_ids) == 5 + assert seen_by_http == span_ids + # Nothing is left current once the calls return. + assert _current_span_id() == 0 + + +def test_async_vendor_http_runs_inside_the_exa_span(fake, seen_by_http): + async def call() -> None: + client = AsyncExa(api_key=EXA_KEY, api_base=fake.origin) + try: + await client.search("q") + await client.get_contents(["https://example.com/0"]) + async for _ in await client.stream_answer("q"): + pass + finally: + await client.client.aclose() + + with instrumented() as traced: + asyncio.run(call()) + + span_ids = [span.context.span_id for span in traced.spans()] + assert len(span_ids) == 3 + assert seen_by_http == span_ids + + +def test_exa_span_is_a_child_of_the_active_span(fake): + async def call(client) -> None: + try: + await client.answer("q") + finally: + await client.client.aclose() + + with instrumented() as traced: + tracer = traced.provider.get_tracer("agent") + with tracer.start_as_current_span("agent.step") as parent: + Exa(api_key=EXA_KEY, base_url=fake.origin).search("q") + asyncio.run(call(AsyncExa(api_key=EXA_KEY, api_base=fake.origin))) + + by_name = {span.name: span for span in traced.spans()} + agent = by_name["agent.step"] + assert agent.context.span_id == parent.get_span_context().span_id + for name in ("exa.search", "exa.answer"): + assert by_name[name].parent.span_id == agent.context.span_id + assert by_name[name].context.trace_id == agent.context.trace_id + + +def test_exa_span_is_a_root_span_without_a_parent(fake): + with instrumented() as traced: + Exa(api_key=EXA_KEY, base_url=fake.origin).search("q") + + assert traced.one().parent is None diff --git a/python/frameworks/exa/tests/test_contract_harness.py b/python/frameworks/exa/tests/test_contract_harness.py new file mode 100644 index 000000000..95f641d6e --- /dev/null +++ b/python/frameworks/exa/tests/test_contract_harness.py @@ -0,0 +1,188 @@ +"""Shared-harness contract test for traceAI-exa (TH-8324, TH-8339 Receiver). + +The real ``exa-py`` client makes real HTTP calls to a loopback fake of the +Exa API (``Exa(base_url=...)``). Spans leave through the real +``fi_instrumentation.register()`` OTLP exporter into ``harness.Receiver``. +Nothing calls Exa; every key is a placeholder. +""" + +from __future__ import annotations + +import json +import sys +import threading +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from typing import Any, Dict, List, Tuple + +import pytest + +pytest.importorskip("exa_py", reason="exa-py must be installed to test its instrumentor") +pytest.importorskip("opentelemetry.proto", reason="the harness decodes OTLP protobuf") + +sys.path.insert(0, str(Path(__file__).resolve().parents[3] / "tests")) + +from harness import Receiver, _flatten_attributes # noqa: E402 + +FI_API_KEY = "placeholder-fi-api-key" +FI_SECRET_KEY = "placeholder-fi-secret-key" +EXA_KEY = "placeholder-exa-key-must-not-be-exported" +PROJECT = "exa-contract" +# Response text the fake API returns. None of it may reach a span. +RESULT_TITLE = "RESULT-TITLE-MUST-NOT-BE-EXPORTED" +ANSWER_TEXT = "ANSWER-TEXT-MUST-NOT-BE-EXPORTED" + +_ROUTES: Dict[Tuple[str, str], Tuple[int, Dict[str, Any]]] = { + ("POST", "/search"): ( + 200, + { + "requestId": "req-1", + "results": [ + {"id": "doc-1", "url": "https://example.com/a", "title": RESULT_TITLE}, + {"id": "doc-2", "url": "https://example.com/b", "title": RESULT_TITLE}, + ], + }, + ), + ("POST", "/answer"): ( + 200, + { + "answer": ANSWER_TEXT, + "citations": [{"id": "doc-1", "url": "https://example.com/a", "title": RESULT_TITLE}], + }, + ), +} + + +class FakeExa: + """Loopback stand-in for api.exa.ai. A query of ``fail-401`` gets HTTP 401.""" + + def __init__(self) -> None: + self.calls: List[Tuple[str, str, Dict[str, Any]]] = [] + owner = self + + class Handler(BaseHTTPRequestHandler): + def do_POST(self) -> None: # noqa: N802 + length = int(self.headers.get("Content-Length", "0") or 0) + body = json.loads(self.rfile.read(length) or b"{}") + path = self.path.split("?")[0] + owner.calls.append(("POST", path, body)) + status, payload = _ROUTES.get(("POST", path), (404, {"error": "not found"})) + if body.get("query") == "fail-401": + status, payload = 401, {"error": "invalid api key"} + data = json.dumps(payload).encode() + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(data))) + self.end_headers() + self.wfile.write(data) + + def log_message(self, *_: Any) -> None: + return + + self._server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + self.origin = f"http://127.0.0.1:{self._server.server_port}" + self._thread = threading.Thread(target=self._server.serve_forever, daemon=True) + self._thread.start() + + def close(self) -> None: + self._server.shutdown() + self._server.server_close() + self._thread.join() + + +@pytest.fixture() +def fake_exa(): + server = FakeExa() + yield server + server.close() + + +def _journey(receiver: Receiver, fake: FakeExa, monkeypatch: pytest.MonkeyPatch) -> List[dict]: + from exa_py import Exa + from fi_instrumentation import register + from fi_instrumentation.fi_types import ProjectType + + from traceai_exa import ExaInstrumentor + + monkeypatch.setenv("FI_BASE_URL", receiver.origin) + monkeypatch.setenv("FI_API_KEY", FI_API_KEY) + monkeypatch.setenv("FI_SECRET_KEY", FI_SECRET_KEY) + provider = register(project_type=ProjectType.OBSERVE, project_name=PROJECT, verbose=False) + instrumentor = ExaInstrumentor() + instrumentor.instrument(tracer_provider=provider) + try: + exa = Exa(api_key=EXA_KEY, base_url=fake.origin) + assert len(exa.search("open telemetry retrieval", num_results=2).results) == 2 + assert exa.answer("what is OTLP?").answer == ANSWER_TEXT + with pytest.raises(ValueError): + exa.search("fail-401") + assert provider.force_flush(timeout_millis=10_000) + finally: + instrumentor.uninstrument() + provider.shutdown() + return receiver.spans() + + +def test_real_client_calls_reach_the_collector_contract(fake_exa, monkeypatch): + with Receiver() as receiver: + spans = _journey(receiver, fake_exa, monkeypatch) + exports = receiver.requests() + + # The real SDK made the HTTP calls (no monkeypatched client methods). + assert [(m, p) for m, p, _ in fake_exa.calls] == [ + ("POST", "/search"), + ("POST", "/answer"), + ("POST", "/search"), + ] + + # Collector contract on every export. + assert exports + for export in exports: + assert export["path"] == "/tracer/v1/traces" + assert export["headers"]["x-api-key"] == FI_API_KEY + assert export["headers"]["x-secret-key"] == FI_SECRET_KEY + assert "authorization" not in export["headers"] + for resource in export["resource_attributes"]: + assert resource["project_name"] == PROJECT + assert resource["project_type"] == "observe" + + by_name: Dict[str, List[dict]] = {} + for span in spans: + by_name.setdefault(span["name"], []).append(span) + assert sorted((name, len(group)) for name, group in by_name.items()) == [ + ("exa.answer", 1), + ("exa.search", 2), + ] + + ok_search, failed_search = by_name["exa.search"] + ok = _flatten_attributes(ok_search["attributes"]) + assert ok["gen_ai.span.kind"] == "RETRIEVER" + assert ok["gen_ai.retrieval.query"] == "open telemetry retrieval" + assert ok["input.value"] == "open telemetry retrieval" + assert int(ok["exa.result_count"]) == 2 + # register()'s processor also promotes UNSET to OK on export + # (fi_instrumentation/otel.py _auto_set_ok_status), so the wrapper's own + # OK is pinned by the in-memory tests (test_async_parity, test_streams). + assert ok_search["status"]["code"] == "STATUS_CODE_OK" + assert by_name["exa.answer"][0]["status"]["code"] == "STATUS_CODE_OK" + + answer = _flatten_attributes(by_name["exa.answer"][0]["attributes"]) + assert answer["gen_ai.span.kind"] == "RETRIEVER" + # AnswerResponse carries citations, not results; the fake returns one. + assert int(answer["exa.result_count"]) == 1 + assert not any("model" in key for key in answer) + + failed = _flatten_attributes(failed_search["attributes"]) + assert failed["gen_ai.retrieval.query"] == "fail-401" + assert failed_search["status"]["code"] == "STATUS_CODE_ERROR" + assert any(event["name"] == "exception" for event in failed_search.get("events", [])) + + +def test_no_key_or_response_content_is_exported(fake_exa, monkeypatch): + with Receiver() as receiver: + spans = _journey(receiver, fake_exa, monkeypatch) + exports = receiver.requests() + + wire = json.dumps(spans) + json.dumps(exports) + for secret in (EXA_KEY, RESULT_TITLE, ANSWER_TEXT, "https://example.com/a"): + assert secret not in wire, secret diff --git a/python/frameworks/exa/tests/test_document_count.py b/python/frameworks/exa/tests/test_document_count.py new file mode 100644 index 000000000..efcd4b2ef --- /dev/null +++ b/python/frameworks/exa/tests/test_document_count.py @@ -0,0 +1,124 @@ +"""exa.result_count from the real SDK against the loopback fake. + +The count is what the call returned: search/get_contents results, answer +citations, and for streams the citations accumulated over every chunk. When +the count is unknown (error, cancellation, unrecognised result) the attribute +is absent rather than 0. +""" + +from __future__ import annotations + +import asyncio + +import pytest + +pytest.importorskip("exa_py", reason="exa-py must be installed to test its instrumentor") + +from _support import ( # noqa: E402 + ANSWER_CITATIONS, + EXA_KEY, + FAIL_QUERY, + SEARCH_RESULTS, + STREAM_CITATIONS, + FakeExa, + attrs, + instrumented, +) +from exa_py import AsyncExa, Exa # noqa: E402 + +COUNT = "exa.result_count" + + +@pytest.fixture() +def fake(): + with FakeExa() as server: + yield server + + +def _counts(traced): + return [(span.name, attrs(span).get(COUNT)) for span in traced.spans()] + + +def test_sync_counts_results_and_citations(fake): + with instrumented() as traced: + client = Exa(api_key=EXA_KEY, base_url=fake.origin) + client.search("q") + assert len(client.answer("q").citations) == ANSWER_CITATIONS + client.get_contents(["https://example.com/0"]) + + assert _counts(traced) == [ + ("exa.search", SEARCH_RESULTS), + ("exa.answer", ANSWER_CITATIONS), + ("exa.get_contents", 1), + ] + + +def test_async_counts_results_and_citations(fake): + async def call() -> None: + client = AsyncExa(api_key=EXA_KEY, api_base=fake.origin) + try: + await client.search("q") + await client.answer("q") + finally: + await client.client.aclose() + + with instrumented() as traced: + asyncio.run(call()) + + assert _counts(traced) == [("exa.search", SEARCH_RESULTS), ("exa.answer", ANSWER_CITATIONS)] + + +def test_sync_streams_count_citations_accumulated_over_chunks(fake): + with instrumented() as traced: + client = Exa(api_key=EXA_KEY, base_url=fake.origin) + answer_citations = sum(len(chunk.citations or []) for chunk in client.stream_answer("q")) + search_citations = sum(len(chunk.citations or []) for chunk in client.stream_search("q")) + + assert answer_citations == search_citations == STREAM_CITATIONS + assert _counts(traced) == [ + ("exa.answer", STREAM_CITATIONS), + ("exa.search", STREAM_CITATIONS), + ] + + +def test_async_streams_count_citations_accumulated_over_chunks(fake): + async def call() -> None: + client = AsyncExa(api_key=EXA_KEY, api_base=fake.origin) + try: + async for _ in await client.stream_answer("q"): + pass + async for _ in await client.stream_search("q"): + pass + finally: + await client.client.aclose() + + with instrumented() as traced: + asyncio.run(call()) + + assert _counts(traced) == [ + ("exa.answer", STREAM_CITATIONS), + ("exa.search", STREAM_CITATIONS), + ] + + +def test_unknown_counts_are_omitted_not_zero(fake): + with instrumented() as traced: + client = Exa(api_key=EXA_KEY, base_url=fake.origin) + with pytest.raises(ValueError): + client.search(FAIL_QUERY) + stream = client.stream_answer("q") + next(iter(stream)) + stream.close() # cancelled after one chunk: the total is unknown + + assert _counts(traced) == [("exa.search", None), ("exa.answer", None)] + + +def test_unrecognised_result_has_no_count(monkeypatch): + def search(_self, _query, **_kwargs): + return object() + + monkeypatch.setattr(Exa, "search", search) + with instrumented() as traced: + Exa(api_key=EXA_KEY).search("q") + + assert COUNT not in attrs(traced.one()) diff --git a/python/frameworks/exa/tests/test_example.py b/python/frameworks/exa/tests/test_example.py new file mode 100644 index 000000000..f234b5c5b --- /dev/null +++ b/python/frameworks/exa/tests/test_example.py @@ -0,0 +1,75 @@ +"""examples/search.py runs end to end through the shared harness. + +harness.run starts the example as a subprocess with FI_BASE_URL pointed at +harness.Receiver and EXA_BASE_URL pointed at the loopback Exa fake, so the +real exporter and the real exa-py client both run. Nothing calls Exa. +""" + +from __future__ import annotations + +import json +import os +import sys +from pathlib import Path + +import pytest + +pytest.importorskip("exa_py", reason="exa-py must be installed to run the example") +pytest.importorskip("opentelemetry.proto", reason="the harness decodes OTLP protobuf") + +from _support import CONTENT_MARKERS, FakeExa # noqa: E402 +from harness import Receiver, _flatten_attributes, run # noqa: E402 + +PACKAGE = Path(__file__).resolve().parents[1] +PYTHON_ROOT = PACKAGE.parents[1] +EXAMPLE = PACKAGE / "examples" / "search.py" +FI_API_KEY = "placeholder-fi-api-key" +FI_SECRET_KEY = "placeholder-fi-secret-key" + + +def test_example_exports_one_exa_search_span(): + with FakeExa() as fake, Receiver() as receiver: + env = { + key: value + for key, value in os.environ.items() + if not key.startswith(("FI_", "EXA_", "OTEL_")) + } + env.update( + { + "FI_BASE_URL": receiver.origin, + "FI_API_KEY": FI_API_KEY, + "FI_SECRET_KEY": FI_SECRET_KEY, + "EXA_BASE_URL": fake.origin, + "PYTHONPATH": os.pathsep.join( + [str(PACKAGE), str(PYTHON_ROOT), os.environ.get("PYTHONPATH", "")] + ), + } + ) + result = run([sys.executable, str(EXAMPLE)], env, None, timeout=120) + spans = receiver.spans() + exports = receiver.requests() + + assert not result.timed_out + assert result.returncode == 0, result.stderr.decode(errors="replace") + assert fake.paths() == ["/search"] + + assert [span["name"] for span in spans] == ["exa.search"] + span = spans[0] + values = _flatten_attributes(span["attributes"]) + assert values["gen_ai.span.kind"] == "RETRIEVER" + assert values["gen_ai.retrieval.query"] == "TraceAI Exa instrumentation example" + assert values["input.value"] == "TraceAI Exa instrumentation example" + assert int(values["exa.result_count"]) == 1 + assert span["status"]["code"] == "STATUS_CODE_OK" + + assert exports + for export in exports: + assert export["path"] == "/tracer/v1/traces" + assert export["headers"]["x-api-key"] == FI_API_KEY + for resource in export["resource_attributes"]: + assert resource["project_name"] == "exa-example" + + wire = json.dumps(spans) + assert "exa-dummy-key" not in wire + for marker in CONTENT_MARKERS: + assert marker not in wire diff --git a/python/frameworks/exa/tests/test_exit.py b/python/frameworks/exa/tests/test_exit.py new file mode 100644 index 000000000..a93a7fb9e --- /dev/null +++ b/python/frameworks/exa/tests/test_exit.py @@ -0,0 +1,76 @@ +"""A stream left open at interpreter exit still exports its span (PRD AC-09). + +register() installs an atexit hook that shuts the tracer provider down. A +stream still referenced from a module global is only finalised after atexit +hooks run, too late for the exporter, so the instrumentor ends open stream +spans from its own atexit hook, which runs first (atexit is LIFO and +instrument() follows register()). +""" + +from __future__ import annotations + +import os +import sys +from pathlib import Path + +import pytest + +pytest.importorskip("exa_py", reason="exa-py must be installed to run the script") +pytest.importorskip("opentelemetry.proto", reason="the harness decodes OTLP protobuf") + +from _support import FakeExa # noqa: E402 +from harness import Receiver, _flatten_attributes, run # noqa: E402 + +PACKAGE = Path(__file__).resolve().parents[1] +PYTHON_ROOT = PACKAGE.parents[1] + +SCRIPT = """ +import os + +from exa_py import Exa +from fi_instrumentation import register +from fi_instrumentation.fi_types import ProjectType +from traceai_exa import ExaInstrumentor + +provider = register( + project_name="exa-exit", project_type=ProjectType.OBSERVE, batch={batch}, verbose=False +) +ExaInstrumentor().instrument(tracer_provider=provider) +client = Exa(api_key="exa-dummy-key", base_url=os.environ["EXA_BASE_URL"]) +stream = client.stream_search("left open at exit") +next(iter(stream)) +# The script ends here with the stream still referenced by a module global. +""" + + +@pytest.mark.parametrize("batch", [True, False], ids=["batch", "simple"]) +def test_stream_left_open_at_exit_exports_a_cancelled_span(tmp_path, batch): + script = tmp_path / "left_open.py" + script.write_text(SCRIPT.format(batch=batch)) + with FakeExa() as fake, Receiver() as receiver: + env = { + key: value + for key, value in os.environ.items() + if not key.startswith(("FI_", "EXA_", "OTEL_")) + } + env.update( + { + "FI_BASE_URL": receiver.origin, + "FI_API_KEY": "placeholder-fi-api-key", + "FI_SECRET_KEY": "placeholder-fi-secret-key", + "EXA_BASE_URL": fake.origin, + "PYTHONPATH": os.pathsep.join( + [str(PACKAGE), str(PYTHON_ROOT), os.environ.get("PYTHONPATH", "")] + ), + } + ) + result = run([sys.executable, str(script)], env, None, timeout=120) + spans = receiver.spans() + + assert not result.timed_out + assert result.returncode == 0, result.stderr.decode(errors="replace") + assert [span["name"] for span in spans] == ["exa.search"] + (span,) = spans + assert span["status"]["code"] == "STATUS_CODE_ERROR" + assert span["status"]["message"] == "cancelled" + assert _flatten_attributes(span["attributes"])["exa.cancelled"] is True diff --git a/python/frameworks/exa/tests/test_instrumentor.py b/python/frameworks/exa/tests/test_instrumentor.py new file mode 100644 index 000000000..566008449 --- /dev/null +++ b/python/frameworks/exa/tests/test_instrumentor.py @@ -0,0 +1,115 @@ +"""Unit tests for Exa instrumentation.""" + +from __future__ import annotations + +from types import SimpleNamespace +from typing import Any, Dict + +import pytest +from opentelemetry.sdk.resources import Resource +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +from opentelemetry.trace import StatusCode + +pytest.importorskip("exa_py", reason="exa-py must be installed to test its instrumentor") +from exa_py.api import Exa # noqa: E402 +from traceai_exa import ExaInstrumentor # noqa: E402 + +_API_KEY = "exa-api-key-should-not-appear-in-a-span" + + +def _tracing() -> tuple[TracerProvider, InMemorySpanExporter]: + exporter = InMemorySpanExporter() + provider = TracerProvider(resource=Resource.create({"service.name": "test-exa"})) + provider.add_span_processor(SimpleSpanProcessor(exporter)) + return provider, exporter + + +def _instrument(provider: TracerProvider) -> ExaInstrumentor: + instrumentor = ExaInstrumentor() + instrumentor.instrument(tracer_provider=provider) + return instrumentor + + +def _attrs(exporter: InMemorySpanExporter) -> Dict[str, Any]: + spans = exporter.get_finished_spans() + assert len(spans) == 1 + return dict(spans[0].attributes or {}) + + +def test_search_records_retrieval_contract(monkeypatch: pytest.MonkeyPatch) -> None: + def search(_self: Exa, _query: str, **_kwargs: Any) -> Any: + return SimpleNamespace(results=[SimpleNamespace(), SimpleNamespace()]) + + monkeypatch.setattr(Exa, "search", search) + provider, exporter = _tracing() + instrumentor = _instrument(provider) + try: + Exa(api_key=_API_KEY).search("retrieval query") + finally: + instrumentor.uninstrument() + + spans = exporter.get_finished_spans() + assert len(spans) == 1 + span = spans[0] + attrs = dict(span.attributes or {}) + assert span.name == "exa.search" + assert attrs["gen_ai.span.kind"] == "RETRIEVER" + assert attrs["gen_ai.retrieval.query"] == "retrieval query" + assert attrs["exa.result_count"] == 2 + assert all(_API_KEY not in str(value) for value in attrs.values()) + + +def test_search_exception_is_recorded_and_reraised( + monkeypatch: pytest.MonkeyPatch, +) -> None: + def search(_self: Exa, _query: str, **_kwargs: Any) -> Any: + raise RuntimeError("Exa is unavailable") + + monkeypatch.setattr(Exa, "search", search) + provider, exporter = _tracing() + instrumentor = _instrument(provider) + try: + with pytest.raises(RuntimeError, match="Exa is unavailable"): + Exa(api_key=_API_KEY).search("retrieval query") + finally: + instrumentor.uninstrument() + + span = exporter.get_finished_spans()[0] + assert span.status.status_code is StatusCode.ERROR + assert any(event.name == "exception" for event in span.events) + + +def test_search_and_contents_uses_search_span_name( + monkeypatch: pytest.MonkeyPatch, +) -> None: + def search_and_contents(_self: Exa, _query: str, **_kwargs: Any) -> Any: + return SimpleNamespace(results=[]) + + monkeypatch.setattr(Exa, "search_and_contents", search_and_contents) + provider, exporter = _tracing() + instrumentor = _instrument(provider) + try: + Exa(api_key=_API_KEY).search_and_contents("deprecated alias") + finally: + instrumentor.uninstrument() + + assert exporter.get_finished_spans()[0].name == "exa.search" + + +def test_answer_has_no_model_attribute(monkeypatch: pytest.MonkeyPatch) -> None: + def answer(_self: Exa, _query: str, **_kwargs: Any) -> Any: + return SimpleNamespace(answer="answer", citations=[]) + + monkeypatch.setattr(Exa, "answer", answer) + provider, exporter = _tracing() + instrumentor = _instrument(provider) + try: + Exa(api_key=_API_KEY).answer("answer query", model="exa") + finally: + instrumentor.uninstrument() + + spans = exporter.get_finished_spans() + assert spans[0].name == "exa.answer" + assert not any("model" in key for key in _attrs(exporter)) diff --git a/python/frameworks/exa/tests/test_packaging.py b/python/frameworks/exa/tests/test_packaging.py new file mode 100644 index 000000000..f0183fae1 --- /dev/null +++ b/python/frameworks/exa/tests/test_packaging.py @@ -0,0 +1,77 @@ +"""Packaging contract: dependency ranges and declared Python versions.""" + +from __future__ import annotations + +import re +from pathlib import Path + +from packaging.requirements import Requirement +from packaging.specifiers import SpecifierSet + +from traceai_exa import ExaInstrumentor +from traceai_exa.package import _instruments + +PYPROJECT = Path(__file__).resolve().parents[1] / "pyproject.toml" + + +def _dependency(name: str) -> SpecifierSet: + match = re.search( + r'^{0}\s*=\s*"([^"]+)"'.format(re.escape(name)), PYPROJECT.read_text(), re.M + ) + assert match, name + return SpecifierSet(match.group(1)) + + +def test_exa_py_is_a_2x_range_not_an_exact_pin(): + (requirement,) = ExaInstrumentor().instrumentation_dependencies() + instruments = Requirement(requirement) + assert instruments.name == "exa-py" + # pyproject and the runtime dependency check agree. + assert instruments.specifier == _dependency("exa-py") + for version in ("2.25.0", "2.25.1", "2.26.0", "2.99.0"): + assert instruments.specifier.contains(version), version + for version in ("2.24.0", "3.0.0", "3.1.0"): + assert not instruments.specifier.contains(version), version + assert _instruments == ExaInstrumentor().instrumentation_dependencies() + + +def test_fi_instrumentation_floor_is_1_1_0(): + # 0.1.11/0.1.12 export to the old create_otel_span path and the published + # 1.0.0 fails while building its headers; 1.1.0 is the first good release. + spec = _dependency("fi-instrumentation-otel") + assert spec.contains("1.1.0") + assert spec.contains("1.2.0") + for version in ("0.1.11", "0.1.12", "0.1.16", "1.0.0"): + assert not spec.contains(version), version + + +def test_classifiers_list_only_the_tested_python_versions(): + # The suite runs on 3.10, 3.11, 3.12 and 3.13 (PRD matrix 3.10/3.13, plus 3.11 and 3.12). + versions = re.findall( + r'"Programming Language :: Python :: (3\.\d+)"', PYPROJECT.read_text() + ) + assert versions == ["3.10", "3.11", "3.12", "3.13"] + + +def test_python_range_starts_at_the_lowest_tested_version(): + # fi-instrumentation-otel 1.1.0 requires Python >3.9, and 3.9 is not tested. + match = re.search(r'^python\s*=\s*"([^"]+)"', PYPROJECT.read_text(), re.M) + assert match + spec = SpecifierSet(match.group(1)) + assert not spec.contains("3.9.20") + for version in ("3.10.0", "3.11.9", "3.12.7", "3.13.1"): + assert spec.contains(version), version + assert not spec.contains("3.14.0") + + +def test_vendor_async_streams_have_aclose_only_from_2_26(): + # The README says exa-py's own async streams gained aclose() in 2.26.0 and have none + # before; a traced stream has aclose() either way (the stream tests call it). + from importlib.metadata import version + + from exa_py.api import AsyncStreamAnswerResponse, AsyncStreamSearchResponse + from packaging.version import Version + + vendor_has_aclose = Version(version("exa-py")) >= Version("2.26.0") + assert hasattr(AsyncStreamSearchResponse, "aclose") is vendor_has_aclose + assert hasattr(AsyncStreamAnswerResponse, "aclose") is vendor_has_aclose diff --git a/python/frameworks/exa/tests/test_privacy_config.py b/python/frameworks/exa/tests/test_privacy_config.py new file mode 100644 index 000000000..2afb242fe --- /dev/null +++ b/python/frameworks/exa/tests/test_privacy_config.py @@ -0,0 +1,398 @@ +"""TraceConfig and context: hidden inputs, FITracer attributes, and safe error text.""" + +from __future__ import annotations + +import asyncio + +import pytest +from opentelemetry import trace as trace_api +from opentelemetry.sdk.trace import TracerProvider + +pytest.importorskip("exa_py", reason="exa-py must be installed to test its instrumentor") + +from _support import ( # noqa: E402 + ECHO_PADDING, + ECHO_QUERY, + EXA_KEY, + FAIL_QUERY, + RESULT_TITLE, + FakeExa, + attrs, + instrumented, +) +from exa_py import AsyncExa, Exa # noqa: E402 +from fi_instrumentation import TraceConfig, using_session # noqa: E402 + +HIDDEN = "__REDACTED__" +PRIVATE = "private question" + + +@pytest.fixture() +def fake(): + with FakeExa() as server: + yield server + + +def _client(fake: FakeExa) -> Exa: + return Exa(api_key=EXA_KEY, base_url=fake.origin) + + +def _exception_event(span): + (event,) = [event for event in span.events if event.name == "exception"] + return dict(event.attributes) + + +# --- hide_inputs ------------------------------------------------------------ + + +def test_hide_inputs_config_hides_the_query_and_keeps_the_count(fake): + with instrumented(config=TraceConfig(hide_inputs=True)) as traced: + _client(fake).search(PRIVATE) + + span_attributes = attrs(traced.one()) + assert span_attributes["input.value"] == HIDDEN + assert "gen_ai.retrieval.query" not in span_attributes + assert span_attributes["exa.result_count"] == 2 + assert PRIVATE not in traced.wire() + # Hiding is on the span only; the vendor still got the caller's query. + assert fake.calls[0][2]["query"] == PRIVATE + + +def test_fi_hide_inputs_environment_variable_is_honoured(fake, monkeypatch): + monkeypatch.setenv("FI_HIDE_INPUTS", "true") + with instrumented() as traced: + _client(fake).answer(PRIVATE) + + span_attributes = attrs(traced.one()) + assert span_attributes["input.value"] == HIDDEN + assert "gen_ai.retrieval.query" not in span_attributes + assert PRIVATE not in traced.wire() + + +def test_hide_inputs_applies_to_streams(fake): + with instrumented(config=TraceConfig(hide_inputs=True)) as traced: + for _ in _client(fake).stream_search(PRIVATE): + pass + + span_attributes = attrs(traced.one()) + assert span_attributes["input.value"] == HIDDEN + assert "gen_ai.retrieval.query" not in span_attributes + assert PRIVATE not in traced.wire() + + +def test_hide_inputs_wins_over_capture_urls(fake): + urls = ["https://example.com/a?token=t-1", "https://example.com/b"] + with instrumented(capture_urls=True, config=TraceConfig(hide_inputs=True)) as traced: + _client(fake).get_contents(urls) + + span_attributes = attrs(traced.one()) + assert span_attributes["exa.url_count"] == 2 + assert "exa.urls" not in span_attributes + assert "example.com" not in traced.wire() + + +def test_config_must_be_a_trace_config(): + from traceai_exa import ExaInstrumentor + + instrumentor = ExaInstrumentor() + try: + with pytest.raises(TypeError, match="TraceConfig"): + instrumentor.instrument( + tracer_provider=TracerProvider(), config={"hide_inputs": True} + ) + # Nothing was wrapped, so a later instrument() works normally. + assert not getattr(Exa.search, "__wrapped__", None) + finally: + instrumentor.uninstrument() + + +# --- FITracer context attributes ------------------------------------------- + + +def test_context_attributes_reach_the_exa_span(fake): + with instrumented() as traced: + with using_session("session-1"): + _client(fake).search("context") + + assert attrs(traced.one())["session.id"] == "session-1" + + +# --- error text -------------------------------------------------------------- + + +def _echo_error_cases(): + def sync_search(client_fake, query): + _client(client_fake).search(query) + + def sync_stream(client_fake, query): + for _ in _client(client_fake).stream_search(query): + pass + + def async_search(client_fake, query): + async def call() -> None: + client = AsyncExa(api_key=EXA_KEY, api_base=client_fake.origin) + try: + await client.search(query) + finally: + await client.client.aclose() + + asyncio.run(call()) + + return [sync_search, sync_stream, async_search] + + +@pytest.mark.parametrize("call", _echo_error_cases(), ids=["search", "stream", "async"]) +def test_error_text_has_the_key_redacted_and_is_capped(fake, call): + query = "{0} find {1}".format(ECHO_QUERY, EXA_KEY) + with instrumented() as traced: + with pytest.raises(ValueError) as caught: + call(fake, query) + + # The caller's exception is unchanged: it still carries the server's text. + assert EXA_KEY in str(caught.value) + assert ECHO_PADDING in str(caught.value) + + span = traced.one() + assert EXA_KEY not in traced.wire() + description = span.status.description + assert description.startswith("ValueError: Request failed with status code 400") + assert "[redacted]" in description + assert len(description.encode("utf-8")) <= len("ValueError: ") + 1024 + + event = _exception_event(span) + assert event["exception.type"] == "ValueError" + assert "[redacted]" in event["exception.message"] + assert len(event["exception.message"].encode("utf-8")) <= 1024 + assert "exception.stacktrace" not in event + + +@pytest.mark.parametrize("call", _echo_error_cases(), ids=["search", "stream", "async"]) +def test_hidden_inputs_also_leave_error_text(fake, call): + query = "{0} {1}".format(ECHO_QUERY, PRIVATE) + with instrumented(config=TraceConfig(hide_inputs=True)) as traced: + with pytest.raises(ValueError) as caught: + call(fake, query) + + assert PRIVATE in str(caught.value) + span = traced.one() + assert PRIVATE not in traced.wire() + assert HIDDEN in span.status.description + assert HIDDEN in _exception_event(span)["exception.message"] + + +def test_short_error_text_is_kept_whole(fake): + with instrumented() as traced: + with pytest.raises(ValueError) as caught: + _client(fake).search(FAIL_QUERY) + + span = traced.one() + assert span.status.description == "ValueError: {0}".format(caught.value) + assert _exception_event(span)["exception.message"] == str(caught.value) + + +def test_context_is_restored_after_a_vendor_error(fake): + with instrumented(): + with pytest.raises(ValueError): + _client(fake).search(FAIL_QUERY) + assert not trace_api.get_current_span().get_span_context().is_valid + + +# --- streams that fail mid-iteration ---------------------------------------- + + +class _Chunk: + citations = None + + +class _BrokenStream: + """Yields one chunk, then fails as a dropped connection would.""" + + def __init__(self) -> None: + self.closed = False + + def __iter__(self): + yield _Chunk() + raise ConnectionError("connection dropped") + + async def _agen(self): + yield _Chunk() + raise ConnectionError("connection dropped") + + def __aiter__(self): + return self._agen() + + def close(self) -> None: + self.closed = True + + +def _tracer(traced): + from fi_instrumentation import FITracer + + return FITracer(traced.provider.get_tracer("test"), config=TraceConfig()) + + +def test_sync_stream_failing_mid_iteration_is_an_error_not_a_cancel(): + from traceai_exa._wrappers import StreamWrapper + + with instrumented() as traced: + wrapper = StreamWrapper(_tracer(traced), "exa.search") + stream = wrapper(lambda *a, **k: _BrokenStream(), None, ("q",), {}) + next(stream) + with pytest.raises(ConnectionError): + next(stream) + + span = traced.one() + assert span.status.description == "ConnectionError: connection dropped" + assert "exa.cancelled" not in attrs(span) + assert "exa.result_count" not in attrs(span) + + +def test_async_stream_failing_mid_iteration_is_an_error_not_a_cancel(): + from traceai_exa._wrappers import AsyncStreamWrapper + + async def run(traced) -> None: + wrapper = AsyncStreamWrapper(_tracer(traced), "exa.search") + stream = await wrapper(lambda *a, **k: _BrokenStream(), None, ("q",), {}) + await stream.__anext__() + with pytest.raises(ConnectionError): + await stream.__anext__() + + with instrumented() as traced: + asyncio.run(run(traced)) + + span = traced.one() + assert span.status.description == "ConnectionError: connection dropped" + assert "exa.cancelled" not in attrs(span) + + +# --- get_contents with Result objects --------------------------------------- + + +def test_capture_urls_reads_only_the_url_of_result_objects(fake): + client = _client(fake) + results = client.search("seed").results + with instrumented(capture_urls=True) as traced: + client.get_contents(results) + + span_attributes = attrs(traced.one()) + assert span_attributes["exa.url_count"] == len(results) + assert list(span_attributes["exa.urls"]) == [result.url for result in results] + assert RESULT_TITLE not in traced.wire() + + +# --- get_contents error text (verify r3 V5) -------------------------------- + + +def test_hidden_inputs_remove_requested_urls_from_error_text(fake): + urls = [ + "https://example.com/{0}?token=t-1".format(ECHO_QUERY), + "https://example.com/b?key={0}".format(EXA_KEY), + ] + with instrumented(config=TraceConfig(hide_inputs=True)) as traced: + with pytest.raises(ValueError) as caught: + _client(fake).get_contents(urls) + + assert urls[0] in str(caught.value) + span = traced.one() + assert "example.com" not in traced.wire() + assert EXA_KEY not in traced.wire() + assert HIDDEN in span.status.description + assert attrs(span)["exa.url_count"] == 2 + + +def test_requested_urls_stay_in_error_text_by_default_with_the_key_redacted(fake): + urls = ["https://example.com/{0}?key={1}".format(ECHO_QUERY, EXA_KEY)] + with instrumented() as traced: + with pytest.raises(ValueError): + _client(fake).get_contents(urls) + + description = traced.one().status.description + assert "https://example.com/{0}?key=[redacted]".format(ECHO_QUERY) in description + assert EXA_KEY not in traced.wire() + + +# --- an exception whose str() raises (verify r3 V4) ------------------------- + + +class _Unprintable(Exception): + def __str__(self) -> str: + raise RuntimeError("str() failed") + + +def _raise_unprintable(*args, **kwargs): + raise _Unprintable() + + +def _assert_unprintable_span(traced) -> None: + span = traced.one() + assert span.end_time is not None + assert span.status.description == "_Unprintable" + event = _exception_event(span) + assert event["exception.type"].endswith("_Unprintable") + assert event["exception.message"] == "[redacted]" + assert "exception.stacktrace" not in event + + +def test_unprintable_error_is_reraised_unchanged_and_the_span_ends(): + from traceai_exa._wrappers import OperationWrapper + + with instrumented() as traced: + wrapper = OperationWrapper(_tracer(traced), "exa.search") + with pytest.raises(_Unprintable): + wrapper(_raise_unprintable, None, ("q",), {}) + + _assert_unprintable_span(traced) + + +def test_unprintable_async_error_is_reraised_unchanged_and_the_span_ends(): + from traceai_exa._wrappers import AsyncOperationWrapper + + async def wrapped(*args, **kwargs): + raise _Unprintable() + + async def run(traced) -> None: + wrapper = AsyncOperationWrapper(_tracer(traced), "exa.search") + with pytest.raises(_Unprintable): + await wrapper(wrapped, None, ("q",), {}) + + with instrumented() as traced: + asyncio.run(run(traced)) + + _assert_unprintable_span(traced) + + +def test_unprintable_mid_stream_error_ends_the_stream_span(): + from traceai_exa._wrappers import StreamWrapper + + class _UnprintableStream: + def __iter__(self): + yield _Chunk() + raise _Unprintable() + + def close(self) -> None: + return None + + with instrumented() as traced: + wrapper = StreamWrapper(_tracer(traced), "exa.search") + stream = wrapper(lambda *a, **k: _UnprintableStream(), None, ("q",), {}) + next(stream) + with pytest.raises(_Unprintable): + next(stream) + + _assert_unprintable_span(traced) + + +def test_exception_event_exports_no_stack_trace_or_local_path(fake): + """The event never carries frames: no local path, user name or host name.""" + import getpass + import socket + + with instrumented() as traced: + with pytest.raises(ValueError): + _client(fake).search(FAIL_QUERY) + span = traced.one() + wire = traced.wire() + event = _exception_event(span) + assert set(event) <= {"exception.type", "exception.message", "exception.escaped"} + for needle in ("Traceback", 'File "', "site-packages", getpass.getuser(), socket.gethostname()): + assert needle not in wire diff --git a/python/frameworks/exa/tests/test_query_attributes.py b/python/frameworks/exa/tests/test_query_attributes.py new file mode 100644 index 000000000..d4cb0db2d --- /dev/null +++ b/python/frameworks/exa/tests/test_query_attributes.py @@ -0,0 +1,176 @@ +"""Request attributes from the real SDK against the loopback fake.""" + +from __future__ import annotations + +import asyncio + +import pytest + +pytest.importorskip("exa_py", reason="exa-py must be installed to test its instrumentor") + +from _support import EXA_KEY, FakeExa, attrs, instrumented # noqa: E402 +from exa_py import AsyncExa, Exa # noqa: E402 + + +@pytest.fixture() +def fake(): + with FakeExa() as server: + yield server + + +def test_api_key_embedded_in_query_is_redacted(fake): + query = "find {0} please".format(EXA_KEY) + with instrumented() as traced: + Exa(api_key=EXA_KEY, base_url=fake.origin).search(query) + + assert attrs(traced.one())["gen_ai.retrieval.query"] == "find [redacted] please" + assert EXA_KEY not in traced.wire() + # Redaction is on the span only; the vendor still sent the caller's query. + assert fake.calls[0][2]["query"] == query + + +def test_api_key_embedded_in_async_query_is_redacted(fake): + async def call() -> None: + client = AsyncExa(api_key=EXA_KEY, api_base=fake.origin) + try: + await client.search("async {0}".format(EXA_KEY)) + finally: + await client.client.aclose() + + with instrumented() as traced: + asyncio.run(call()) + + assert attrs(traced.one())["gen_ai.retrieval.query"] == "async [redacted]" + assert EXA_KEY not in traced.wire() + + +@pytest.mark.parametrize( + "query", + [ + "a" * 2000, + "\u20ac" * 500, # 3 UTF-8 bytes each: 1024 is not a character boundary + "\U0001f50d" * 300, # 4 UTF-8 bytes each + "x" + "\u00e9" * 600, # 2-byte characters after an odd offset + ], + ids=["ascii", "3-byte", "4-byte", "2-byte-offset"], +) +def test_query_is_capped_at_1kb_of_utf8_on_a_character_boundary(fake, query): + with instrumented() as traced: + Exa(api_key=EXA_KEY, base_url=fake.origin).search(query) + + recorded = attrs(traced.one())["gen_ai.retrieval.query"] + assert len(recorded.encode("utf-8")) <= 1024 + # The longest whole-character prefix that fits: one more character would not. + assert recorded == query[: len(recorded)] + assert len(query[: len(recorded) + 1].encode("utf-8")) > 1024 + + +def test_input_value_carries_the_same_redacted_capped_query(fake): + query = "{0} {1}".format(EXA_KEY, "\u20ac" * 500) + with instrumented() as traced: + client = Exa(api_key=EXA_KEY, base_url=fake.origin) + client.search(query) + client.answer(query) + list(client.stream_search(query)) + + spans = traced.spans() + assert [span.name for span in spans] == ["exa.search", "exa.answer", "exa.search"] + for span in spans: + values = attrs(span) + assert values["input.value"] == values["gen_ai.retrieval.query"] + assert values["input.value"].startswith("[redacted] \u20ac") + assert len(values["input.value"].encode("utf-8")) <= 1024 + + +SECRET_URL = "https://x.example/a?token=URL-SECRET" + + +def test_get_contents_records_the_url_count_not_the_urls(fake): + with instrumented() as traced: + response = Exa(api_key=EXA_KEY, base_url=fake.origin).get_contents( + [SECRET_URL, "https://x.example/b"] + ) + + assert len(response.results) == 2 + span = traced.one() + values = attrs(span) + assert span.name == "exa.get_contents" + assert values["exa.url_count"] == 2 + assert values["exa.result_count"] == 2 + assert "exa.urls" not in values + assert "gen_ai.retrieval.query" not in values + assert "input.value" not in values + for secret in ("URL-SECRET", "x.example"): + assert secret not in traced.wire() + + +def test_get_contents_counts_every_input_shape(fake): + with instrumented() as traced: + client = Exa(api_key=EXA_KEY, base_url=fake.origin) + found = client.search("documents to fetch") # two Result objects + client.get_contents("https://x.example/one") + client.get_contents(urls=["https://x.example/1", "https://x.example/2", "https://x.example/3"]) + client.get_contents(found.results) + + counts = [ + attrs(span).get("exa.url_count") + for span in traced.spans() + if span.name == "exa.get_contents" + ] + assert counts == [1, 3, 2] + assert "x.example" not in traced.wire() + assert "https://example.com/" not in traced.wire() + + +def test_async_get_contents_records_the_url_count_not_the_urls(fake): + async def call() -> None: + client = AsyncExa(api_key=EXA_KEY, api_base=fake.origin) + try: + await client.get_contents([SECRET_URL]) + finally: + await client.client.aclose() + + with instrumented() as traced: + asyncio.run(call()) + + assert attrs(traced.one())["exa.url_count"] == 1 + assert "URL-SECRET" not in traced.wire() + + +def test_capture_urls_opt_in_records_at_most_20_redacted_capped_urls(fake): + urls = ["https://x.example/{0}".format(i) for i in range(25)] + urls[0] = "https://x.example/k?key={0}".format(EXA_KEY) + urls[1] = "https://x.example/" + "\u20ac" * 500 + with instrumented(capture_urls=True) as traced: + Exa(api_key=EXA_KEY, base_url=fake.origin).get_contents(urls) + + values = attrs(traced.one()) + assert values["exa.url_count"] == 25 + captured = list(values["exa.urls"]) + assert len(captured) == 20 + assert captured[0] == "https://x.example/k?key=[redacted]" + assert all(len(url.encode("utf-8")) <= 1024 for url in captured) + assert captured[1] == urls[1][: len(captured[1])] + assert captured[2:] == urls[2:20] + assert EXA_KEY not in traced.wire() + + +def test_capture_urls_does_not_add_urls_to_search_spans(fake): + with instrumented(capture_urls=True) as traced: + Exa(api_key=EXA_KEY, base_url=fake.origin).search("plain search") + + assert "exa.urls" not in attrs(traced.one()) + + +def test_missing_api_key_raises_in_the_constructor_before_any_span(monkeypatch): + # PRD J5 expected an ERROR span for a missing key. exa-py raises in + # Exa()/AsyncExa() __init__, which is not a traced call, so there is no + # span to mark; the README documents this. + monkeypatch.delenv("EXA_API_KEY", raising=False) + with instrumented() as traced: + with pytest.raises(ValueError, match="API key"): + Exa() + with pytest.raises(ValueError, match="API key"): + AsyncExa() + + assert traced.spans() == [] diff --git a/python/frameworks/exa/tests/test_streams.py b/python/frameworks/exa/tests/test_streams.py new file mode 100644 index 000000000..6b6501c23 --- /dev/null +++ b/python/frameworks/exa/tests/test_streams.py @@ -0,0 +1,348 @@ +"""Stream lifecycle from the real SDK against the loopback fake's SSE routes.""" + +from __future__ import annotations + +import asyncio +import gc +import logging + +import pytest +from opentelemetry.trace import StatusCode + +pytest.importorskip("exa_py", reason="exa-py must be installed to test its instrumentor") + +from _support import ( # noqa: E402 + CONTENT_MARKERS, + EXA_KEY, + FAIL_QUERY, + SLOW_QUERY, + STREAM_CHUNKS, + STREAM_CITATIONS, + FakeExa, + attrs, + instrumented, +) +from exa_py import AsyncExa, Exa # noqa: E402 +from exa_py.api import ( # noqa: E402 + AsyncStreamAnswerResponse, + AsyncStreamSearchResponse, + StreamAnswerResponse, + StreamSearchResponse, +) + + +@pytest.fixture() +def fake(): + with FakeExa() as server: + yield server + + +def test_sync_streams_keep_the_vendor_type(fake): + with instrumented(): + client = Exa(api_key=EXA_KEY, base_url=fake.origin) + search = client.stream_search("q") + answer = client.stream_answer("q") + assert isinstance(search, StreamSearchResponse) + assert isinstance(answer, StreamAnswerResponse) + list(search) + list(answer) + + +def test_async_streams_keep_the_vendor_type(fake): + async def call() -> None: + client = AsyncExa(api_key=EXA_KEY, api_base=fake.origin) + try: + search = await client.stream_search("q") + answer = await client.stream_answer("q") + assert isinstance(search, AsyncStreamSearchResponse) + assert isinstance(answer, AsyncStreamAnswerResponse) + async for _ in search: + pass + async for _ in answer: + pass + finally: + await client.client.aclose() + + with instrumented(): + asyncio.run(call()) + + +def _assert_cancelled(span) -> None: + values = attrs(span) + assert span.status.status_code is StatusCode.ERROR + assert span.status.description == "cancelled" + assert values["exa.cancelled"] is True + # A cancelled stream's total is unknown, and cancelling is not an exception. + assert "exa.result_count" not in values + assert not [event for event in span.events if event.name == "exception"] + + +async def _until(predicate) -> None: + for _ in range(500): + if predicate(): + return + await asyncio.sleep(0.01) + raise AssertionError("condition not reached") + + +def test_sync_stream_closed_after_one_chunk_is_cancelled(fake): + with instrumented() as traced: + stream = Exa(api_key=EXA_KEY, base_url=fake.origin).stream_search("q") + next(iter(stream)) + stream.close() + _assert_cancelled(traced.one()) + + +def test_sync_stream_dropped_mid_iteration_is_cancelled_on_gc(fake): + with instrumented() as traced: + stream = Exa(api_key=EXA_KEY, base_url=fake.origin).stream_answer("q") + next(iter(stream)) + assert traced.spans() == [] + del stream + gc.collect() + _assert_cancelled(traced.one()) + + +def test_async_stream_aclosed_after_one_chunk_is_cancelled(fake): + async def call(traced) -> None: + client = AsyncExa(api_key=EXA_KEY, api_base=fake.origin) + try: + stream = await client.stream_answer("q") + await stream.__anext__() + await stream.aclose() + _assert_cancelled(traced.one()) + finally: + await client.client.aclose() + + with instrumented() as traced: + asyncio.run(call(traced)) + + +def test_async_stream_dropped_mid_iteration_is_cancelled_on_gc(fake): + async def call(traced) -> None: + client = AsyncExa(api_key=EXA_KEY, api_base=fake.origin) + try: + stream = await client.stream_search("q") + await stream.__anext__() + raw = stream._raw_response + del stream + gc.collect() + _assert_cancelled(traced.one()) + await raw.aclose() + finally: + await client.client.aclose() + + with instrumented() as traced: + asyncio.run(call(traced)) + + +def test_cancelled_async_call_is_marked_cancelled(fake): + async def call() -> None: + client = AsyncExa(api_key=EXA_KEY, api_base=fake.origin) + try: + task = asyncio.ensure_future(client.search(SLOW_QUERY)) + await _until(lambda: fake.calls) + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + finally: + await client.client.aclose() + + with instrumented() as traced: + asyncio.run(call()) + + _assert_cancelled(traced.one()) + + +def test_async_stream_cancelled_mid_chunk_is_marked_cancelled(fake): + async def call() -> None: + client = AsyncExa(api_key=EXA_KEY, api_base=fake.origin) + try: + stream = await client.stream_answer(SLOW_QUERY) + await stream.__anext__() # the fake now stalls before chunk two + task = asyncio.ensure_future(stream.__anext__()) + await asyncio.sleep(0.05) + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + await stream.aclose() # already ended: must not end it again + finally: + await client.client.aclose() + + with instrumented() as traced: + asyncio.run(call()) + + _assert_cancelled(traced.one()) + + +async def _close_after_one_chunk(fake, after_close=lambda: None) -> type: + """Call close() on an async stream after one chunk; return what it raised. + + ``after_close`` runs while the stream is still referenced, so a span ended + there was ended by close() and not by garbage collection. + """ + client = AsyncExa(api_key=EXA_KEY, api_base=fake.origin) + try: + stream = await client.stream_search("q") + await stream.__aiter__().__anext__() + raised: type = type(None) + try: + stream.close() + except Exception as error: # exa-py 2.25.0 + httpx raise here; see below + raised = type(error) + after_close() + await stream._raw_response.aclose() + return raised + finally: + await client.client.aclose() + + +def test_async_stream_close_ends_the_span_now_and_still_delegates(fake): + # The vendor's close() calls httpx's sync close on an async response, + # which raises. Tracing must not change that outcome. + vendor_outcome = asyncio.run(_close_after_one_chunk(fake)) + + with instrumented() as traced: + outcome = asyncio.run( + _close_after_one_chunk(fake, after_close=lambda: _assert_cancelled(traced.one())) + ) + + assert outcome is vendor_outcome + + +def test_async_stream_aclose_releases_the_http_response(fake): + async def call(traced) -> None: + client = AsyncExa(api_key=EXA_KEY, api_base=fake.origin) + try: + stream = await client.stream_answer("q") + await stream.__anext__() + await stream.aclose() + _assert_cancelled(traced.one()) + assert stream._raw_response.is_closed + finally: + await client.client.aclose() + + with instrumented() as traced: + asyncio.run(call(traced)) + + +def test_sync_stream_close_ends_the_span_even_if_the_vendor_close_fails(fake, monkeypatch): + def broken_close(_self): + raise OSError("socket already gone") + + monkeypatch.setattr(StreamSearchResponse, "close", broken_close) + with instrumented() as traced: + stream = Exa(api_key=EXA_KEY, base_url=fake.origin).stream_search("q") + next(iter(stream)) + with pytest.raises(OSError, match="socket already gone"): + stream.close() + _assert_cancelled(traced.one()) + + +@pytest.fixture() +def double_end_warnings(caplog): + """The SDK logs, rather than raises, when an ended span is touched again.""" + caplog.set_level(logging.WARNING, logger="opentelemetry.sdk.trace") + return lambda: [r.getMessage() for r in caplog.records if "ended span" in r.getMessage()] + + +def _assert_complete(span) -> None: + values = attrs(span) + assert span.status.status_code is StatusCode.OK + assert values["exa.result_count"] == STREAM_CITATIONS + assert "exa.cancelled" not in values + assert not span.events + + +def test_sync_streams_complete_as_one_ok_span_and_are_ended_once(fake, double_end_warnings): + with instrumented() as traced: + client = Exa(api_key=EXA_KEY, base_url=fake.origin) + for method in (client.stream_search, client.stream_answer): + stream = method("q") + assert len(list(stream)) == STREAM_CHUNKS + stream.close() # after completion: must not re-end or flip to ERROR + del stream + gc.collect() + + assert [span.name for span in traced.spans()] == ["exa.search", "exa.answer"] + for span in traced.spans(): + _assert_complete(span) + for marker in CONTENT_MARKERS: + assert marker not in traced.wire() + assert double_end_warnings() == [] + + +def test_async_streams_complete_as_one_ok_span_and_are_ended_once(fake, double_end_warnings): + async def call() -> None: + client = AsyncExa(api_key=EXA_KEY, api_base=fake.origin) + try: + for method in (client.stream_search, client.stream_answer): + stream = await method("q") + chunks = [chunk async for chunk in stream] + assert len(chunks) == STREAM_CHUNKS + await stream.aclose() # after completion: no second end + del stream + gc.collect() + finally: + await client.client.aclose() + + with instrumented() as traced: + asyncio.run(call()) + + assert [span.name for span in traced.spans()] == ["exa.search", "exa.answer"] + for span in traced.spans(): + _assert_complete(span) + assert double_end_warnings() == [] + + +def test_cancelled_streams_are_ended_once(fake, double_end_warnings): + with instrumented() as traced: + stream = Exa(api_key=EXA_KEY, base_url=fake.origin).stream_search("q") + next(iter(stream)) + stream.close() + stream.close() + del stream + gc.collect() + + _assert_cancelled(traced.one()) + assert double_end_warnings() == [] + + +def _assert_failed(span) -> None: + assert span.status.status_code is StatusCode.ERROR + assert "401" in span.status.description + assert [event.name for event in span.events] == ["exception"] + assert "exa.cancelled" not in attrs(span) + assert "exa.result_count" not in attrs(span) + + +def test_sync_stream_non_200_is_an_error_and_reraised(fake, double_end_warnings): + with instrumented() as traced: + client = Exa(api_key=EXA_KEY, base_url=fake.origin) + for method in (client.stream_search, client.stream_answer): + with pytest.raises(ValueError, match="401"): + method(FAIL_QUERY) + + assert [span.name for span in traced.spans()] == ["exa.search", "exa.answer"] + for span in traced.spans(): + _assert_failed(span) + assert double_end_warnings() == [] + + +def test_async_stream_non_200_is_an_error_and_reraised(fake, double_end_warnings): + async def call() -> None: + client = AsyncExa(api_key=EXA_KEY, api_base=fake.origin) + try: + for method in (client.stream_search, client.stream_answer): + with pytest.raises(ValueError, match="401"): + await method(FAIL_QUERY) + finally: + await client.client.aclose() + + with instrumented() as traced: + asyncio.run(call()) + + assert [span.name for span in traced.spans()] == ["exa.search", "exa.answer"] + for span in traced.spans(): + _assert_failed(span) + assert double_end_warnings() == [] diff --git a/python/frameworks/exa/tests/test_uninstrument.py b/python/frameworks/exa/tests/test_uninstrument.py new file mode 100644 index 000000000..de3b8bf64 --- /dev/null +++ b/python/frameworks/exa/tests/test_uninstrument.py @@ -0,0 +1,75 @@ +"""uninstrument() restores every wrapped vendor method by identity.""" + +from __future__ import annotations + +import asyncio + +import pytest + +pytest.importorskip("exa_py", reason="exa-py must be installed to test its instrumentor") + +from _support import EXA_KEY, FakeExa, instrumented # noqa: E402 +from exa_py import api as exa_api # noqa: E402 + +METHODS = ( + "search", + "get_contents", + "answer", + "search_and_contents", + "stream_search", + "stream_answer", +) +CLIENTS = ("Exa", "AsyncExa") + + +def _class_methods(): + return { + (client, method): getattr(exa_api, client).__dict__[method] + for client in CLIENTS + for method in METHODS + } + + +def test_every_method_is_defined_on_its_own_class(): + # AsyncExa defines its own twins; nothing is reached only by inheritance, + # so wrapping both classes cannot double-wrap one function. + assert len(_class_methods()) == 12 + + +def test_instrument_wraps_and_uninstrument_restores_all_twelve_methods(): + originals = _class_methods() + + with instrumented(): + wrapped = _class_methods() + assert all(wrapped[key] is not originals[key] for key in originals) + assert all(hasattr(wrapped[key], "__wrapped__") for key in originals) + + restored = _class_methods() + assert all(restored[key] is originals[key] for key in originals), [ + key for key in originals if restored[key] is not originals[key] + ] + + +def test_no_spans_after_uninstrument(): + with FakeExa() as fake: + with instrumented() as traced: + pass + + client = exa_api.Exa(api_key=EXA_KEY, base_url=fake.origin) + client.search("q") + client.answer("q") + list(client.stream_search("q")) + + async def call() -> None: + async_client = exa_api.AsyncExa(api_key=EXA_KEY, api_base=fake.origin) + try: + await async_client.search("q") + async for _ in await async_client.stream_answer("q"): + pass + finally: + await async_client.client.aclose() + + asyncio.run(call()) + + assert len(fake.calls) == 5 + assert traced.spans() == [] diff --git a/python/frameworks/exa/traceai_exa/__init__.py b/python/frameworks/exa/traceai_exa/__init__.py new file mode 100644 index 000000000..02743a9cb --- /dev/null +++ b/python/frameworks/exa/traceai_exa/__init__.py @@ -0,0 +1,141 @@ +"""OpenTelemetry instrumentation for the Exa client.""" + +import atexit +import logging +from importlib import import_module +from typing import Any, Collection, Dict, Tuple + +from fi_instrumentation import FITracer, TraceConfig +from opentelemetry import trace as trace_api +from opentelemetry.instrumentation.instrumentor import BaseInstrumentor +from wrapt import wrap_function_wrapper + +from traceai_exa._wrappers import ( + AsyncOperationWrapper, + AsyncStreamWrapper, + OperationWrapper, + StreamWrapper, + end_open_streams, +) +from traceai_exa.package import _instruments +from traceai_exa.version import __version__ + +logger = logging.getLogger(__name__) +logger.addHandler(logging.NullHandler()) + +_MODULE = "exa_py.api" +_OPERATIONS = { + "search": "exa.search", + "get_contents": "exa.get_contents", + "answer": "exa.answer", + # The SDK keeps this deprecated alias for backwards compatibility. It is a + # search operation, so it deliberately shares the search span name. + "search_and_contents": "exa.search", +} +_STREAM_OPERATIONS = { + "stream_search": "exa.search", + "stream_answer": "exa.answer", +} + + +class ExaInstrumentor(BaseInstrumentor): + """Instrument Exa's search, contents, answer, and streaming APIs. + + ``instrument()`` accepts ``tracer_provider``, ``capture_urls`` and + ``config`` (a ``fi_instrumentation.TraceConfig``; by default one built + from the ``FI_*`` environment variables, such as ``FI_HIDE_INPUTS``). + Spans come from an ``FITracer``, so the config's masking and PII + redaction apply, and ``using_session`` / ``using_user`` / + ``using_metadata`` attributes reach the Exa spans. + """ + + __slots__ = ("_original_methods",) + + def instrumentation_dependencies(self) -> Collection[str]: + return _instruments + + def _instrument(self, **kwargs: Any) -> None: + config = kwargs.get("config") + if config is None: + config = TraceConfig() + elif not isinstance(config, TraceConfig): + raise TypeError( + "config must be a fi_instrumentation.TraceConfig, got {0}".format( + type(config).__name__ + ) + ) + tracer_provider = kwargs.get("tracer_provider") or trace_api.get_tracer_provider() + tracer = FITracer( + trace_api.get_tracer(__name__, __version__, tracer_provider), config=config + ) + # Off by default: get_contents records only a URL count. With + # capture_urls=True it also records up to 20 requested URLs, each + # with the Exa key redacted and capped at 1 KB. Hidden inputs win. + capture_urls = bool(kwargs.get("capture_urls", False)) + hide_inputs = bool(config.hide_inputs) + pii = bool(config.pii_redaction) + api_module = import_module(_MODULE) + self._original_methods: Dict[Tuple[str, str], Any] = {} + + for client_name in ("Exa", "AsyncExa"): + is_async = client_name == "AsyncExa" + for method_name, span_name in _OPERATIONS.items(): + options = { + "contents": method_name == "get_contents", + "capture_urls": capture_urls, + "hide_inputs": hide_inputs, + "pii": pii, + } + wrapper = ( + AsyncOperationWrapper(tracer, span_name, **options) + if is_async + else OperationWrapper(tracer, span_name, **options) + ) + self._wrap_method(api_module, client_name, method_name, wrapper) + + for method_name, span_name in _STREAM_OPERATIONS.items(): + wrapper = ( + AsyncStreamWrapper(tracer, span_name, hide_inputs=hide_inputs, pii=pii) + if is_async + else StreamWrapper(tracer, span_name, hide_inputs=hide_inputs, pii=pii) + ) + self._wrap_method(api_module, client_name, method_name, wrapper) + + # register() adds an atexit hook that shuts the provider down. atexit + # runs hooks last-in first-out, so (re-)registering here makes open + # stream spans end before that shutdown, not after it. + atexit.unregister(end_open_streams) + atexit.register(end_open_streams) + logger.debug("Exa instrumentation enabled") + + def _wrap_method( + self, + api_module: Any, + client_name: str, + method_name: str, + wrapper: Any, + ) -> None: + client = getattr(api_module, client_name) + # Only a method defined on this class itself: AsyncExa subclasses Exa, + # and wrapping an inherited method would wrap Exa's (already wrapped) + # function a second time and leave it wrapped after uninstrument(). + original = vars(client).get(method_name) + if original is None: + return + + self._original_methods[(client_name, method_name)] = original + wrap_function_wrapper( + api_module, + "{0}.{1}".format(client_name, method_name), + wrapper, + ) + + def _uninstrument(self, **kwargs: Any) -> None: + api_module = import_module(_MODULE) + for (client_name, method_name), original in self._original_methods.items(): + setattr(getattr(api_module, client_name), method_name, original) + self._original_methods.clear() + logger.debug("Exa instrumentation disabled") + + +__all__ = ["ExaInstrumentor", "__version__"] diff --git a/python/frameworks/exa/traceai_exa/_wrappers.py b/python/frameworks/exa/traceai_exa/_wrappers.py new file mode 100644 index 000000000..6a3e64419 --- /dev/null +++ b/python/frameworks/exa/traceai_exa/_wrappers.py @@ -0,0 +1,552 @@ +"""wrapt wrappers for the supported Exa client methods.""" + +from __future__ import annotations + +import asyncio +import inspect +import weakref +from typing import Any, Callable, Mapping, Optional, Sequence, Tuple + +import wrapt +from fi_instrumentation import REDACTED_VALUE +from fi_instrumentation.instrumentation.pii_redaction import redact_pii_in_string +from opentelemetry import trace as trace_api +from opentelemetry.trace import Span, Status, StatusCode, Tracer + +_SPAN_KIND = "gen_ai.span.kind" +_INPUT_VALUE = "input.value" +_RETRIEVAL_QUERY = "gen_ai.retrieval.query" +_RESULT_COUNT = "exa.result_count" +_URL_COUNT = "exa.url_count" +_URLS = "exa.urls" +_CANCELLED = "exa.cancelled" +_RETRIEVER = "RETRIEVER" +# get_contents fetches given URLs; it is not a search. +_TOOL = "TOOL" +_REDACTED = "[redacted]" +_MAX_QUERY_BYTES = 1024 +_MAX_CAPTURED_URLS = 20 + + +def _redact(value: str, api_key: Optional[str]) -> str: + return value.replace(api_key, _REDACTED) if api_key else value + + +class _Scrub: + """What must not leave the process in error text for one call. + + ``api_key`` is replaced by ``[redacted]``; with hidden inputs, each + request input (the query, or get_contents' URLs; already key-redacted) + is replaced by ``__REDACTED__``, longest first; with ``pii_redaction``, + PII is replaced last. Only verbatim occurrences are found. + """ + + __slots__ = ("api_key", "hidden", "pii") + + def __init__( + self, api_key: Optional[str], hidden: Sequence[str] = (), pii: bool = False + ) -> None: + self.api_key = api_key + self.hidden = tuple(sorted({value for value in hidden if value}, key=len, reverse=True)) + self.pii = pii + + def __call__(self, text: str) -> str: + text = _redact(text, self.api_key) + for value in self.hidden: + text = text.replace(value, REDACTED_VALUE) + if self.pii: + text = redact_pii_in_string(text) + return text + + +def _query(args: Sequence[Any], kwargs: Mapping[str, Any]) -> Optional[str]: + """Return the search or answer query argument, never client configuration.""" + value = kwargs.get("query") + if value is None and args: + value = args[0] + return None if value is None else str(value) + + +def _requested_urls(args: Sequence[Any], kwargs: Mapping[str, Any]) -> Optional[list]: + """Return get_contents' requested URLs, or None when the shape is unknown. + + exa-py accepts one URL, a list of URLs, or a list of Result objects. Only a + Result's ``url`` is read, so text and highlights cannot reach a span. + """ + value = kwargs["urls"] if "urls" in kwargs else (args[0] if args else None) + if isinstance(value, str): + return [value] + if isinstance(value, (list, tuple)): + return [item if isinstance(item, str) else getattr(item, "url", None) for item in value] + return None + + +def _request_attributes( + instance: Any, + args: Sequence[Any], + kwargs: Mapping[str, Any], + contents: bool, + capture_urls: bool, + hide_inputs: bool = False, + pii: bool = False, +) -> Tuple[dict, _Scrub]: + attributes: dict = {_SPAN_KIND: _TOOL if contents else _RETRIEVER} + api_key = _api_key(instance) + if contents: + # get_contents: a URL count by default. URLs can carry tokens or + # personal data, so they are recorded only with capture_urls=True, + # and never when inputs are hidden. + urls = _requested_urls(args, kwargs) + if urls is None: + return attributes, _Scrub(api_key, pii=pii) + attributes[_URL_COUNT] = len(urls) + if hide_inputs: + # An error body can echo the URLs; hidden inputs keep them out + # of error text as well as attributes. + hidden = [_redact(url, api_key) for url in urls if isinstance(url, str)] + return attributes, _Scrub(api_key, hidden, pii) + if capture_urls: + attributes[_URLS] = [ + _cap(_redact(url, api_key)) + for url in urls[:_MAX_CAPTURED_URLS] + if isinstance(url, str) + ] + return attributes, _Scrub(api_key, pii=pii) + + query = _query(args, kwargs) + if query is None: + return attributes, _Scrub(api_key, pii=pii) + if hide_inputs: + # FITracer masks input.value too; gen_ai.retrieval.query is not one + # of its keys, so it is left out here. + attributes[_INPUT_VALUE] = REDACTED_VALUE + return attributes, _Scrub(api_key, [_redact(query, api_key)], pii) + query = _cap(_redact(query, api_key)) + attributes[_RETRIEVAL_QUERY] = query + # The backend input panel reads input.value; keep both keys. + attributes[_INPUT_VALUE] = query + return attributes, _Scrub(api_key, pii=pii) + + +def _cap(value: str, limit: int = _MAX_QUERY_BYTES) -> str: + """Return the longest whole-character prefix of at most ``limit`` UTF-8 bytes.""" + # A cut inside a multi-byte character leaves an invalid tail; "ignore" + # drops it. "replace" keeps a lone surrogate from raising in user code. + return value.encode("utf-8", "replace")[:limit].decode("utf-8", "ignore") + + +def _api_key(instance: Any) -> Optional[str]: + """Return the client's Exa key so it can be removed from span values. + + exa-py keeps the key only in ``headers["x-api-key"]``; ``api_key`` is a + fallback for clients that expose it as an attribute. + """ + key = None + headers = getattr(instance, "headers", None) + if isinstance(headers, Mapping): + key = headers.get("x-api-key") + if not key: + key = getattr(instance, "api_key", None) + return key if isinstance(key, str) and key else None + + +def _document_count(result: Any) -> Optional[int]: + """Count returned documents without inspecting their content. + + SearchResponse (search, get_contents) has ``results``; AnswerResponse has + ``citations``. Anything else is an unknown count: None, never 0. + """ + for name in ("results", "citations"): + documents = getattr(result, name, None) + if documents is not None: + try: + return len(documents) + except TypeError: + return None + return None + + +def _chunk_citations(chunk: Any) -> int: + """Citations carried by one StreamChunk (stream_search / stream_answer).""" + try: + return len(getattr(chunk, "citations", None) or ()) + except TypeError: + return 0 + + +class _BaseWrapper: + def __init__( + self, + tracer: Tracer, + span_name: str, + *, + contents: bool = False, + capture_urls: bool = False, + hide_inputs: bool = False, + pii: bool = False, + ) -> None: + self._tracer = tracer + self._span_name = span_name + self._contents = contents + self._capture_urls = capture_urls + self._hide_inputs = hide_inputs + self._pii = pii + + def _start_span( + self, instance: Any, args: Sequence[Any], kwargs: Mapping[str, Any] + ) -> Tuple[Optional[Span], Optional[_Scrub]]: + """Start the span; (None, None) runs the call untraced.""" + scrub: Optional[_Scrub] + try: + attributes, scrub = _request_attributes( + instance, + args, + kwargs, + self._contents, + self._capture_urls, + self._hide_inputs, + self._pii, + ) + except Exception: # an attribute must never break the user's call + # Unknown key and query: error text is withheld (see _error_text). + attributes = {_SPAN_KIND: _TOOL if self._contents else _RETRIEVER} + scrub = None + try: + return self._tracer.start_span(self._span_name, attributes=attributes), scrub + except Exception: # a tracer failure must never reach the caller + return None, None + + @staticmethod + def _finish_ok(span: Span, document_count: Optional[int]) -> None: + try: + if document_count is not None: + span.set_attribute(_RESULT_COUNT, document_count) + span.set_status(Status(StatusCode.OK)) + except Exception: + pass + finally: + span.end() + + @staticmethod + def _finish_error(span: Span, error: BaseException, scrub: Optional[_Scrub]) -> None: + # No document count: nothing was returned, so the count is unknown. + # The event is built here rather than by span.record_exception(), + # which calls str(error) unguarded; the span must end whatever the + # error's str() does, and the caller's exception must stay the same. + try: + description, event = _error_text(error, scrub) + event["exception.type"] = _exception_type(error) + event["exception.escaped"] = "False" + span.add_event("exception", attributes=event) + span.set_status(Status(StatusCode.ERROR, description)) + except Exception: + pass + finally: + span.end() + + @staticmethod + def _finish_cancelled(span: Span) -> None: + # Cancellation is not an exception: no event and no document count. + span.set_attribute(_CANCELLED, True) + span.set_status(Status(StatusCode.ERROR, "cancelled")) + span.end() + + +def _exception_type(error: BaseException) -> str: + """The exception type as the OpenTelemetry SDK's record_exception names it.""" + module = type(error).__module__ + qualname = type(error).__qualname__ + return "{0}.{1}".format(module, qualname) if module and module != "builtins" else qualname + + +def _error_text(error: BaseException, scrub: Optional[_Scrub]) -> Tuple[str, dict]: + """The status description and exception-event text for ``error``. + + exa-py copies the server's response body into its exceptions, and a body + can repeat the request. The key (and, with hidden inputs, the query) is + removed and the message is capped at 1 KB; the caller's exception object + is not changed. No stack trace is exported: frames carry local file paths + (and so usernames and hostnames), and chained exceptions' text. + """ + name = type(error).__name__ + if scrub is None: + return name, {"exception.message": _REDACTED} + try: + message = _cap(scrub(str(error))) + except Exception: + return name, {"exception.message": _REDACTED} + description = "{0}: {1}".format(name, message) if message else name + return description, {"exception.message": message} + + +def _current(span: Span) -> Any: + """Make ``span`` current for the vendor call so HTTP client spans nest under it. + + The span is ended by the wrapper, not on exit, and errors are recorded once + by ``_finish_error``. + """ + return trace_api.use_span( + span, end_on_exit=False, record_exception=False, set_status_on_exception=False + ) + + +class OperationWrapper(_BaseWrapper): + """Trace a synchronous Exa operation.""" + + def __call__( + self, + wrapped: Callable[..., Any], + instance: Any, + args: tuple, + kwargs: Mapping[str, Any], + ) -> Any: + span, scrub = self._start_span(instance, args, kwargs) + if span is None: + return wrapped(*args, **kwargs) + try: + with _current(span): + result = wrapped(*args, **kwargs) + except BaseException as error: + self._finish_error(span, error, scrub) + raise + self._finish_ok(span, _document_count(result)) + return result + + +class AsyncOperationWrapper(_BaseWrapper): + """Trace an asynchronous Exa operation.""" + + async def __call__( + self, + wrapped: Callable[..., Any], + instance: Any, + args: tuple, + kwargs: Mapping[str, Any], + ) -> Any: + span, scrub = self._start_span(instance, args, kwargs) + if span is None: + return await wrapped(*args, **kwargs) + try: + with _current(span): + result = await wrapped(*args, **kwargs) + except asyncio.CancelledError: + self._finish_cancelled(span) + raise + except BaseException as error: + self._finish_error(span, error, scrub) + raise + self._finish_ok(span, _document_count(result)) + return result + + +# Stream spans not yet ended. A stream still referenced at interpreter exit +# is finalised after atexit hooks, when the exporter is already shut down, so +# end_open_streams() ends these first (see ExaInstrumentor._instrument). +_OPEN_STREAMS: "weakref.WeakSet[_StreamState]" = weakref.WeakSet() + + +def end_open_streams() -> None: + """End every still-open stream span as cancelled; registered with atexit.""" + for state in list(_OPEN_STREAMS): + try: + state.cancelled() + except Exception: # exit must not fail because of tracing + pass + + +class _StreamState: + """Ends one stream span exactly once; counts citations as chunks arrive.""" + + def __init__(self, span: Span, scrub: Optional[_Scrub]) -> None: + self.span = span + self.scrub = scrub + self.finished = False + # Citations accumulate over chunks; the total is known only at the end. + self.citations = 0 + _OPEN_STREAMS.add(self) + + def observe(self, chunk: Any) -> None: + self.citations += _chunk_citations(chunk) + + def _claim(self) -> bool: + if self.finished: + return False + self.finished = True + _OPEN_STREAMS.discard(self) + return True + + def ok(self) -> None: + if self._claim(): + _BaseWrapper._finish_ok(self.span, self.citations) + + def error(self, error: BaseException) -> None: + if self._claim(): + _BaseWrapper._finish_error(self.span, error, self.scrub) + + def cancelled(self) -> None: + if self._claim(): + _BaseWrapper._finish_cancelled(self.span) + + +def _state_of(proxy: Any) -> Optional[_StreamState]: + # A proxy whose __init__ failed has no state; __del__ must not raise. + try: + return proxy._self_state + except Exception: + return None + + +class _TracedStream(wrapt.ObjectProxy): # type: ignore[misc] + """A StreamSearchResponse/StreamAnswerResponse that ends its span. + + An ObjectProxy, so ``isinstance`` against the vendor class still holds. + """ + + def __init__(self, response: Any, span: Span, scrub: Optional[_Scrub]) -> None: + super().__init__(response) + self._self_state = _StreamState(span, scrub) + self._self_iterator = None + + def __iter__(self) -> "_TracedStream": + return self + + def __next__(self) -> Any: + state = self._self_state + try: + if self._self_iterator is None: + self._self_iterator = iter(self.__wrapped__) + chunk = next(self._self_iterator) + except StopIteration: + state.ok() + raise + except BaseException as error: + state.error(error) + raise + state.observe(chunk) + return chunk + + def close(self) -> None: + """End the span as cancelled (no-op after completion), then close.""" + self._self_state.cancelled() + try: + close = getattr(self._self_iterator, "close", None) + if close is not None: + close() + finally: + self.__wrapped__.close() + + def __del__(self) -> None: + state = _state_of(self) + if state is not None: + state.cancelled() + + +class StreamWrapper(_BaseWrapper): + """Trace a synchronous Exa stream for its full iterator lifetime.""" + + def __call__( + self, + wrapped: Callable[..., Any], + instance: Any, + args: tuple, + kwargs: Mapping[str, Any], + ) -> Any: + span, scrub = self._start_span(instance, args, kwargs) + if span is None: + return wrapped(*args, **kwargs) + try: + with _current(span): + response = wrapped(*args, **kwargs) + except BaseException as error: + self._finish_error(span, error, scrub) + raise + return _TracedStream(response, span, scrub) + + +class _TracedAsyncStream(wrapt.ObjectProxy): # type: ignore[misc] + """An AsyncStreamSearchResponse/AsyncStreamAnswerResponse that ends its span.""" + + def __init__(self, response: Any, span: Span, scrub: Optional[_Scrub]) -> None: + super().__init__(response) + self._self_state = _StreamState(span, scrub) + self._self_iterator = None + + def __aiter__(self) -> "_TracedAsyncStream": + return self + + async def __anext__(self) -> Any: + state = self._self_state + try: + if self._self_iterator is None: + self._self_iterator = self.__wrapped__.__aiter__() + chunk = await self._self_iterator.__anext__() + except StopAsyncIteration: + state.ok() + raise + except asyncio.CancelledError: + state.cancelled() + raise + except BaseException as error: + state.error(error) + raise + state.observe(chunk) + return chunk + + def close(self) -> None: + """End the span as cancelled, then delegate to the vendor's close(). + + exa-py's async close() raises RuntimeError (2.25.0: httpx's sync close on an + async response; 2.26.0: a pointer to aclose()); that outcome is the vendor's + and is not changed. + """ + self._self_state.cancelled() + self.__wrapped__.close() + + async def aclose(self) -> None: + """End the span as cancelled, then release the HTTP response.""" + self._self_state.cancelled() + try: + close = getattr(self._self_iterator, "aclose", None) + if close is not None: + await close() + finally: + # exa-py 2.26.0 has aclose(); 2.25.0 has none, but its httpx response does. + response_close = getattr(self.__wrapped__, "aclose", None) + if response_close is None: + raw_response = getattr(self.__wrapped__, "_raw_response", None) + response_close = getattr(raw_response, "aclose", None) + if response_close is not None: + await response_close() + + def __del__(self) -> None: + state = _state_of(self) + if state is not None: + state.cancelled() + + +class AsyncStreamWrapper(_BaseWrapper): + """Trace an asynchronous Exa stream for its full iterator lifetime.""" + + async def __call__( + self, + wrapped: Callable[..., Any], + instance: Any, + args: tuple, + kwargs: Mapping[str, Any], + ) -> Any: + span, scrub = self._start_span(instance, args, kwargs) + if span is None: + response = wrapped(*args, **kwargs) + return await response if inspect.isawaitable(response) else response + try: + with _current(span): + response = wrapped(*args, **kwargs) + if inspect.isawaitable(response): + response = await response + except asyncio.CancelledError: + self._finish_cancelled(span) + raise + except BaseException as error: + self._finish_error(span, error, scrub) + raise + return _TracedAsyncStream(response, span, scrub) diff --git a/python/frameworks/exa/traceai_exa/package.py b/python/frameworks/exa/traceai_exa/package.py new file mode 100644 index 000000000..b5cc88302 --- /dev/null +++ b/python/frameworks/exa/traceai_exa/package.py @@ -0,0 +1,2 @@ +# Tested against exa-py 2.25.0; any later 2.x is accepted. +_instruments = ("exa-py >= 2.25.0, < 3",) diff --git a/python/frameworks/exa/traceai_exa/py.typed b/python/frameworks/exa/traceai_exa/py.typed new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/python/frameworks/exa/traceai_exa/py.typed @@ -0,0 +1 @@ + diff --git a/python/frameworks/exa/traceai_exa/version.py b/python/frameworks/exa/traceai_exa/version.py new file mode 100644 index 000000000..3dc1f76bc --- /dev/null +++ b/python/frameworks/exa/traceai_exa/version.py @@ -0,0 +1 @@ +__version__ = "0.1.0" diff --git a/python/tests/harness/__init__.py b/python/tests/harness/__init__.py new file mode 100644 index 000000000..cf79047f1 --- /dev/null +++ b/python/tests/harness/__init__.py @@ -0,0 +1,238 @@ +"""Small, loopback-only utilities for OTLP contract tests.""" + +from __future__ import annotations + +import copy +import difflib +import json +import subprocess +import threading +from http import HTTPStatus +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from typing import Any, Mapping, Optional, Sequence, Union +from urllib.parse import urlsplit +from urllib.request import Request, urlopen + + +class Receiver: + """An in-memory OTLP/HTTP receiver for tests. + + The receiver only listens on loopback and never forwards received spans. + """ + + def __init__(self) -> None: + self._spans: list[dict[str, Any]] = [] + self._requests: list[dict[str, Any]] = [] + self._lock = threading.Lock() + owner = self + + class Handler(BaseHTTPRequestHandler): + def _read_body(self) -> bytes: + # The Node OTLP exporter streams with Transfer-Encoding: chunked + # and sends no Content-Length, so both framings are accepted. + if "chunked" in self.headers.get("Transfer-Encoding", "").lower(): + chunks = [] + while True: + size_line = self.rfile.readline().split(b";", 1)[0].strip() + size = int(size_line, 16) + if size == 0: + while self.rfile.readline() not in (b"\r\n", b"\n", b""): + pass + return b"".join(chunks) + chunks.append(self.rfile.read(size)) + self.rfile.readline() + length = int(self.headers.get("Content-Length", "0")) + return self.rfile.read(length) + + def do_POST(self) -> None: # noqa: N802 - required by BaseHTTPRequestHandler + path = urlsplit(self.path).path + if path not in ("/v1/traces", "/tracer/v1/traces"): + self.send_error(HTTPStatus.NOT_FOUND) + return + + try: + body = self._read_body() + content_type = self.headers.get("Content-Type", "").lower() + + if content_type.startswith("application/json"): + request = json.loads(body.decode("utf-8")) + elif content_type.startswith(("application/x-protobuf", "application/protobuf")): + from google.protobuf.json_format import MessageToDict + from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ( + ExportTraceServiceRequest, + ) + + protobuf_request = ExportTraceServiceRequest() + protobuf_request.ParseFromString(body) + request = MessageToDict(protobuf_request) + else: + self.send_error(HTTPStatus.UNSUPPORTED_MEDIA_TYPE) + return + + spans: list[dict[str, Any]] = [] + resource_attributes: list[dict[str, Any]] = [] + for resource_spans in request.get( + "resourceSpans", request.get("resource_spans", []) + ): + resource_attributes.append( + _flatten_attributes( + resource_spans.get("resource", {}).get("attributes", []) + ) + ) + for scope_spans in resource_spans.get( + "scopeSpans", resource_spans.get("scope_spans", []) + ): + spans.extend(scope_spans.get("spans", [])) + except (ImportError, TypeError, UnicodeDecodeError, ValueError) as error: + self.send_error(HTTPStatus.BAD_REQUEST, str(error)) + return + + with owner._lock: + owner._spans.extend(copy.deepcopy(spans)) + owner._requests.append( + { + "path": path, + "headers": { + key.lower(): value for key, value in self.headers.items() + }, + "resource_attributes": resource_attributes, + } + ) + self.send_response(HTTPStatus.OK) + self.send_header("Content-Length", "0") + self.end_headers() + + def log_message(self, _format: str, *args: Any) -> None: + return + + self._server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + self.origin = "http://127.0.0.1:{0}".format(self._server.server_port) + self.endpoint = "{0}/v1/traces".format(self.origin) + self.collector_endpoint = "{0}/tracer/v1/traces".format(self.origin) + self._thread = threading.Thread(target=self._server.serve_forever, daemon=True) + self._thread.start() + + def __enter__(self) -> "Receiver": + return self + + def __exit__(self, _type: Any, _value: Any, _traceback: Any) -> None: + self._server.shutdown() + self._server.server_close() + self._thread.join() + + def spans(self) -> list[dict[str, Any]]: + """Return a copy of the decoded spans received so far.""" + with self._lock: + return copy.deepcopy(self._spans) + + def requests(self) -> list[dict[str, Any]]: + """Return one record per accepted export: path, lower-cased headers, resource attributes.""" + with self._lock: + return copy.deepcopy(self._requests) + + def clear(self) -> None: + """Remove every decoded span and request record received so far.""" + with self._lock: + self._spans.clear() + self._requests.clear() + + +def _flatten_attributes(attributes: Sequence[Mapping[str, Any]]) -> dict[str, Any]: + """Turn OTLP key/value attributes into a plain dict of scalar values.""" + flat: dict[str, Any] = {} + for attribute in attributes: + value = attribute.get("value", {}) + for kind in ("stringValue", "boolValue", "intValue", "doubleValue"): + if kind in value: + flat[attribute.get("key", "")] = value[kind] + break + else: + flat[attribute.get("key", "")] = value + return flat + + +def run( + argv: Sequence[str], + env: Mapping[str, str], + stdin: Optional[bytes], + timeout: float, +) -> subprocess.CompletedProcess[bytes]: + """Run one process and return its captured output, including after a timeout.""" + process = subprocess.Popen( + argv, + env=dict(env), + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + timed_out = False + try: + stdout, stderr = process.communicate(input=stdin, timeout=timeout) + except subprocess.TimeoutExpired: + timed_out = True + process.kill() + stdout, stderr = process.communicate() + + result = subprocess.CompletedProcess(argv, process.returncode, stdout, stderr) + result.timed_out = timed_out + return result + + +def post_otlp(spans: dict[str, Any], endpoint: str) -> int: + """Post one JSON-encoded OTLP request to a loopback receiver.""" + parsed = urlsplit(endpoint) + if parsed.scheme != "http" or parsed.hostname != "127.0.0.1": + raise ValueError("post_otlp only sends to a loopback HTTP endpoint") + + request = Request( + endpoint, + data=json.dumps(spans).encode("utf-8"), + headers={"Content-Type": "application/json"}, + method="POST", + ) + with urlopen(request, timeout=5) as response: + return response.status + + +def compare(actual: Sequence[dict[str, Any]], golden_path: Union[str, Path]) -> None: + """Fail with a diff unless span names, attributes, and statuses match a golden.""" + with Path(golden_path).open(encoding="utf-8") as golden_file: + expected = json.load(golden_file) + + def canonical(spans: Sequence[dict[str, Any]]) -> list[dict[str, Any]]: + normalized = [] + for span in spans: + attributes = span.get("attributes", []) + if isinstance(attributes, dict): + attributes = [ + {"key": key, "value": value} for key, value in attributes.items() + ] + normalized.append( + { + "name": span.get("name", ""), + "attributes": sorted( + attributes, + key=lambda attribute: json.dumps( + attribute, sort_keys=True, separators=(",", ":") + ), + ), + "status": span.get("status", {}), + } + ) + return normalized + + actual_normalized = canonical(actual) + expected_normalized = canonical(expected) + if actual_normalized != expected_normalized: + diff = "\n".join( + difflib.unified_diff( + json.dumps(expected_normalized, indent=2, sort_keys=True).splitlines(), + json.dumps(actual_normalized, indent=2, sort_keys=True).splitlines(), + fromfile=str(golden_path), + tofile="actual", + lineterm="", + ) + ) + print(diff) + raise AssertionError("spans do not match golden") diff --git a/python/tests/harness/test_harness.py b/python/tests/harness/test_harness.py new file mode 100644 index 000000000..bd61d5adc --- /dev/null +++ b/python/tests/harness/test_harness.py @@ -0,0 +1,189 @@ +"""Contract tests for the shared loopback-only harness.""" + +import json +import sys + +import pytest + +from tests.harness import Receiver, compare, post_otlp, run + + +def test_receiver_stores_posted_span_and_can_clear() -> None: + payload = { + "resourceSpans": [ + { + "scopeSpans": [ + { + "spans": [ + { + "traceId": "trace-id", + "spanId": "span-id", + "name": "harness.receiver", + "attributes": [ + {"key": "component", "value": {"stringValue": "test"}} + ], + "status": {"code": 1}, + } + ] + } + ] + } + ] + } + + with Receiver() as receiver: + assert post_otlp(payload, receiver.endpoint) == 200 + assert receiver.spans() == payload["resourceSpans"][0]["scopeSpans"][0]["spans"] + receiver.clear() + assert receiver.spans() == [] + + +def test_receiver_accepts_the_collector_path() -> None: + payload = { + "resourceSpans": [ + {"scopeSpans": [{"spans": [{"name": "collector.path"}]}]} + ] + } + + with Receiver() as receiver: + assert post_otlp(payload, receiver.collector_endpoint) == 200 + assert receiver.spans() == [{"name": "collector.path"}] + + +def _post_chunked(url: str, body: bytes, headers: dict) -> int: + """Send a body with Transfer-Encoding: chunked, as the Node OTLP exporter does.""" + import http.client + from urllib.parse import urlsplit + + parts = urlsplit(url) + connection = http.client.HTTPConnection(parts.hostname, parts.port, timeout=5) + try: + chunks = [body[: len(body) // 2], body[len(body) // 2 :]] + connection.request( + "POST", + parts.path, + body=iter(chunks), + headers={**headers, "Transfer-Encoding": "chunked"}, + encode_chunked=True, + ) + return connection.getresponse().status + finally: + connection.close() + + +def test_receiver_decodes_a_chunked_body() -> None: + payload = { + "resourceSpans": [ + {"scopeSpans": [{"spans": [{"name": "chunked.body"}]}]} + ] + } + + with Receiver() as receiver: + status = _post_chunked( + receiver.collector_endpoint, + json.dumps(payload).encode("utf-8"), + {"Content-Type": "application/json"}, + ) + assert status == 200 + assert receiver.spans() == [{"name": "chunked.body"}] + + +def test_receiver_records_path_headers_and_resource_attributes() -> None: + payload = { + "resourceSpans": [ + { + "resource": { + "attributes": [ + {"key": "project_name", "value": {"stringValue": "harness"}}, + {"key": "project_type", "value": {"stringValue": "observe"}}, + ] + }, + "scopeSpans": [{"spans": [{"name": "with.resource"}]}], + } + ] + } + + with Receiver() as receiver: + status = _post_chunked( + receiver.collector_endpoint, + json.dumps(payload).encode("utf-8"), + { + "Content-Type": "application/json", + "X-Api-Key": "test-key", + "X-Secret-Key": "test-secret", + }, + ) + assert status == 200 + [request] = receiver.requests() + assert request["path"] == "/tracer/v1/traces" + assert request["headers"]["x-api-key"] == "test-key" + assert request["headers"]["x-secret-key"] == "test-secret" + assert "authorization" not in request["headers"] + assert request["resource_attributes"] == [ + {"project_name": "harness", "project_type": "observe"} + ] + receiver.clear() + assert receiver.requests() == [] + + +def test_run_captures_output_and_kills_timed_out_process() -> None: + completed = run( + [ + sys.executable, + "-c", + "import sys; print('stdout'); sys.stderr.write('stderr\\n')", + ], + env={}, + stdin=b"", + timeout=1, + ) + assert completed.returncode == 0 + assert completed.stdout == b"stdout\n" + assert completed.stderr == b"stderr\n" + assert completed.timed_out is False + + timed_out = run( + [sys.executable, "-c", "import time; time.sleep(30)"], + env={}, + stdin=b"", + timeout=0.1, + ) + assert timed_out.timed_out is True + assert timed_out.returncode != 0 + + +def test_compare_accepts_normalized_match_and_prints_mismatch_diff(tmp_path, capsys) -> None: + golden = [ + { + "traceId": "golden-trace", + "spanId": "golden-span", + "startTimeUnixNano": "1", + "endTimeUnixNano": "2", + "name": "harness.compare", + "attributes": [ + {"key": "alpha", "value": {"stringValue": "a"}}, + {"key": "beta", "value": {"intValue": "2"}}, + ], + "status": {"code": 1}, + } + ] + golden_path = tmp_path / "spans.golden.json" + golden_path.write_text(json.dumps(golden), encoding="utf-8") + + actual = [ + { + "traceId": "other-trace", + "spanId": "other-span", + "startTimeUnixNano": "3", + "endTimeUnixNano": "4", + "name": "harness.compare", + "attributes": list(reversed(golden[0]["attributes"])), + "status": {"code": 1}, + } + ] + compare(actual, golden_path) + + actual[0]["name"] = "harness.mismatch" + with pytest.raises(AssertionError, match="spans do not match golden"): + compare(actual, golden_path) + assert "---" in capsys.readouterr().out