diff --git a/python/frameworks/tavily/CHANGELOG.md b/python/frameworks/tavily/CHANGELOG.md new file mode 100644 index 000000000..10476fc16 --- /dev/null +++ b/python/frameworks/tavily/CHANGELOG.md @@ -0,0 +1,33 @@ +# Changelog + +All notable changes to `traceAI-tavily` will be documented here. + +## [0.1.0] - Unreleased + +### Fixed +- Plain `pytest` in the package directory now collects every test, as for the other traceAI Python + packages: the three modules that use the shared harness put `python/tests` on `sys.path` + themselves instead of relying on a `PYTHONPATH` (collection failed with + `No module named 'harness'`). +- The exception event no longer carries the raw traceback (local file paths, username, code frames): + `exception.stacktrace` is now the frames-free `: ` summary, with the key (and a hidden + query) removed. Tests scan the span events in default and hidden-input modes. + +### Verified +- Live 2026-10-17 against api.tavily.com (tavily-python 0.8.5): search, extract, keyless search and a bad-key + error exported to a local Future AGI; stored rows scanned. `langchain-tavily` 0.2.18 `TavilySearch` run: + not traced by this package. +- Re-verified 2026-10-16 against tavily-python 0.8.4 and 0.8.5 (latest): full suite, plus an end-to-end run + into a local Future AGI deployment. 0.8.5 renames `search`'s `max_hours` to `max_age_hours`; a test covers both. + +### Added +- `TavilyInstrumentor` for `TavilyClient` and `AsyncTavilyClient` (`tavily-python>=0.8.4,<1`): one span per + `search` (`tavily.search`, `gen_ai.span.kind=RETRIEVER`) and `extract` (`tavily.extract`, `TOOL`). +- The key-redacted query (capped at 1024 UTF-8 bytes) in `input.value`, and for `search` also in + `gen_ai.retrieval.query`; URL, result and failed-result counts. URLs and response content are never recorded. +- The API key is removed from every recorded text wherever the client keeps it. +- `TraceConfig` support: `hide_inputs` also removes the query from error text; `pii_redaction` also + covers the error status and the exception event. +- Errors set ERROR with the original exception re-raised; cancelled async calls end ERROR `cancelled`. +- Measurement tests showing the bare client emits nothing on its own and the LangChain tool path is + already traced by `traceAI-langchain`. diff --git a/python/frameworks/tavily/README.md b/python/frameworks/tavily/README.md new file mode 100644 index 000000000..fea6c8525 --- /dev/null +++ b/python/frameworks/tavily/README.md @@ -0,0 +1,298 @@ +# traceAI-tavily + +OpenTelemetry instrumentation for the bare [`tavily-python`](https://github.com/tavily-ai/tavily-python) +client: `TavilyClient` and `AsyncTavilyClient`, `search` and `extract`. + +This package wraps Tavily client calls. It is not a Tavily exporter; Tavily does +not publish one. It exists because a measurement showed the bare client emits +nothing on its own (see [What Tavily already emits](#what-tavily-already-emits)). +Alpha: validated against a local test server and a local Future AGI deployment, and in a live run against api.tavily.com on 2026-10-17 (see [Live vendor verification](#live-vendor-verification-2026-10-17)). + +## Installation + +```bash +pip install traceAI-tavily +``` + +It accepts `tavily-python>=0.8.4,<1` and `fi-instrumentation-otel>=1.1.0`, and is +tested with `tavily-python` 0.8.4 and 0.8.5 (the latest release on 2026-10-16; +0.8.5 only renames the `max_hours` argument of `search` to `max_age_hours`; this +package reads the query by name, so it is unaffected) on Python 3.10, 3.11, 3.12 and 3.13. The 0.8.5 test run and the +local Future AGI end-to-end run were on Python 3.12. + +## Which path do you use? + +Tavily reaches an application in two ways. They are different integrations. + +### LangChain tool: use traceAI-langchain, not this package + +`langchain_community.tools.tavily_search.TavilySearchResults` (the tool in the +LangGraph examples under `python/frameworks/langchain/examples/`) is already +traced by `traceAI-langchain`. Measured with a LangGraph `ToolNode` running it: +one span per tool call, `gen_ai.span.kind` = `TOOL`, named +`tavily_search_results_json`, with `gen_ai.tool.name`, `input.value` (the query) +and `output.value` (the tool's result text). + +Installing this package as well adds no span to that path: +`TavilySearchResults` posts to the Tavily API with `requests` itself and never +calls `TavilyClient`. The current recommended tool, `langchain_tavily.TavilySearch` +(`langchain-tavily` 0.2.18; the Tavily docs mark the community tool as deprecated), +was run live on 2026-10-17 with this package instrumented: it returned results +and this package produced 0 spans, so it needs `traceAI-langchain` too. + +`traceAI-langchain` exports the result text by default. `FI_HIDE_OUTPUTS=true` +masks the tool span's `output.value`, but the text is then the next agent turn's +input and still leaves through that span's `input.value`. Only +`FI_HIDE_INPUTS=true` together with `FI_HIDE_OUTPUTS=true` kept it in the +process in the measurement. + +### Bare client: use this package + +```python +from fi_instrumentation import register +from fi_instrumentation.fi_types import ProjectType +from tavily import TavilyClient +from traceai_tavily import TavilyInstrumentor + +tracer_provider = register(project_name="my-agent", project_type=ProjectType.OBSERVE) +TavilyInstrumentor().instrument(tracer_provider=tracer_provider) + +client = TavilyClient(api_key="tvly-...") +client.search("What is OpenTelemetry?") +``` + +`examples/search.py` is the same script; the test suite runs it. + +### A LangChain tool you wrote over TavilyClient + +If your own LangChain tool calls `TavilyClient.search` and both instrumentors +are on, one tool call gives two spans: the LangChain tool span and a +`tavily.search` span. `tavily.search` is a child of whatever OpenTelemetry span +is current when the client is called, and starts its own trace only when no +span is current. `traceAI-langchain` does not make its tool span current, so +`tavily.search` is not a child of it. To make it one, make the tool span current +around the call: + +```python +from opentelemetry import trace +from traceai_langchain import get_current_span + +with trace.use_span( + get_current_span(), + end_on_exit=False, + record_exception=False, + set_status_on_exception=False, +): + response = client.search(query) +``` + +`record_exception=False, set_status_on_exception=False` leave the tool span's +error to `traceAI-langchain`, which records it once; without them a failing +call records the same exception twice on the tool span. The `tavily.search` +span records its own exception event either way. + +If you do not want the second span, do not instrument Tavily in that process. + +## What Tavily already emits + +`tests/test_measurement.py` runs both paths against a local Tavily fake and +exports through `register()` to a local collector receiver (tavily-python 0.8.4, +langchain-community 0.4.2, langchain-core 1.5.2, langgraph 1.2.2): + +| Path | Without traceAI-tavily | With traceAI-tavily | +| --- | --- | --- | +| `TavilyClient` / `AsyncTavilyClient` `search` + `extract`, `register()` only (also as the global provider), with or without `traceAI-langchain` | 0 spans for 4 calls | 1 span per call (`search` RETRIEVER, `extract` TOOL) | +| LangGraph `ToolNode` running `TavilySearchResults`, `traceAI-langchain` on | 1 TOOL span | 1 TOOL span (unchanged) | +| Your LangChain tool calling `TavilyClient.search` | 1 TOOL span | 2 spans: the TOOL span and a RETRIEVER `tavily.search` (see above) | + +tavily-python 0.8.4 contains no OpenTelemetry code. + +## Instrumented calls + +| Call | Span | +| --- | --- | +| `TavilyClient.search`, `AsyncTavilyClient.search` | `tavily.search` | +| `TavilyClient.extract`, `AsyncTavilyClient.extract` | `tavily.extract` | + +The deprecated `tavily.Client` subclass is covered through `TavilyClient`. +`crawl`, `map`, `research`, `get_research`, `feedback`, `get_search_context`, +`qna_search` and `get_company_info` are not traced. + +Each span is a child of the active span, or a root span when there is none. It +is current while tavily-python sends the request, so a span from an HTTP client +instrumentation nests under it. + +## Span attributes + +| Attribute | Calls | Value | +| --- | --- | --- | +| `gen_ai.span.kind` | all | `RETRIEVER` for `search`; `TOOL` for `extract`, which fetches the pages it is given. | +| `gen_ai.tool.name` | all | `tavily.search` or `tavily.extract` | +| `input.value` | `search`; `extract` when `query` is given | The query, with the client's Tavily key replaced by `[redacted]`, then cut to 1024 bytes of UTF-8 on a character boundary. | +| `gen_ai.retrieval.query` | `search` | The same text as `input.value`. Not recorded when inputs are hidden. | +| `tavily.url_count` | `extract` | Number of requested URLs (a single URL string counts as 1). | +| `tavily.result_count` | successful calls | Length of the response's `results`. | +| `tavily.failed_result_count` | successful `extract` | Length of the response's `failed_results`. | +| `tavily.cancelled` | cancelled `AsyncTavilyClient` calls | `true` | + +A count is absent when it is unknown (an error, a cancellation, or a response +whose list is missing or not a list); it is never written as 0 for unknown. An +`extract` with some failed URLs is still a successful call. + +The query is read by its parameter name, `query`, from the method's signature +(read once per method), so it does not depend on argument positions. If the +arguments do not fit the signature, no `input.value` is recorded; the call +still goes to tavily-python unchanged. + +Context attributes from `fi_instrumentation` (`using_session`, `using_user`, +...) are added to the span; `session.id` and `user.id` are tested. +`suppress_tracing()` records nothing. + +## Privacy + +- **Response content is never recorded**: no result titles, content, raw + content, answers, images or extracted pages. There is no setting that adds it. +- **URLs are never recorded**: `extract` records only how many URLs were + requested, because URLs can carry tokens or personal data. +- **The Tavily key is never recorded.** It is removed from every recorded text + (`input.value`, the error status and the exception event's message and + stacktrace) wherever tavily-python keeps it: `TavilyClient.api_key`, + `TavilyClient.headers["Authorization"]`, the requests session's + `Authorization` header (including a session you passed in), and the httpx + client's `Authorization` header for `AsyncTavilyClient` (which has no + `api_key` attribute). If one of those places cannot be read, no free text is + recorded for that call: no `input.value`, the exception type as the error + description, and `[redacted]` as the exception message. +- **Hiding queries**: `FI_HIDE_INPUTS=true`, or + `instrument(config=TraceConfig(hide_inputs=True))`, records `input.value` as + `__REDACTED__` and drops `gen_ai.retrieval.query`. It also removes the query text from error messages: after the + key is redacted, each verbatim occurrence of the query in the error status + description and the exception event's message and stacktrace becomes + `__REDACTED__`. Other error text is recorded as thrown, with the key + redacted. +- **PII**: `FI_PII_REDACTION=true`, or `TraceConfig(pii_redaction=True)`, + replaces emails, phone numbers and similar with tokens such as + `` in every attribute and, after the key (and a hidden query) + is removed, in the error status and the exception event. `config` must be a `fi_instrumentation.TraceConfig`; anything else + raises `TypeError`. + +## Errors and cancellation + +An exception from the call (tested with `InvalidAPIKeyError`, +`BadRequestError`, `TavilyKeylessLimitError`, `requests.HTTPError` and +`httpx.HTTPStatusError`) sets the span status to ERROR with +`: ` (key redacted, the query too when inputs are +hidden, 1 KB cap), records one exception event, and is re-raised unchanged. +The exception event never carries the raw traceback (it would hold local file +paths, the username and code frames): `exception.stacktrace` is the same +frames-free `: ` summary as the status, and +`exception.type` is the class name. The caller still gets the original +exception with its own traceback. +There is no retry. + +Cancelling an `AsyncTavilyClient` call ends the span with status ERROR, +description `cancelled`, and `tavily.cancelled` = `true`, without an exception +event. A `TavilyClient` call cannot be cancelled. + +A failure inside the instrumentation (reading arguments, starting or ending the +span, reading the result) is logged at debug level on the `traceai_tavily` +logger and never changes what the call returns or raises; if the span cannot +start, the call runs untraced. + +## Instrumentor behaviour + +- `uninstrument()` puts the original methods back; later calls are not traced. +- `instrument()` checks the installed `tavily-python` against `>=0.8.4,<1`; on + another release it logs the conflict and wraps nothing (OpenTelemetry's + `BaseInstrumentor` behaviour). +- If a tavily-python release moves `TavilyClient`, `AsyncTavilyClient` or one of + the two methods, `instrument()` logs a WARNING naming it, skips it, and + instruments the rest. + +## Keyless clients + +tavily-python 0.8.5 lets `TavilyClient()` run with no key (`search` and +`extract` only; past the free allowance it raises `TavilyKeylessLimitError`). +The wrapper traces these calls like keyed ones: there is no key to redact, the +query is recorded (or hidden with `FI_HIDE_INPUTS`), and a limit error is an +ERROR span. Covered by tests and by one live keyless search. The `project_id` +and session header options on the client are not read or recorded. + +## Live vendor verification (2026-10-17) + +Real `api.tavily.com` with a Future AGI-owned key, `tavily-python` 0.8.5, +Python 3.12, exporting through `register()` to a local Future AGI: an agent span +with one `search` (RETRIEVER, 1 result), one `extract` (TOOL, 1 URL, 0 failed), +one keyless `search`, and one call with an invalid key (ERROR, +`InvalidAPIKeyError: Unauthorized: missing or invalid API key`). `fi_verify` +passed (1 trace, 5 spans); the same run on the pinned 0.8.4 (no keyless client) passed with 4 spans and the same scan. Scanning every stored column of the 5 rows (including +span events, resource and other attributes): the Tavily key, `tvly-`, the +response text and response URL, `Traceback`, `File "`, `/Users/`, the username +and the hostname appear in 0 rows. A unique query marker appears in 3 of 5 rows +by default and in 0 of 5 with `FI_HIDE_INPUTS=true` and `FI_HIDE_OUTPUTS=true`. +Future AGI stores an exception event's name and time only, so the exported +event attributes were checked separately on the span objects an exporter +receives (`tests/test_privacy.py`, and a live bad-key call: the event holds +`InvalidAPIKeyError: Unauthorized: missing or invalid API key.` and no frames). + +## Limits + +- Only `search` and `extract`; see [Instrumented calls](#instrumented-calls). +- No usage, credits, request id or search parameters (depth, topic, domains) + are recorded. +- Live verification was one search, one extract, one keyless search and one + bad-key call (below); no load or long-running test. + +## Tests + +Every Tavily call goes to a loopback fake and every key is a placeholder. + +As for the other traceAI Python packages, install the package and run `pytest` +in its directory (no `PYTHONPATH` needed; the tests that use the shared harness +in `python/tests/harness` add it to `sys.path` themselves). This tests against +the published `fi-instrumentation-otel`: + +```bash +cd python/frameworks/tavily +pip install -e . pytest +pytest +``` + +The two commands below instead run from the repository root against the +`fi_instrumentation` source in this checkout, with pinned versions. Run them +with any `PYTHONPATH` of your own unset; they set their own. + +Base suite: + +```bash +PYTHONPATH="python/frameworks/tavily:python:python/tests" uv run --no-project --python 3.11 \ + --with pytest --with pytest-asyncio --with opentelemetry-api --with opentelemetry-sdk \ + --with opentelemetry-instrumentation --with opentelemetry-exporter-otlp-proto-http \ + --with wrapt --with requests --with jsonschema --with protobuf --with opentelemetry-proto \ + --with 'tavily-python==0.8.4' \ + pytest python/frameworks/tavily/tests -q -p no:cacheprovider --noconftest -o addopts= -rs +``` + +With the LangChain extras: + +```bash +PYTHONPATH="python/frameworks/tavily:python:python/tests" uv run --no-project --python 3.11 \ + --with pytest --with pytest-asyncio --with opentelemetry-api --with opentelemetry-sdk \ + --with opentelemetry-instrumentation --with opentelemetry-exporter-otlp-proto-http \ + --with 'wrapt<2' --with requests --with jsonschema --with protobuf --with opentelemetry-proto \ + --with 'tavily-python==0.8.4' --with langchain-community==0.4.2 --with langchain-core==1.5.2 \ + --with langgraph==1.2.2 \ + pytest python/frameworks/tavily/tests -q -p no:cacheprovider --noconftest -o addopts= -rs +``` + +The base command (and plain `pytest` without the LangChain packages installed) +skips the 7 measurement tests that run LangChain or LangGraph (the AC-01 and +AC-04 LangGraph runs, the bare-client run with traceAI-langchain on, and the two +custom-tool runs, all in `tests/test_measurement.py`); the +second command adds the extras they need and runs them. It pins `wrapt<2` +(`fi-instrumentation-otel` declares `wrapt ^1.15.0`); the base command takes +the newest `wrapt`. `--python` can be 3.10, 3.11, 3.12 or 3.13. + +These measurement tests live in this package, not under +`python/frameworks/langchain/`. That is a deviation from PRD section 13, made to +keep the change in one package. diff --git a/python/frameworks/tavily/examples/search.py b/python/frameworks/tavily/examples/search.py new file mode 100644 index 000000000..63963d9e2 --- /dev/null +++ b/python/frameworks/tavily/examples/search.py @@ -0,0 +1,36 @@ +"""Trace one Tavily search with traceAI-tavily. + +Needs TAVILY_API_KEY plus the Future AGI settings register() reads +(FI_API_KEY, FI_SECRET_KEY, and FI_BASE_URL for a self-hosted collector). +TAVILY_BASE_URL is optional and only points the client at another Tavily +endpoint, such as a local test server. +""" + +import os + +from fi_instrumentation import register +from fi_instrumentation.fi_types import ProjectType +from tavily import TavilyClient + +from traceai_tavily import TavilyInstrumentor + + +def main() -> None: + tracer_provider = register( + project_name="tavily-example", + project_type=ProjectType.OBSERVE, + verbose=False, + ) + TavilyInstrumentor().instrument(tracer_provider=tracer_provider) + + client = TavilyClient( + api_key=os.environ["TAVILY_API_KEY"], + api_base_url=os.environ.get("TAVILY_BASE_URL") or None, + ) + response = client.search("What is OpenTelemetry?", max_results=2) + print("results: {0}".format(len(response["results"]))) + tracer_provider.force_flush() + + +if __name__ == "__main__": + main() diff --git a/python/frameworks/tavily/pyproject.toml b/python/frameworks/tavily/pyproject.toml new file mode 100644 index 000000000..1e339f136 --- /dev/null +++ b/python/frameworks/tavily/pyproject.toml @@ -0,0 +1,34 @@ +[tool.poetry] +name = "traceAI-tavily" +version = "0.1.0" +description = "OpenTelemetry instrumentation for the Tavily Python client" +authors = ["Future AGI "] +readme = "README.md" +packages = [ + { include = "traceai_tavily" } +] +keywords = ["opentelemetry", "tavily", "search", "tracing", "observability"] +classifiers = [ + "Development Status :: 3 - Alpha", + "Intended Audience :: Developers", + "License :: OSI Approved :: Apache Software License", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Topic :: Software Development :: Libraries :: Python Modules", +] + +[tool.poetry.dependencies] +python = ">=3.10,<3.14" +fi-instrumentation-otel = ">=1.1.0" +tavily-python = ">=0.8.4,<1" +wrapt = ">=1.15.0" + +[tool.poetry.group.dev.dependencies] +pytest = ">=7.0.0" + +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" diff --git a/python/frameworks/tavily/tests/_support.py b/python/frameworks/tavily/tests/_support.py new file mode 100644 index 000000000..eace02c3e --- /dev/null +++ b/python/frameworks/tavily/tests/_support.py @@ -0,0 +1,61 @@ +"""In-memory tracing helpers for the real-SDK tests. + +Spans go to a plain SDK ``TracerProvider`` with an in-memory exporter, never +through ``fi_instrumentation.register()``: register()'s processor turns UNSET +into OK before export, so only an in-memory provider shows the wrapper's own +status. +""" + +from __future__ import annotations + +import contextlib +from dataclasses import dataclass +from typing import Any, Dict, Iterator, List + +from opentelemetry.sdk.resources import Resource +from opentelemetry.sdk.trace import ReadableSpan, TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter + + +@dataclass +class Traced: + exporter: InMemorySpanExporter + provider: TracerProvider + + def spans(self) -> List[ReadableSpan]: + return list(self.exporter.get_finished_spans()) + + def one(self) -> ReadableSpan: + spans = self.spans() + assert len(spans) == 1, [span.name for span in spans] + return spans[0] + + def wire(self) -> str: + """Everything an exporter could send: attributes, events, status.""" + return "".join(span.to_json() for span in self.spans()) + + +def new_provider() -> Traced: + exporter = InMemorySpanExporter() + provider = TracerProvider(resource=Resource.create({"service.name": "test-tavily"})) + provider.add_span_processor(SimpleSpanProcessor(exporter)) + return Traced(exporter, provider) + + +@contextlib.contextmanager +def instrumented(**options: Any) -> Iterator[Traced]: + """Instrument Tavily against a fresh in-memory provider; uninstrument after.""" + from traceai_tavily import TavilyInstrumentor + + traced = new_provider() + instrumentor = TavilyInstrumentor() + instrumentor.instrument(tracer_provider=traced.provider, **options) + try: + yield traced + finally: + instrumentor.uninstrument() + + +def attrs(span: ReadableSpan) -> Dict[str, Any]: + return dict(span.attributes or {}) diff --git a/python/frameworks/tavily/tests/_tavily_fake.py b/python/frameworks/tavily/tests/_tavily_fake.py new file mode 100644 index 000000000..21c68b338 --- /dev/null +++ b/python/frameworks/tavily/tests/_tavily_fake.py @@ -0,0 +1,190 @@ +"""A loopback stand-in for api.tavily.com, shared by every Tavily test. + +``FakeTavily`` speaks enough of the Tavily HTTP API for tavily-python 0.8.4 +(``TavilyClient`` / ``AsyncTavilyClient``) and for LangChain's Tavily tool +(``langchain_community`` posts to the same ``/search`` route). It listens on +127.0.0.1 only and every key a test uses is a placeholder. + +Scripted queries (search) and URLs (extract): + +* ``fail-401`` gets HTTP 401 (``InvalidAPIKeyError``). +* ``fail-500`` gets HTTP 500 (``requests.HTTPError`` / ``httpx.HTTPStatusError``). +* ``keyless-limit`` gets HTTP 429 with the keyless error envelope + (``TavilyKeylessLimitError`` on a keyless client). +* a query starting ``echo-400`` gets HTTP 400 whose detail repeats the query, + so an error message can carry whatever the query carried. +* ``slow`` (a query, or a URL containing it) is held until the fake closes, + so a caller can cancel mid-call. +* ``odd-results`` gets HTTP 200 whose ``results`` is not a list. +* an extract URL containing ``fail`` is returned in ``failed_results``. +""" + +from __future__ import annotations + +import json +import threading +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from typing import Any, Dict, List, Tuple + +TAVILY_KEY = "tvly-placeholder-key-must-not-be-exported" +FAIL_401 = "fail-401" +FAIL_500 = "fail-500" +KEYLESS_LIMIT = "keyless-limit" +ECHO_400 = "echo-400" +SLOW = "slow" +ODD_RESULTS = "odd-results" + +# Response content the fake returns. None of it may reach a span. +RESULT_TITLE = "RESULT-TITLE-MUST-NOT-BE-EXPORTED" +RESULT_CONTENT = "RESULT-CONTENT-MUST-NOT-BE-EXPORTED" +RAW_CONTENT = "RAW-CONTENT-MUST-NOT-BE-EXPORTED" +ANSWER_TEXT = "ANSWER-TEXT-MUST-NOT-BE-EXPORTED" +EXTRACT_ERROR = "EXTRACT-ERROR-MUST-NOT-BE-EXPORTED" +CONTENT_MARKERS = (RESULT_TITLE, RESULT_CONTENT, RAW_CONTENT, ANSWER_TEXT, EXTRACT_ERROR) + +DEFAULT_SEARCH_RESULTS = 2 + + +def _result(index: int) -> Dict[str, Any]: + return { + "url": "https://example.com/result-{0}".format(index), + "title": RESULT_TITLE, + "content": RESULT_CONTENT, + "score": 0.9, + "raw_content": RAW_CONTENT, + } + + +class _QuietServer(ThreadingHTTPServer): + daemon_threads = True + + def handle_error(self, request: Any, client_address: Any) -> None: + # A cancelled client leaves a broken pipe behind; that is expected. + return + + +class FakeTavily: + """Loopback Tavily API. ``calls`` records (path, lower-cased headers, body).""" + + def __init__(self) -> None: + self.calls: List[Tuple[str, Dict[str, str], Dict[str, Any]]] = [] + self._lock = threading.Lock() + self._release = threading.Event() + owner = self + + class Handler(BaseHTTPRequestHandler): + def do_POST(self) -> None: # noqa: N802 - BaseHTTPRequestHandler API + length = int(self.headers.get("Content-Length", "0") or 0) + body = json.loads(self.rfile.read(length) or b"{}") + path = self.path.split("?")[0] + headers = {key.lower(): value for key, value in self.headers.items()} + with owner._lock: + owner.calls.append((path, headers, body)) + if path == "/search": + self._search(body) + elif path == "/extract": + self._extract(body) + else: + self._send(404, {"detail": {"error": "not found"}}) + + def _search(self, body: Dict[str, Any]) -> None: + query = str(body.get("query", "")) + if query == FAIL_401: + self._send(401, {"detail": {"error": "Unauthorized: invalid API key."}}) + elif query == FAIL_500: + self._send(500, {"detail": {"error": "internal error"}}) + elif query == KEYLESS_LIMIT: + self._send( + 429, + { + "error": { + "code": "keyless_rate_limited", + "message": "Keyless limit reached.", + "window": "day", + "retry_after_seconds": 60, + "next_actions": [], + } + }, + ) + elif query.startswith(ECHO_400): + self._send(400, {"detail": {"error": "Bad query: {0}".format(query)}}) + elif query == ODD_RESULTS: + self._send(200, {"query": query, "results": {"unexpected": "shape"}}) + elif query == SLOW: + owner._release.wait(30) + self._send(200, {"query": query, "results": []}) + else: + count = int(body.get("max_results") or DEFAULT_SEARCH_RESULTS) + self._send( + 200, + { + "query": query, + "answer": ANSWER_TEXT, + "images": [], + "follow_up_questions": None, + "results": [_result(i) for i in range(count)], + "response_time": 0.01, + "request_id": "req-search-1", + }, + ) + + def _extract(self, body: Dict[str, Any]) -> None: + urls = body.get("urls", []) + if isinstance(urls, str): + urls = [urls] + if any(SLOW in str(url) for url in urls): + owner._release.wait(30) + results = [] + failed = [] + for url in urls: + if "fail" in str(url): + failed.append({"url": url, "error": EXTRACT_ERROR}) + else: + results.append({"url": url, "raw_content": RAW_CONTENT, "images": []}) + self._send( + 200, + { + "results": results, + "failed_results": failed, + "response_time": 0.01, + "request_id": "req-extract-1", + }, + ) + + def _send(self, status: int, payload: Any) -> None: + data = json.dumps(payload).encode("utf-8") + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(data))) + self.end_headers() + self.wfile.write(data) + + def log_message(self, *_: Any) -> None: + return + + self._server = _QuietServer(("127.0.0.1", 0), Handler) + self.origin = "http://127.0.0.1:{0}".format(self._server.server_port) + self._thread = threading.Thread( + target=self._server.serve_forever, kwargs={"poll_interval": 0.05}, daemon=True + ) + self._thread.start() + + def paths(self) -> List[str]: + with self._lock: + return [path for path, _, _ in self.calls] + + def bodies(self) -> List[Dict[str, Any]]: + with self._lock: + return [body for _, _, body in self.calls] + + def close(self) -> None: + self._release.set() + self._server.shutdown() + self._server.server_close() + self._thread.join() + + def __enter__(self) -> "FakeTavily": + return self + + def __exit__(self, *_: Any) -> None: + self.close() diff --git a/python/frameworks/tavily/tests/measurement/bare_client.py b/python/frameworks/tavily/tests/measurement/bare_client.py new file mode 100644 index 000000000..106243b43 --- /dev/null +++ b/python/frameworks/tavily/tests/measurement/bare_client.py @@ -0,0 +1,90 @@ +"""TH-8327 measurement: what does a bare Tavily client export with register()? + +Run by tests/test_measurement.py through ``harness.run``. The script calls the +real tavily-python client (``TavilyClient`` and ``AsyncTavilyClient``, +``search`` and ``extract``) against the loopback fake in ``TAVILY_BASE_URL``, +with only ``fi_instrumentation.register()`` exporting to ``FI_BASE_URL``. + +All calls run inside one ``measurement.control`` span from the registered +provider. That span proves the export path works, so any other span that +reaches the receiver came from Tavily instrumentation. + +Options: + --with-langchain-instrumentor also instrument traceAI-langchain, the existing + traceAI instrumentor that traces Tavily as a + LangChain tool. + --with-tavily-instrumentor also instrument traceAI-tavily (if installed). + +Prints one JSON line with what the client returned (counts only). +""" + +from __future__ import annotations + +import asyncio +import json +import os +import sys + +from fi_instrumentation import register +from fi_instrumentation.fi_types import ProjectType +from tavily import AsyncTavilyClient, TavilyClient + +QUERY = "th-8327 bare client measurement" +URLS = ["https://example.com/a", "https://example.com/b"] + + +async def _async_calls(api_key: str, base_url: str) -> dict: + client = AsyncTavilyClient(api_key=api_key, api_base_url=base_url) + try: + search = await client.search(QUERY, max_results=2) + extract = await client.extract(URLS) + finally: + await client.close() + return { + "async_search_results": len(search["results"]), + "async_extract_results": len(extract["results"]), + } + + +def main() -> None: + # Global as well: a library with native OpenTelemetry support would pick up + # the global provider, so this is the most generous setup for the vendor. + provider = register( + project_name=os.environ.get("MEASURE_PROJECT", "tavily-measurement"), + project_type=ProjectType.OBSERVE, + batch=False, + set_global_tracer_provider=True, + verbose=False, + ) + instrumented = [] + if "--with-langchain-instrumentor" in sys.argv: + from traceai_langchain import LangChainInstrumentor + + LangChainInstrumentor().instrument(tracer_provider=provider) + instrumented.append("traceai_langchain") + if "--with-tavily-instrumentor" in sys.argv: + from traceai_tavily import TavilyInstrumentor + + TavilyInstrumentor().instrument(tracer_provider=provider) + instrumented.append("traceai_tavily") + + api_key = os.environ["TAVILY_API_KEY"] + base_url = os.environ["TAVILY_BASE_URL"] + tracer = provider.get_tracer("th8327.measurement") + with tracer.start_as_current_span("measurement.control"): + client = TavilyClient(api_key=api_key, api_base_url=base_url) + search = client.search(QUERY, max_results=2) + extract = client.extract(URLS) + client.close() + summary = { + "instrumented": instrumented, + "search_results": len(search["results"]), + "extract_results": len(extract["results"]), + } + summary.update(asyncio.run(_async_calls(api_key, base_url))) + provider.force_flush() + print(json.dumps(summary)) + + +if __name__ == "__main__": + main() diff --git a/python/frameworks/tavily/tests/measurement/langgraph_tool.py b/python/frameworks/tavily/tests/measurement/langgraph_tool.py new file mode 100644 index 000000000..dcc8a7195 --- /dev/null +++ b/python/frameworks/tavily/tests/measurement/langgraph_tool.py @@ -0,0 +1,151 @@ +"""TH-8327 measurement: the LangGraph path with traceAI-langchain on. + +Run by tests/test_measurement.py through ``harness.run``. This is the shape of +``python/frameworks/langchain/examples/langgraph_agent_supervisor.py``: a +LangGraph ``StateGraph`` whose agent asks for ``TavilySearchResults`` +(``langchain_community.tools.tavily_search``, the import that example uses) +and a ``ToolNode`` that runs it. The model is replaced by a scripted agent +node, so no LLM is called. ``TavilySearchResults`` posts to the loopback fake +in ``TAVILY_BASE_URL``; spans leave through ``fi_instrumentation.register()`` +to ``FI_BASE_URL``. + +Options: + --with-tavily-instrumentor also instrument traceAI-tavily (PRD J5: the + LangChain tool plus the bare-client wrapper). + --custom-tool use a LangChain ``@tool`` that calls + ``TavilyClient.search`` instead of + ``TavilySearchResults``. + --use-langchain-span in the custom tool, make traceAI-langchain's + tool span current around the client call + (``traceai_langchain.get_current_span()``). + --fail ask the tool for the fake's scripted 401 + (``fail-401``), so the client call raises + ``InvalidAPIKeyError``; the ``ToolNode`` turns it + into an error ``ToolMessage``. + +Prints one JSON line with the tool output length and the message count. +""" + +# No ``from __future__ import annotations``: LangGraph resolves the node and +# router annotations at runtime, and these are local names. + +import json +import os +import sys +import warnings + +from fi_instrumentation import register +from fi_instrumentation.fi_types import ProjectType + +QUERY = "th-8327 langgraph tool measurement" +# The fake answers this query with HTTP 401 (tests/_tavily_fake.py FAIL_401). +FAIL_QUERY = "fail-401" + + +def _custom_tool(): + """A LangChain tool written over the bare client, as an app might.""" + from langchain_core.tools import tool + from tavily import TavilyClient + + @tool + def tavily_web_search(query: str) -> str: + """Search the web with Tavily and return the result URLs.""" + client = TavilyClient( + api_key=os.environ["TAVILY_API_KEY"], + api_base_url=os.environ["TAVILY_BASE_URL"], + ) + if "--use-langchain-span" in sys.argv: + from opentelemetry import trace + from traceai_langchain import get_current_span + + with trace.use_span( + get_current_span(), + end_on_exit=False, + record_exception=False, + set_status_on_exception=False, + ): + response = client.search(query, max_results=2) + else: + response = client.search(query, max_results=2) + return json.dumps([result["url"] for result in response["results"]]) + + return tavily_web_search + + +def main() -> None: + provider = register( + project_name=os.environ.get("MEASURE_PROJECT", "tavily-langgraph-measurement"), + project_type=ProjectType.OBSERVE, + batch=False, + verbose=False, + ) + from traceai_langchain import LangChainInstrumentor + + LangChainInstrumentor().instrument(tracer_provider=provider) + instrumented = ["traceai_langchain"] + if "--with-tavily-instrumentor" in sys.argv: + from traceai_tavily import TavilyInstrumentor + + TavilyInstrumentor().instrument(tracer_provider=provider) + instrumented.append("traceai_tavily") + + from langchain_core.messages import AIMessage, HumanMessage, ToolMessage + from langgraph.graph import END, START, MessagesState, StateGraph + from langgraph.prebuilt import ToolNode + + if "--custom-tool" in sys.argv: + tool = _custom_tool() + else: + with warnings.catch_warnings(): + # TavilySearchResults is deprecated upstream in favour of + # langchain-tavily; the example on the base branch still imports it. + warnings.simplefilter("ignore") + import langchain_community.utilities.tavily_search as tavily_utilities + from langchain_community.tools.tavily_search import TavilySearchResults + + # The API wrapper reads this module constant on every call. + tavily_utilities.TAVILY_API_URL = os.environ["TAVILY_BASE_URL"] + tool = TavilySearchResults(max_results=2) + + fail = "--fail" in sys.argv + query = FAIL_QUERY if fail else QUERY + + def agent(state: MessagesState) -> dict: + if any(isinstance(message, ToolMessage) for message in state["messages"]): + return {"messages": [AIMessage(content="done")]} + call = {"name": tool.name, "args": {"query": query}, "id": "call-tavily-1"} + return {"messages": [AIMessage(content="", tool_calls=[call])]} + + def route(state: MessagesState) -> str: + return "tools" if state["messages"][-1].tool_calls else END + + graph = StateGraph(MessagesState) + graph.add_node("agent", agent) + # ToolNode's default re-raises a tool's own exception; --fail keeps the + # graph running so the error spans are exported. + tool_node = ToolNode([tool], handle_tool_errors=True) if fail else ToolNode([tool]) + graph.add_node("tools", tool_node) + graph.add_edge(START, "agent") + graph.add_conditional_edges("agent", route, ["tools", END]) + graph.add_edge("tools", "agent") + + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + result = graph.compile().invoke({"messages": [HumanMessage(content=QUERY)]}) + provider.force_flush() + tool_messages = [m for m in result["messages"] if isinstance(m, ToolMessage)] + print( + json.dumps( + { + "instrumented": instrumented, + "tool_name": tool.name, + "messages": len(result["messages"]), + "tool_messages": len(tool_messages), + "tool_status": [getattr(m, "status", None) for m in tool_messages], + } + ) + ) + + +if __name__ == "__main__": + main() diff --git a/python/frameworks/tavily/tests/test_consistency.py b/python/frameworks/tavily/tests/test_consistency.py new file mode 100644 index 000000000..01d4ad5d6 --- /dev/null +++ b/python/frameworks/tavily/tests/test_consistency.py @@ -0,0 +1,63 @@ +"""Family consistency: span kinds, the retrieval query key, and PII in error text.""" + +from __future__ import annotations + +import pytest + +pytest.importorskip("tavily", reason="tavily-python must be installed to test its instrumentor") + +from _support import attrs, instrumented # noqa: E402 +from _tavily_fake import FakeTavily # noqa: E402 +from fi_instrumentation import TraceConfig # noqa: E402 +from tavily import TavilyClient # noqa: E402 + +KEY = "tvly-placeholder-consistency-key" + + +@pytest.fixture() +def fake(): + with FakeTavily() as server: + yield server + + +def _client(fake: FakeTavily) -> TavilyClient: + return TavilyClient(api_key=KEY, api_base_url=fake.origin) + + +def test_search_is_a_retriever_with_the_repo_query_key_and_extract_is_a_tool(fake): + with instrumented() as traced: + client = _client(fake) + client.search("open telemetry") + client.extract(["https://example.com/a"], query="rerank me") + + search, extract = traced.spans() + assert attrs(search)["gen_ai.span.kind"] == "RETRIEVER" + assert attrs(search)["gen_ai.retrieval.query"] == "open telemetry" + assert attrs(extract)["gen_ai.span.kind"] == "TOOL" + assert "gen_ai.retrieval.query" not in attrs(extract) + assert attrs(extract)["input.value"] == "rerank me" + + +def test_hidden_inputs_drop_the_retrieval_query(fake): + with instrumented(config=TraceConfig(hide_inputs=True)) as traced: + _client(fake).search("private question") + + values = attrs(traced.one()) + assert values["input.value"] == "__REDACTED__" + assert "gen_ai.retrieval.query" not in values + assert "private question" not in traced.wire() + + +def test_pii_redaction_also_covers_error_text(fake): + from _tavily_fake import ECHO_400 # the fake's 400 echoes the query + + with instrumented(config=TraceConfig(pii_redaction=True)) as traced: + with pytest.raises(Exception): + _client(fake).search("{0} mail jane.doe@example.com".format(ECHO_400)) + + span = traced.one() + (event,) = [event for event in span.events if event.name == "exception"] + for text in (span.status.description, event.attributes["exception.message"], + event.attributes["exception.stacktrace"]): + assert "jane.doe@example.com" not in text + assert "" in event.attributes["exception.message"] diff --git a/python/frameworks/tavily/tests/test_contract_harness.py b/python/frameworks/tavily/tests/test_contract_harness.py new file mode 100644 index 000000000..57779223e --- /dev/null +++ b/python/frameworks/tavily/tests/test_contract_harness.py @@ -0,0 +1,160 @@ +"""Collector contract through the shared harness (TH-8339 Receiver). + +The real tavily-python clients make real HTTP calls to the loopback fake; +spans leave through the real ``fi_instrumentation.register()`` OTLP exporter +into ``harness.Receiver``. Nothing calls api.tavily.com; every key is a +placeholder. + +register()'s processor turns UNSET into OK before export, so a wire-level OK +does not prove the wrapper set it; test_spans.py asserts the wrapper's own OK +on an in-memory provider. +""" + +from __future__ import annotations + +import asyncio +import json +import sys +from pathlib import Path +from typing import Any, Dict, List, Tuple + +import pytest + +pytest.importorskip("tavily", reason="tavily-python must be installed to test its instrumentor") +pytest.importorskip("opentelemetry.proto", reason="the harness decodes OTLP protobuf") + +# The shared harness lives in python/tests (as for the other harness users). +sys.path.insert(0, str(Path(__file__).resolve().parents[3] / "tests")) + +from _tavily_fake import CONTENT_MARKERS, FAIL_401, TAVILY_KEY, FakeTavily # noqa: E402 +from harness import Receiver, _flatten_attributes # noqa: E402 + +FI_API_KEY = "placeholder-fi-api-key" +FI_SECRET_KEY = "placeholder-fi-secret-key" +PROJECT = "tavily-contract" +URLS = ["https://example.com/private?token=abc", "https://example.com/fail-b"] +KEYED_QUERY = "agent asked about {0}".format(TAVILY_KEY) + + +def _journey( + receiver: Receiver, fake: FakeTavily, monkeypatch: pytest.MonkeyPatch +) -> Tuple[List[dict], List[dict], Dict[str, Any]]: + from fi_instrumentation import register + from fi_instrumentation.fi_types import ProjectType + from tavily import AsyncTavilyClient, InvalidAPIKeyError, TavilyClient + + from traceai_tavily import TavilyInstrumentor + + monkeypatch.setenv("FI_BASE_URL", receiver.origin) + monkeypatch.setenv("FI_API_KEY", FI_API_KEY) + monkeypatch.setenv("FI_SECRET_KEY", FI_SECRET_KEY) + provider = register(project_type=ProjectType.OBSERVE, project_name=PROJECT, verbose=False) + instrumentor = TavilyInstrumentor() + instrumentor.instrument(tracer_provider=provider) + returned: Dict[str, Any] = {} + + async def async_calls() -> None: + client = AsyncTavilyClient(api_key=TAVILY_KEY, api_base_url=fake.origin) + try: + returned["async_search"] = await client.search("async question", max_results=1) + returned["async_extract"] = await client.extract(URLS[:1]) + finally: + await client.close() + + try: + tracer = provider.get_tracer("contract") + with tracer.start_as_current_span("agent-turn"): + client = TavilyClient(api_key=TAVILY_KEY, api_base_url=fake.origin) + returned["search"] = client.search(KEYED_QUERY, max_results=2) + returned["extract"] = client.extract(URLS) + with pytest.raises(InvalidAPIKeyError): + client.search(FAIL_401) + asyncio.run(async_calls()) + assert provider.force_flush(timeout_millis=10_000) + finally: + instrumentor.uninstrument() + provider.shutdown() + return receiver.spans(), receiver.requests(), returned + + +@pytest.fixture() +def journey(monkeypatch): + with FakeTavily() as fake, Receiver() as receiver: + spans, exports, returned = _journey(receiver, fake, monkeypatch) + paths = fake.paths() + return spans, exports, returned, paths + + +def test_real_client_calls_reach_the_collector_contract(journey): + spans, exports, _, paths = journey + + # The real SDK made the HTTP calls. + assert paths == ["/search", "/extract", "/search", "/search", "/extract"] + + # Collector contract on every export. + assert exports + for export in exports: + assert export["path"] == "/tracer/v1/traces" + assert export["headers"]["x-api-key"] == FI_API_KEY + assert export["headers"]["x-secret-key"] == FI_SECRET_KEY + assert "authorization" not in export["headers"] + for resource in export["resource_attributes"]: + assert resource["project_name"] == PROJECT + assert resource["project_type"] == "observe" + + by_name: Dict[str, List[dict]] = {} + for span in spans: + by_name.setdefault(span["name"], []).append(span) + assert sorted((name, len(group)) for name, group in by_name.items()) == [ + ("agent-turn", 1), + ("tavily.extract", 2), + ("tavily.search", 3), + ] + parent = by_name["agent-turn"][0] + for name in ("tavily.search", "tavily.extract"): + for span in by_name[name]: + values = _flatten_attributes(span["attributes"]) + # search is a retrieval; extract fetches the pages it is given. + expected = "RETRIEVER" if name == "tavily.search" else "TOOL" + assert values["gen_ai.span.kind"] == expected + assert values["gen_ai.tool.name"] == name + assert span["parentSpanId"] == parent["spanId"] + assert span["traceId"] == parent["traceId"] + + search_ok, search_failed, async_search = by_name["tavily.search"] + ok = _flatten_attributes(search_ok["attributes"]) + assert ok["input.value"] == "agent asked about [redacted]" + assert int(ok["tavily.result_count"]) == 2 + assert search_ok["status"]["code"] == "STATUS_CODE_OK" + + failed = _flatten_attributes(search_failed["attributes"]) + assert failed["input.value"] == FAIL_401 + assert "tavily.result_count" not in failed + assert search_failed["status"]["code"] == "STATUS_CODE_ERROR" + assert search_failed["status"]["message"] == ( + "InvalidAPIKeyError: Unauthorized: invalid API key." + ) + assert [event["name"] for event in search_failed.get("events", [])] == ["exception"] + + assert int(_flatten_attributes(async_search["attributes"])["tavily.result_count"]) == 1 + + extract, async_extract = by_name["tavily.extract"] + values = _flatten_attributes(extract["attributes"]) + assert int(values["tavily.url_count"]) == 2 + assert int(values["tavily.result_count"]) == 1 + assert int(values["tavily.failed_result_count"]) == 1 + assert "input.value" not in values + assert int(_flatten_attributes(async_extract["attributes"])["tavily.url_count"]) == 1 + + +def test_no_key_urls_or_response_content_are_exported(journey): + spans, exports, returned, _ = journey + + # Control: the content reached the caller, so its absence below is real. + caller = json.dumps(returned) + assert all(marker in caller for marker in CONTENT_MARKERS) + assert URLS[0] in caller + + wire = json.dumps(spans) + json.dumps(exports) + for secret in (TAVILY_KEY, "tvly-", *CONTENT_MARKERS, *URLS, "token=abc"): + assert secret not in wire, secret diff --git a/python/frameworks/tavily/tests/test_errors.py b/python/frameworks/tavily/tests/test_errors.py new file mode 100644 index 000000000..99e1b597b --- /dev/null +++ b/python/frameworks/tavily/tests/test_errors.py @@ -0,0 +1,190 @@ +"""Errors and cancellation: ERROR status, exception re-raised unchanged.""" + +from __future__ import annotations + +import asyncio +from typing import Any + +import pytest + +pytest.importorskip("tavily", reason="tavily-python must be installed to test its instrumentor") + +from opentelemetry.trace import StatusCode # noqa: E402 + +from _support import attrs, instrumented # noqa: E402 +from _tavily_fake import ( # noqa: E402 + FAIL_401, + FAIL_500, + KEYLESS_LIMIT, + SLOW, + TAVILY_KEY, + FakeTavily, +) + + +@pytest.fixture() +def fake(): + with FakeTavily() as server: + yield server + + +def _sync(fake: FakeTavily, api_key: Any = TAVILY_KEY) -> Any: + from tavily import TavilyClient + + return TavilyClient(api_key=api_key, api_base_url=fake.origin) + + +async def _async(fake: FakeTavily, method: str, *args: Any, api_key: Any = TAVILY_KEY) -> Any: + from tavily import AsyncTavilyClient + + client = AsyncTavilyClient(api_key=api_key, api_base_url=fake.origin) + try: + return await getattr(client, method)(*args) + finally: + await client.close() + + +def _exception_events(span: Any) -> list: + return [event for event in span.events if event.name == "exception"] + + +def _assert_error_span(span: Any, error: BaseException) -> None: + assert span.status.status_code is StatusCode.ERROR + assert span.status.description == "{0}: {1}".format(type(error).__name__, error) + (event,) = _exception_events(span) + assert event.attributes["exception.type"].endswith(type(error).__name__) + values = attrs(span) + assert "tavily.result_count" not in values + assert "tavily.cancelled" not in values + + +@pytest.mark.parametrize("mode", ["sync", "async"]) +def test_invalid_key_sets_error_and_reraises(fake, mode): + from tavily import InvalidAPIKeyError + + with instrumented() as traced: + with pytest.raises(InvalidAPIKeyError) as info: + if mode == "sync": + _sync(fake).search(FAIL_401) + else: + asyncio.run(_async(fake, "search", FAIL_401)) + + assert str(info.value) == "Unauthorized: invalid API key." + span = traced.one() + assert span.name == "tavily.search" + assert attrs(span)["input.value"] == FAIL_401 + _assert_error_span(span, info.value) + + +def test_sync_http_error_sets_error_and_reraises(fake): + import requests + + with instrumented() as traced: + with pytest.raises(requests.HTTPError) as info: + _sync(fake).search(FAIL_500) + + _assert_error_span(traced.one(), info.value) + + +def test_async_http_error_sets_error_and_reraises(fake): + import httpx + + with instrumented() as traced: + with pytest.raises(httpx.HTTPStatusError) as info: + asyncio.run(_async(fake, "search", FAIL_500)) + + _assert_error_span(traced.one(), info.value) + + +@pytest.mark.parametrize("mode", ["sync", "async"]) +def test_keyless_limit_sets_error_reraises_and_does_not_retry(fake, monkeypatch, mode): + """PRD J4: TavilyKeylessLimitError is ERROR and re-raised; no retry loop.""" + from tavily import TavilyKeylessLimitError + + monkeypatch.delenv("TAVILY_API_KEY", raising=False) + with instrumented() as traced: + with pytest.raises(TavilyKeylessLimitError) as info: + if mode == "sync": + _sync(fake, api_key=None).search(KEYLESS_LIMIT) + else: + asyncio.run(_async(fake, "search", KEYLESS_LIMIT, api_key=None)) + + assert info.value.code == "keyless_rate_limited" + assert fake.paths() == ["/search"] + assert fake.calls[0][1].get("x-tavily-access-mode") == "keyless" + _assert_error_span(traced.one(), info.value) + + +@pytest.mark.parametrize("mode", ["sync", "async"]) +def test_a_current_parent_that_records_nothing_leaves_one_exception_event(fake, mode): + """The README's pattern for a LangChain tool span, with plain OpenTelemetry. + + ``use_span(..., record_exception=False, set_status_on_exception=False)`` + makes the parent current without recording the error on it, so the + parent's owner (traceAI-langchain's on_tool_error) records it once. + """ + from opentelemetry import trace + from tavily import InvalidAPIKeyError + + with instrumented() as traced: + parent = traced.provider.get_tracer("test").start_span("langchain-tool") + with pytest.raises(InvalidAPIKeyError): + with trace.use_span( + parent, + end_on_exit=False, + record_exception=False, + set_status_on_exception=False, + ): + if mode == "sync": + _sync(fake).search(FAIL_401) + else: + asyncio.run(_async(fake, "search", FAIL_401)) + parent.end() + + spans = {span.name: span for span in traced.spans()} + assert sorted(spans) == ["langchain-tool", "tavily.search"] + tool, search = spans["langchain-tool"], spans["tavily.search"] + assert _exception_events(tool) == [] + assert tool.status.status_code is StatusCode.UNSET + assert search.parent is not None + assert search.parent.span_id == tool.context.span_id + assert search.context.trace_id == tool.context.trace_id + assert len(_exception_events(search)) == 1 + assert search.status.status_code is StatusCode.ERROR + + +@pytest.mark.parametrize( + "method, argument", + [("search", SLOW), ("extract", ["https://example.com/" + SLOW])], +) +def test_cancelled_async_call_is_error_cancelled(fake, method, argument): + async def scenario() -> None: + task = asyncio.ensure_future(_async(fake, method, argument)) + while not fake.calls: + await asyncio.sleep(0.01) + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + + with instrumented() as traced: + asyncio.run(scenario()) + + span = traced.one() + assert span.name == "tavily.{0}".format(method) + assert span.status.status_code is StatusCode.ERROR + assert span.status.description == "cancelled" + values = attrs(span) + assert values["tavily.cancelled"] is True + assert "tavily.result_count" not in values + # Cancellation is not a failure of the call: no exception event. + assert _exception_events(span) == [] + + +def test_successful_calls_have_no_cancelled_attribute(fake): + with instrumented() as traced: + _sync(fake).search("fine") + asyncio.run(_async(fake, "extract", ["https://example.com/a"])) + + for span in traced.spans(): + assert "tavily.cancelled" not in attrs(span) + assert span.status.status_code is StatusCode.OK diff --git a/python/frameworks/tavily/tests/test_example.py b/python/frameworks/tavily/tests/test_example.py new file mode 100644 index 000000000..d17e17fd3 --- /dev/null +++ b/python/frameworks/tavily/tests/test_example.py @@ -0,0 +1,90 @@ +"""examples/search.py runs end to end through the shared harness. + +harness.run starts the example as a subprocess with FI_BASE_URL pointed at +harness.Receiver and TAVILY_BASE_URL at the loopback Tavily fake, so the +real exporter and the real tavily-python client both run. Nothing calls +api.tavily.com. +""" + +from __future__ import annotations + +import json +import os +import sys +from pathlib import Path + +import pytest + +pytest.importorskip("tavily", reason="tavily-python must be installed to run the example") +pytest.importorskip("opentelemetry.proto", reason="the harness decodes OTLP protobuf") + +# The shared harness lives in python/tests (as for the other harness users). +sys.path.insert(0, str(Path(__file__).resolve().parents[3] / "tests")) + +from _tavily_fake import CONTENT_MARKERS, TAVILY_KEY, FakeTavily # noqa: E402 +from harness import Receiver, _flatten_attributes, run # noqa: E402 + +import fi_instrumentation # noqa: E402 + +PACKAGE = Path(__file__).resolve().parents[1] +PYTHON_ROOT = PACKAGE.parents[1] +# Subprocesses import fi_instrumentation from where this process did: the +# source tree in a repo run, the installed wheel in a published-release run. +FI_ROOT = Path(fi_instrumentation.__file__).resolve().parents[1] +EXAMPLE = PACKAGE / "examples" / "search.py" +FI_API_KEY = "placeholder-fi-api-key" +FI_SECRET_KEY = "placeholder-fi-secret-key" + + +def test_example_exports_one_tavily_search_span(): + assert EXAMPLE.is_file(), EXAMPLE + with FakeTavily() as fake, Receiver() as receiver: + env = { + key: value + for key, value in os.environ.items() + if not key.startswith(("FI_", "TAVILY_", "OTEL_")) + } + env.update( + { + "FI_BASE_URL": receiver.origin, + "FI_API_KEY": FI_API_KEY, + "FI_SECRET_KEY": FI_SECRET_KEY, + "TAVILY_API_KEY": TAVILY_KEY, + "TAVILY_BASE_URL": fake.origin, + "PYTHONPATH": os.pathsep.join( + [str(PACKAGE), str(FI_ROOT), os.environ.get("PYTHONPATH", "")] + ), + } + ) + result = run([sys.executable, str(EXAMPLE)], env, None, timeout=120) + spans = receiver.spans() + exports = receiver.requests() + paths = fake.paths() + + assert not result.timed_out + assert result.returncode == 0, result.stderr.decode(errors="replace") + assert paths == ["/search"] + # The example prints only the number of results. + assert result.stdout.decode().strip() == "results: 2" + + assert [span["name"] for span in spans] == ["tavily.search"] + values = _flatten_attributes(spans[0]["attributes"]) + assert values["gen_ai.span.kind"] == "RETRIEVER" + assert values["gen_ai.retrieval.query"] == "What is OpenTelemetry?" + assert values["input.value"] == "What is OpenTelemetry?" + assert int(values["tavily.result_count"]) == 2 + assert spans[0]["status"]["code"] == "STATUS_CODE_OK" + + assert exports + for export in exports: + assert export["path"] == "/tracer/v1/traces" + assert export["headers"]["x-api-key"] == FI_API_KEY + assert export["headers"]["x-secret-key"] == FI_SECRET_KEY + for resource in export["resource_attributes"]: + assert resource["project_name"] == "tavily-example" + assert resource["project_type"] == "observe" + + wire = json.dumps(spans) + assert TAVILY_KEY not in wire + for marker in CONTENT_MARKERS: + assert marker not in wire diff --git a/python/frameworks/tavily/tests/test_instrumentor.py b/python/frameworks/tavily/tests/test_instrumentor.py new file mode 100644 index 000000000..58ace8858 --- /dev/null +++ b/python/frameworks/tavily/tests/test_instrumentor.py @@ -0,0 +1,173 @@ +"""Instrumentor lifecycle: config, context attributes, uninstrument, vendor drift.""" + +from __future__ import annotations + +import asyncio +import logging +from typing import Any + +import pytest + +pytest.importorskip("tavily", reason="tavily-python must be installed to test its instrumentor") + +from fi_instrumentation import TraceConfig, suppress_tracing, using_session, using_user # noqa: E402 +from opentelemetry import trace as trace_api # noqa: E402 + +from _support import attrs, instrumented, new_provider # noqa: E402 +from _tavily_fake import TAVILY_KEY, FakeTavily # noqa: E402 + +_METHODS = [ + ("tavily.tavily", "TavilyClient", "search"), + ("tavily.tavily", "TavilyClient", "extract"), + ("tavily.async_tavily", "AsyncTavilyClient", "search"), + ("tavily.async_tavily", "AsyncTavilyClient", "extract"), +] + + +@pytest.fixture() +def fake(): + with FakeTavily() as server: + yield server + + +def _class(module_name: str, class_name: str) -> Any: + import importlib + + return getattr(importlib.import_module(module_name), class_name) + + +def _search(fake: FakeTavily, query: str = "q") -> Any: + from tavily import TavilyClient + + return TavilyClient(api_key=TAVILY_KEY, api_base_url=fake.origin).search(query) + + +async def _async_extract(fake: FakeTavily) -> Any: + from tavily import AsyncTavilyClient + + client = AsyncTavilyClient(api_key=TAVILY_KEY, api_base_url=fake.origin) + try: + return await client.extract(["https://example.com/a"]) + finally: + await client.close() + + +def test_uninstrument_restores_every_method_and_stops_tracing(fake): + from traceai_tavily import TavilyInstrumentor + + originals = { + (m, c, name): vars(_class(m, c))[name] for m, c, name in _METHODS + } + traced = new_provider() + instrumentor = TavilyInstrumentor() + instrumentor.instrument(tracer_provider=traced.provider) + for (m, c, name), original in originals.items(): + assert vars(_class(m, c))[name] is not original, (c, name) + instrumentor.uninstrument() + + for (m, c, name), original in originals.items(): + assert vars(_class(m, c))[name] is original, (c, name) + _search(fake) + asyncio.run(_async_extract(fake)) + assert traced.spans() == [] + + +def test_instrument_again_after_uninstrument(fake): + with instrumented(): + pass + with instrumented() as traced: + _search(fake) + assert [span.name for span in traced.spans()] == ["tavily.search"] + + +def test_hide_inputs_config_masks_the_query(fake): + with instrumented(config=TraceConfig(hide_inputs=True)) as traced: + _search(fake, "private question") + + values = attrs(traced.one()) + assert values["input.value"] == "__REDACTED__" + assert values["tavily.result_count"] == 2 + assert "private question" not in traced.wire() + + +def test_fi_hide_inputs_environment_variable_masks_the_query(fake, monkeypatch): + monkeypatch.setenv("FI_HIDE_INPUTS", "true") + with instrumented() as traced: + _search(fake, "private question") + + assert attrs(traced.one())["input.value"] == "__REDACTED__" + + +def test_config_must_be_a_trace_config(): + from tavily import TavilyClient + + from traceai_tavily import TavilyInstrumentor + + original = vars(TavilyClient)["search"] + instrumentor = TavilyInstrumentor() + try: + with pytest.raises(TypeError, match="TraceConfig"): + instrumentor.instrument(config={"hide_inputs": True}) + assert vars(TavilyClient)["search"] is original + finally: + instrumentor.uninstrument() + assert vars(TavilyClient)["search"] is original + + +def test_session_and_user_context_reach_the_span(fake): + with instrumented() as traced: + with using_session("session-1"), using_user("user-1"): + _search(fake) + + values = attrs(traced.one()) + assert values["session.id"] == "session-1" + assert values["user.id"] == "user-1" + + +def test_suppress_tracing_records_nothing(fake): + with instrumented() as traced: + with suppress_tracing(): + result = _search(fake) + + assert len(result["results"]) == 2 + assert traced.spans() == [] + + +def test_a_no_op_tracer_provider_is_accepted(fake): + from traceai_tavily import TavilyInstrumentor + + instrumentor = TavilyInstrumentor() + instrumentor.instrument(tracer_provider=trace_api.NoOpTracerProvider()) + try: + assert len(_search(fake)["results"]) == 2 + finally: + instrumentor.uninstrument() + + +def test_a_missing_vendor_class_is_skipped_with_a_warning(fake, monkeypatch, caplog): + import tavily.async_tavily + + monkeypatch.delattr(tavily.async_tavily, "AsyncTavilyClient") + caplog.set_level(logging.WARNING, logger="traceai_tavily") + with instrumented() as traced: + _search(fake) + + assert [span.name for span in traced.spans()] == ["tavily.search"] + warnings = [r.getMessage() for r in caplog.records if r.levelno == logging.WARNING] + assert any("tavily.async_tavily.AsyncTavilyClient" in message for message in warnings) + + +def test_a_missing_vendor_method_is_skipped_with_a_warning(fake, monkeypatch, caplog): + from tavily import TavilyClient + + original_search = vars(TavilyClient)["search"] + monkeypatch.delattr(TavilyClient, "extract") + caplog.set_level(logging.WARNING, logger="traceai_tavily") + with instrumented() as traced: + _search(fake) + + assert [span.name for span in traced.spans()] == ["tavily.search"] + assert vars(TavilyClient)["search"] is original_search + assert "extract" not in vars(TavilyClient) + warnings = [r.getMessage() for r in caplog.records if r.levelno == logging.WARNING] + assert any("TavilyClient.extract" in message for message in warnings) diff --git a/python/frameworks/tavily/tests/test_isolation.py b/python/frameworks/tavily/tests/test_isolation.py new file mode 100644 index 000000000..24ad4e713 --- /dev/null +++ b/python/frameworks/tavily/tests/test_isolation.py @@ -0,0 +1,219 @@ +"""Instrumentation failures never reach the caller. + +Each test breaks one piece of the wrapper's own work (reading the client, +reading arguments, starting the span, making it current, reading the result, +recording the error, ending the span) and checks that the caller still gets +the vendor's own result or exception. Where the vendor path cannot produce +the fault, a wrapper helper or an SDK method is patched to raise. +""" + +from __future__ import annotations + +import asyncio +from typing import Any + +import pytest + +pytest.importorskip("tavily", reason="tavily-python must be installed to test its instrumentor") + +from opentelemetry import trace as trace_api # noqa: E402 +from opentelemetry.sdk.trace import Span as SdkSpan # noqa: E402 +from opentelemetry.trace import StatusCode # noqa: E402 + +from _support import attrs, instrumented # noqa: E402 +from _tavily_fake import FAIL_401, ODD_RESULTS, TAVILY_KEY, FakeTavily # noqa: E402 + + +@pytest.fixture() +def fake(): + with FakeTavily() as server: + yield server + + +def _client(fake: FakeTavily) -> Any: + from tavily import TavilyClient + + return TavilyClient(api_key=TAVILY_KEY, api_base_url=fake.origin) + + +async def _async_search(fake: FakeTavily, query: Any) -> Any: + from tavily import AsyncTavilyClient + + client = AsyncTavilyClient(api_key=TAVILY_KEY, api_base_url=fake.origin) + try: + return await client.search(query) + finally: + await client.close() + + +class _Boom(RuntimeError): + pass + + +def _boom(*_: Any, **__: Any) -> Any: + raise _Boom("instrumentation failure") + + +class _ExplodingHeaders: + def get(self, *_: Any) -> Any: + raise _Boom("headers") + + +def test_an_unreadable_key_holder_drops_free_text_and_keeps_the_call(fake): + """Fail closed: a key in a place that cannot be read could not be redacted.""" + from tavily import InvalidAPIKeyError + + with instrumented() as traced: + client = _client(fake) + # Not used by the SDK after __init__; the wrapper reads it for the key. + client.headers = _ExplodingHeaders() + result = client.search("q " + TAVILY_KEY) + with pytest.raises(InvalidAPIKeyError): + client.search(FAIL_401) + + assert len(result["results"]) == 2 + ok, failed = traced.spans() + assert "input.value" not in attrs(ok) + assert attrs(ok)["tavily.result_count"] == 2 + assert "input.value" not in attrs(failed) + assert failed.status.description == "InvalidAPIKeyError" + (event,) = [event for event in failed.events if event.name == "exception"] + assert event.attributes["exception.message"] == "[redacted]" + assert TAVILY_KEY not in traced.wire() + + +def test_an_unreadable_signature_records_no_query_and_keeps_the_call(fake, monkeypatch): + import inspect + + with instrumented() as traced: + monkeypatch.setattr(inspect, "signature", _boom) + result = _client(fake).search("q") + monkeypatch.undo() + + assert len(result["results"]) == 2 + span = traced.one() + assert "input.value" not in attrs(span) + assert attrs(span)["tavily.result_count"] == 2 + assert span.status.status_code is StatusCode.OK + + +def test_an_unprintable_query_gets_the_vendors_own_error(fake): + class Unprintable: + def __str__(self) -> str: + raise _Boom("str") + + with instrumented() as traced: + with pytest.raises(TypeError, match="not JSON serializable"): + _client(fake).search(Unprintable()) + + span = traced.one() + assert "input.value" not in attrs(span) + assert span.status.status_code is StatusCode.ERROR + assert span.status.description.startswith("TypeError") + + +@pytest.mark.parametrize("mode", ["sync", "async"]) +def test_a_failing_tracer_leaves_the_call_untraced(fake, mode): + class BrokenTracer(trace_api.Tracer): + def start_span(self, *args: Any, **kwargs: Any) -> Any: + raise _Boom("start_span") + + def start_as_current_span(self, *args: Any, **kwargs: Any) -> Any: + raise _Boom("start_as_current_span") + + class BrokenProvider(trace_api.TracerProvider): + def get_tracer(self, *args: Any, **kwargs: Any) -> trace_api.Tracer: + return BrokenTracer() + + from traceai_tavily import TavilyInstrumentor + + instrumentor = TavilyInstrumentor() + instrumentor.instrument(tracer_provider=BrokenProvider()) + try: + if mode == "sync": + result = _client(fake).search("still works") + else: + result = asyncio.run(_async_search(fake, "still works")) + finally: + instrumentor.uninstrument() + + assert len(result["results"]) == 2 + assert fake.paths() == ["/search"] + + +def test_a_failing_context_attach_does_not_reach_the_caller(fake, monkeypatch): + import traceai_tavily._wrappers as wrappers + + monkeypatch.setattr(wrappers.context_api, "attach", _boom) + with instrumented() as traced: + result = _client(fake).search("q") + + assert len(result["results"]) == 2 + assert traced.one().status.status_code is StatusCode.OK + + +@pytest.mark.parametrize("mode", ["sync", "async"]) +def test_an_unexpected_result_shape_omits_the_count(fake, mode): + with instrumented() as traced: + if mode == "sync": + result = _client(fake).search(ODD_RESULTS) + else: + result = asyncio.run(_async_search(fake, ODD_RESULTS)) + + assert result["results"] == {"unexpected": "shape"} + span = traced.one() + # Unknown, so absent: never 0. + assert "tavily.result_count" not in attrs(span) + assert span.status.status_code is StatusCode.OK + + +@pytest.mark.parametrize("mode", ["sync", "async"]) +def test_a_failing_result_reader_still_returns_and_ends_the_span(fake, monkeypatch, mode): + import traceai_tavily._wrappers as wrappers + + monkeypatch.setattr(wrappers, "_count", _boom) + with instrumented() as traced: + if mode == "sync": + result = _client(fake).search("q") + else: + result = asyncio.run(_async_search(fake, "q")) + + assert len(result["results"]) == 2 + span = traced.one() + assert "tavily.result_count" not in attrs(span) + assert span.status.status_code is StatusCode.OK + + +def test_a_failing_exception_recorder_keeps_the_vendor_exception(fake, monkeypatch): + from tavily import InvalidAPIKeyError + + monkeypatch.setattr(SdkSpan, "record_exception", _boom) + with instrumented() as traced: + with pytest.raises(InvalidAPIKeyError): + _client(fake).search(FAIL_401) + + span = traced.one() + assert span.status.status_code is StatusCode.ERROR + + +def test_an_unprintable_vendor_error_is_still_reraised(fake, monkeypatch): + from tavily import InvalidAPIKeyError + + monkeypatch.setattr(InvalidAPIKeyError, "__str__", _boom) + with instrumented() as traced: + with pytest.raises(InvalidAPIKeyError): + _client(fake).search(FAIL_401) + + span = traced.one() + assert span.status.status_code is StatusCode.ERROR + assert span.status.description == "InvalidAPIKeyError" + + +def test_a_failing_span_end_does_not_reach_the_caller(fake, monkeypatch): + from tavily import InvalidAPIKeyError + + monkeypatch.setattr(SdkSpan, "end", _boom) + with instrumented(): + assert len(_client(fake).search("q")["results"]) == 2 + with pytest.raises(InvalidAPIKeyError): + _client(fake).search(FAIL_401) diff --git a/python/frameworks/tavily/tests/test_measurement.py b/python/frameworks/tavily/tests/test_measurement.py new file mode 100644 index 000000000..d10483d25 --- /dev/null +++ b/python/frameworks/tavily/tests/test_measurement.py @@ -0,0 +1,388 @@ +"""TH-8327 step 1: measure what Tavily already emits, before any wrapper. + +D-8327 (PRD r2, architecture TH-8327): a traceAI-tavily wrapper ships only if +the bare client emits nothing. Both measurements run a script through the +shared harness (``harness.run``) against the loopback Tavily fake and export +through the real ``fi_instrumentation.register()`` to ``harness.Receiver``. +Nothing calls api.tavily.com and every key is a placeholder. + +* Bare client (PRD G2 / AC-02): ``TavilyClient`` and ``AsyncTavilyClient`` + ``search`` and ``extract`` with ``register()`` only (also as the global + provider), then with traceAI-langchain instrumented as well. Every call runs + inside one ``measurement.control`` span, so an export that arrives proves the + pipeline works and the Tavily span count is what is left. +* LangGraph path (PRD G1 / AC-01): the example's + ``langchain_community.tools.tavily_search.TavilySearchResults`` run by a + LangGraph ``ToolNode`` with traceAI-langchain on. + +Measured with tavily-python 0.8.4, langchain-community 0.4.2, langchain-core +1.5.2 and langgraph 1.2.2: + +* Bare client: 0 Tavily spans for 4 real HTTP calls, with or without + traceAI-langchain. tavily-python 0.8.4 has no OpenTelemetry code. +* LangGraph path: 1 TOOL span, ``tavily_search_results_json``, with the keys + in ``LANGCHAIN_TOOL_KEYS``. It carries the tool's result text in + ``output.value``, and the next agent turn carries it in ``input.value``; + only ``FI_HIDE_INPUTS`` plus ``FI_HIDE_OUTPUTS`` keeps it all in-process. + +With traceAI-tavily on (the wrapper the first result released): + +* Bare client: one TOOL span per call, children of the caller's span. +* LangGraph path (AC-04): still exactly one TOOL span. ``TavilySearchResults`` + posts with ``requests`` itself and never calls ``TavilyClient``. +* A LangChain ``@tool`` that calls ``TavilyClient.search``: the LangChain + TOOL span plus one ``tavily.search`` span, which starts its own trace + because traceAI-langchain does not make its spans current; wrapping the + call in ``trace.use_span(traceai_langchain.get_current_span(), + end_on_exit=False, record_exception=False, set_status_on_exception=False)`` + nests it, and a failing call leaves one exception event on each span. + +Each test prints one ``MEASUREMENT`` line (run with ``-s`` to see it). +""" + +from __future__ import annotations + +import json +import os +import sys +from pathlib import Path +from typing import Any, Dict, List, Mapping, Optional, Sequence, Tuple + +import pytest + +pytest.importorskip("tavily", reason="tavily-python must be installed to measure it") +pytest.importorskip("opentelemetry.proto", reason="the harness decodes OTLP protobuf") + +# The shared harness lives in python/tests (as for the other harness users). +sys.path.insert(0, str(Path(__file__).resolve().parents[3] / "tests")) + +from _tavily_fake import CONTENT_MARKERS, TAVILY_KEY, FakeTavily # noqa: E402 +from harness import Receiver, _flatten_attributes, run # noqa: E402 + +import fi_instrumentation # noqa: E402 + +TESTS = Path(__file__).resolve().parent +PACKAGE = TESTS.parent +PYTHON_ROOT = PACKAGE.parents[1] +# Subprocesses import fi_instrumentation from where this process did: the +# source tree in a repo run, the installed wheel in a published-release run. +FI_ROOT = Path(fi_instrumentation.__file__).resolve().parents[1] +LANGCHAIN_PACKAGE = PYTHON_ROOT / "frameworks" / "langchain" +MEASUREMENT = TESTS / "measurement" +FI_API_KEY = "placeholder-fi-api-key" +FI_SECRET_KEY = "placeholder-fi-secret-key" +BARE_CLIENT_CALLS = ["/search", "/extract", "/search", "/extract"] + +# The LangChain tool span is TOOL; tavily.search is RETRIEVER. +TRACED_KINDS = ("TOOL", "RETRIEVER") + +# Attribute keys traceAI-langchain puts on the Tavily TOOL span (measured). +LANGCHAIN_TOOL_KEYS = [ + "checkpoint_ns", + "gen_ai.span.kind", + "gen_ai.tool.description", + "gen_ai.tool.name", + "input.value", + "langgraph_checkpoint_ns", + "langgraph_node", + "langgraph_path", + "langgraph_step", + "langgraph_triggers", + "ls_integration", + "metadata", + "output.mime_type", + "output.value", +] +# Every span the LangGraph measurement exports: the graph root, two agent +# turns, two router runs, the tools node and the Tavily tool. +LANGGRAPH_SPANS = sorted( + ["LangGraph", "agent", "agent", "route", "route", "tools", "tavily_search_results_json"] +) + + +def measure( + script: str, + args: Sequence[str] = (), + project: str = "tavily-measurement", + extra_env: Optional[Mapping[str, str]] = None, +) -> Tuple[Dict[str, Any], List[dict], List[dict], List[str]]: + """Run one measurement script; return its summary, spans, exports and fake calls.""" + with FakeTavily() as fake, Receiver() as receiver: + env = { + key: value + for key, value in os.environ.items() + if not key.startswith(("FI_", "TAVILY_", "OTEL_")) + } + env.update( + { + "FI_BASE_URL": receiver.origin, + "FI_API_KEY": FI_API_KEY, + "FI_SECRET_KEY": FI_SECRET_KEY, + "TAVILY_API_KEY": TAVILY_KEY, + "TAVILY_BASE_URL": fake.origin, + "MEASURE_PROJECT": project, + "PYTHONPATH": os.pathsep.join( + [ + str(PACKAGE), + str(LANGCHAIN_PACKAGE), + str(FI_ROOT), + os.environ.get("PYTHONPATH", ""), + ] + ), + } + ) + env.update(extra_env or {}) + result = run([sys.executable, str(MEASUREMENT / script), *args], env, None, timeout=180) + spans = receiver.spans() + exports = receiver.requests() + paths = fake.paths() + assert not result.timed_out + assert result.returncode == 0, result.stderr.decode(errors="replace") + summary = json.loads(result.stdout.decode().strip().splitlines()[-1]) + assert exports + for export in exports: + assert export["path"] == "/tracer/v1/traces" + for resource in export["resource_attributes"]: + assert resource["project_name"] == project + assert resource["project_type"] == "observe" + return summary, spans, exports, paths + + +def kind(span: dict) -> Any: + # traceAI's span-kind key (fi_instrumentation SpanAttributes.GEN_AI_SPAN_KIND). + return _flatten_attributes(span.get("attributes", [])).get("gen_ai.span.kind") + + +def report(label: str, spans: List[dict], **extra: Any) -> None: + table = [{"name": span["name"], "kind": kind(span)} for span in spans] + print("MEASUREMENT {0} {1}".format(label, json.dumps(dict(extra, spans=table)))) + + +def tavily_spans(spans: List[dict]) -> List[dict]: + return [span for span in spans if span["name"] != "measurement.control"] + + +def test_bare_client_emits_no_span_with_register_only(): + summary, spans, _, paths = measure("bare_client.py", project="tavily-bare") + report("bare_client register_only", spans, summary=summary, fake_paths=paths) + + # The real client made four real HTTP calls and got results back... + assert summary["instrumented"] == [] + assert paths == BARE_CLIENT_CALLS + assert summary["search_results"] == summary["async_search_results"] == 2 + assert summary["extract_results"] == summary["async_extract_results"] == 2 + # ...the export path works (the control span arrived)... + assert [span["name"] for span in spans] == ["measurement.control"] + # ...and Tavily added nothing to it. AC-02: 0. + assert tavily_spans(spans) == [] + + +def test_bare_client_emits_no_span_with_the_langchain_instrumentor(): + pytest.importorskip("langchain_core", reason="traceAI-langchain needs langchain-core") + summary, spans, _, paths = measure( + "bare_client.py", ["--with-langchain-instrumentor"], "tavily-bare-langchain" + ) + report("bare_client langchain_instrumentor", spans, summary=summary, fake_paths=paths) + + assert summary["instrumented"] == ["traceai_langchain"] + assert paths == BARE_CLIENT_CALLS + assert [span["name"] for span in spans] == ["measurement.control"] + assert tavily_spans(spans) == [] + + +def langgraph_tool_span(spans: List[dict]) -> dict: + tools = [span for span in spans if kind(span) == "TOOL"] + assert [span["name"] for span in tools] == ["tavily_search_results_json"] + return tools[0] + + +def test_langgraph_tool_path_emits_one_tool_span(): + pytest.importorskip("langchain_community", reason="the example's Tavily tool") + pytest.importorskip("langgraph", reason="the example is a LangGraph graph") + summary, spans, _, paths = measure("langgraph_tool.py", project="tavily-langgraph") + tool = langgraph_tool_span(spans) + values = _flatten_attributes(tool["attributes"]) + report( + "langgraph_tool", + spans, + summary=summary, + fake_paths=paths, + tool_keys=sorted(values), + tool_input_value=values.get("input.value"), + ) + + assert summary["tool_messages"] == 1 and summary["tool_status"] == ["success"] + assert paths == ["/search"] + # AC-01: one TOOL span for one tool call, with these attribute keys. + assert sorted(values) == LANGCHAIN_TOOL_KEYS + assert values["gen_ai.tool.name"] == "tavily_search_results_json" + assert "th-8327 langgraph tool measurement" in values["input.value"] + assert sorted(span["name"] for span in spans) == LANGGRAPH_SPANS + wire = json.dumps(spans) + assert TAVILY_KEY not in wire + # traceAI-langchain records the tool's output, so result text is exported. + assert [marker for marker in CONTENT_MARKERS if marker in wire] + + +def _content_keys(spans: List[dict]) -> List[Tuple[str, str]]: + """(span name, attribute key) pairs whose value carries fake result text.""" + found = [] + for span in spans: + for key, value in _flatten_attributes(span["attributes"]).items(): + if any(marker in json.dumps(value) for marker in CONTENT_MARKERS): + found.append((span["name"], key)) + return sorted(found) + + +def test_langgraph_tool_path_fi_hide_outputs_masks_only_the_tool_output(): + pytest.importorskip("langchain_community", reason="the example's Tavily tool") + pytest.importorskip("langgraph", reason="the example is a LangGraph graph") + summary, spans, _, _ = measure( + "langgraph_tool.py", + project="tavily-langgraph-hide-outputs", + extra_env={"FI_HIDE_OUTPUTS": "true"}, + ) + values = _flatten_attributes(langgraph_tool_span(spans)["attributes"]) + report("langgraph_tool FI_HIDE_OUTPUTS", spans, content_keys=_content_keys(spans)) + + assert values["output.value"] == "__REDACTED__" + # The result text is the next agent turn's input, so it still leaves the + # process through the agent and router spans' input.value. + assert _content_keys(spans) == [("agent", "input.value"), ("route", "input.value")] + + +def test_langgraph_tool_path_drops_result_text_with_inputs_and_outputs_hidden(): + pytest.importorskip("langchain_community", reason="the example's Tavily tool") + pytest.importorskip("langgraph", reason="the example is a LangGraph graph") + summary, spans, _, _ = measure( + "langgraph_tool.py", + project="tavily-langgraph-hidden", + extra_env={"FI_HIDE_OUTPUTS": "true", "FI_HIDE_INPUTS": "true"}, + ) + values = _flatten_attributes(langgraph_tool_span(spans)["attributes"]) + report( + "langgraph_tool FI_HIDE_INPUTS+FI_HIDE_OUTPUTS", spans, content_keys=_content_keys(spans) + ) + + assert values["input.value"] == values["output.value"] == "__REDACTED__" + assert _content_keys(spans) == [] + assert sorted(span["name"] for span in spans) == LANGGRAPH_SPANS + + +# --- With traceAI-tavily on (after the decision) ------------------------------ + + +def test_bare_client_with_the_wrapper_emits_one_span_per_call(): + summary, spans, _, paths = measure( + "bare_client.py", ["--with-tavily-instrumentor"], "tavily-bare-wrapped" + ) + report("bare_client traceai_tavily", spans, summary=summary, fake_paths=paths) + + assert summary["instrumented"] == ["traceai_tavily"] + assert paths == BARE_CLIENT_CALLS + (control,) = [span for span in spans if span["name"] == "measurement.control"] + traced = tavily_spans(spans) + assert [span["name"] for span in traced] == [ + "tavily.search", + "tavily.extract", + "tavily.search", + "tavily.extract", + ] + for span in traced: + assert kind(span) == ("RETRIEVER" if span["name"] == "tavily.search" else "TOOL") + assert span["parentSpanId"] == control["spanId"] + + +def test_langgraph_tool_path_is_not_double_counted_with_the_wrapper(): + """AC-04: TavilySearchResults posts with requests itself, not TavilyClient.""" + pytest.importorskip("langchain_community", reason="the example's Tavily tool") + pytest.importorskip("langgraph", reason="the example is a LangGraph graph") + summary, spans, _, paths = measure( + "langgraph_tool.py", ["--with-tavily-instrumentor"], "tavily-langgraph-wrapped" + ) + report("langgraph_tool traceai_tavily", spans, summary=summary, fake_paths=paths) + + assert summary["instrumented"] == ["traceai_langchain", "traceai_tavily"] + assert paths == ["/search"] + langgraph_tool_span(spans) + assert sorted(span["name"] for span in spans) == LANGGRAPH_SPANS + + +def test_a_langchain_tool_over_the_bare_client_adds_one_tavily_span(): + """A custom LangChain tool that calls TavilyClient, with both instrumentors. + + traceAI-langchain does not make its spans current (callbacks cannot + guarantee a detach), so the tavily.search span starts its own trace unless + the tool makes the LangChain span current itself. + """ + pytest.importorskip("langgraph", reason="the example is a LangGraph graph") + _, plain, _, _ = measure("langgraph_tool.py", ["--custom-tool"], "tavily-custom-tool") + summary, spans, _, paths = measure( + "langgraph_tool.py", + ["--custom-tool", "--with-tavily-instrumentor"], + "tavily-custom-tool-wrapped", + ) + _, nested, _, _ = measure( + "langgraph_tool.py", + ["--custom-tool", "--use-langchain-span", "--with-tavily-instrumentor"], + "tavily-custom-tool-nested", + ) + report("langgraph custom_tool", plain) + report("langgraph custom_tool traceai_tavily", spans, summary=summary, fake_paths=paths) + report("langgraph custom_tool use_langchain_span traceai_tavily", nested) + + assert paths == ["/search"] + # Without the wrapper the bare client inside the tool adds nothing. + assert [span["name"] for span in plain if kind(span) == "TOOL"] == ["tavily_web_search"] + + # With it, exactly one more span: tavily.search, a root in its own trace. + assert len(spans) == len(plain) + 1 + tools = {span["name"]: span for span in spans if kind(span) in TRACED_KINDS} + assert sorted(tools) == ["tavily.search", "tavily_web_search"] + assert "parentSpanId" not in tools["tavily.search"] + assert tools["tavily.search"]["traceId"] != tools["tavily_web_search"]["traceId"] + + # Made current with traceai_langchain.get_current_span(), it nests. + assert len(nested) == len(plain) + 1 + tools = {span["name"]: span for span in nested if kind(span) in TRACED_KINDS} + assert tools["tavily.search"]["parentSpanId"] == tools["tavily_web_search"]["spanId"] + assert tools["tavily.search"]["traceId"] == tools["tavily_web_search"]["traceId"] + + +def _exception_events(span: dict) -> List[dict]: + return [event for event in span.get("events", []) if event.get("name") == "exception"] + + +def test_a_failing_langchain_tool_over_the_bare_client_records_the_error_once(): + """The README's use_span pattern leaves one exception event on each span. + + traceAI-langchain records the tool's error on its span (on_tool_error). A + ``trace.use_span`` that records exceptions would add a second, identical + event to that span, so the tool makes it current with + ``record_exception=False, set_status_on_exception=False``. + """ + pytest.importorskip("langgraph", reason="the example is a LangGraph graph") + summary, spans, _, paths = measure( + "langgraph_tool.py", + ["--custom-tool", "--use-langchain-span", "--with-tavily-instrumentor", "--fail"], + "tavily-custom-tool-error", + ) + report( + "langgraph custom_tool error use_langchain_span traceai_tavily", + spans, + summary=summary, + fake_paths=paths, + ) + + assert paths == ["/search"] + assert summary["tool_status"] == ["error"] + tools = {span["name"]: span for span in spans if kind(span) in TRACED_KINDS} + assert sorted(tools) == ["tavily.search", "tavily_web_search"] + tool, search = tools["tavily_web_search"], tools["tavily.search"] + assert search["parentSpanId"] == tool["spanId"] + assert search["traceId"] == tool["traceId"] + for span in (tool, search): + events = _exception_events(span) + assert len(events) == 1, (span["name"], events) + exception_type = _flatten_attributes(events[0]["attributes"])["exception.type"] + assert exception_type.endswith("InvalidAPIKeyError") diff --git a/python/frameworks/tavily/tests/test_packaging.py b/python/frameworks/tavily/tests/test_packaging.py new file mode 100644 index 000000000..a765953c2 --- /dev/null +++ b/python/frameworks/tavily/tests/test_packaging.py @@ -0,0 +1,197 @@ +"""Packaging contract: dependency ranges, the runtime version check, Pythons, +and the README's test commands.""" + +from __future__ import annotations + +import ast +import os +import re +import subprocess +import sys +from pathlib import Path + +import pytest +from packaging.requirements import Requirement +from packaging.specifiers import SpecifierSet + +from traceai_tavily import TavilyInstrumentor +from traceai_tavily.package import _instruments + +PACKAGE = Path(__file__).resolve().parents[1] +PYPROJECT = PACKAGE / "pyproject.toml" + + +def _dependency(name: str) -> SpecifierSet: + match = re.search( + r'^{0}\s*=\s*"([^"]+)"'.format(re.escape(name)), PYPROJECT.read_text(), re.M + ) + assert match, name + return SpecifierSet(match.group(1)) + + +def test_tavily_python_is_a_range_from_the_tested_release_to_the_next_major(): + (requirement,) = TavilyInstrumentor().instrumentation_dependencies() + instruments = Requirement(requirement) + assert instruments.name == "tavily-python" + # pyproject and the runtime dependency check agree. + assert instruments.specifier == _dependency("tavily-python") + for version in ("0.8.4", "0.8.5", "0.9.0", "0.99.0"): + assert instruments.specifier.contains(version), version + for version in ("0.7.27", "0.8.3", "1.0.0"): + assert not instruments.specifier.contains(version), version + assert tuple(_instruments) == tuple(TavilyInstrumentor().instrumentation_dependencies()) + + +def test_fi_instrumentation_floor_is_1_1_0(): + # 0.1.11/0.1.12 export to the retired create_otel_span path; 1.1.0 is the + # release this package was run against. + spec = _dependency("fi-instrumentation-otel") + assert spec.contains("1.1.0") + assert spec.contains("1.2.0") + for version in ("0.1.11", "0.1.16", "1.0.0"): + assert not spec.contains(version), version + + +TESTED_PYTHONS = ["3.10", "3.11", "3.12", "3.13"] + + +def _classified_pythons() -> list: + return re.findall(r'"Programming Language :: Python :: (3\.\d+)"', PYPROJECT.read_text()) + + +def test_classifiers_list_only_the_tested_python_versions(): + assert _classified_pythons() == TESTED_PYTHONS + + +def test_python_constraint_admits_exactly_the_classified_versions(): + """poetry-core adds a classifier for every minor the ``python`` constraint + admits, so the built wheel lists those whatever pyproject's classifiers say. + The constraint and the classifiers must therefore name the same versions. + """ + match = re.search(r'^python\s*=\s*"([^"]+)"', PYPROJECT.read_text(), re.M) + assert match + admitted = [ + "3.{0}".format(minor) + for minor in range(6, 20) + if SpecifierSet(match.group(1)).contains("3.{0}.0".format(minor)) + ] + assert admitted == _classified_pythons() + + +@pytest.mark.parametrize( + "installed, wraps", + [("0.8.4", True), ("0.9.1", True), ("0.7.27", False), ("1.0.0", False)], +) +def test_instrument_only_wraps_an_in_range_tavily_python(monkeypatch, installed, wraps): + """BaseInstrumentor checks _instruments against the installed release.""" + pytest.importorskip("tavily") + import opentelemetry.instrumentation.dependencies as dependencies + from tavily import TavilyClient + + original = TavilyClient.__dict__["search"] + monkeypatch.setattr(dependencies, "version", lambda name: installed) + instrumentor = TavilyInstrumentor() + instrumentor.instrument() + try: + assert (TavilyClient.__dict__["search"] is not original) is wraps + finally: + instrumentor.uninstrument() + assert TavilyClient.__dict__["search"] is original + + +README = PACKAGE / "README.md" +# The extras the README's second test command adds; measurement tests that +# import them are skipped by the base command. +LANGCHAIN_EXTRAS = ("langchain_core", "langchain_community", "langgraph") + + +def _readme_tests_section() -> str: + match = re.search(r"^## Tests\n(.*?)(?=^## |\Z)", README.read_text(), re.M | re.S) + assert match, "README.md has no '## Tests' section" + return match.group(1) + + +def _tests_needing_langchain_extras() -> list: + """Test functions in test_measurement.py that importorskip a LangChain extra.""" + tree = ast.parse((PACKAGE / "tests" / "test_measurement.py").read_text()) + gated = [] + for node in tree.body: + if not (isinstance(node, ast.FunctionDef) and node.name.startswith("test_")): + continue + for call in ast.walk(node): + if ( + isinstance(call, ast.Call) + and isinstance(call.func, ast.Attribute) + and call.func.attr == "importorskip" + and call.args + and isinstance(call.args[0], ast.Constant) + and call.args[0].value in LANGCHAIN_EXTRAS + ): + gated.append(node.name) + break + return gated + + +def test_readme_gives_plain_pytest_and_both_pinned_test_commands(): + section = _readme_tests_section() + commands = re.findall(r"```bash\n(.*?)```", section, re.S) + assert len(commands) == 3, commands + plain, base, extras = (" ".join(command.replace("\\\n", " ").split()) for command in commands) + # Plain pytest in the package directory, as for the other packages. + assert plain == "cd python/frameworks/tavily pip install -e . pytest pytest" + assert "PYTHONPATH" not in plain + for command in (base, extras): + assert command.startswith('PYTHONPATH="python/frameworks/tavily:python:python/tests" ') + assert "--with 'tavily-python==0.8.4'" in command + assert command.endswith( + " pytest python/frameworks/tavily/tests -q -p no:cacheprovider --noconftest" + " -o addopts= -rs" + ) + assert "--with wrapt " in base and "langchain" not in base + assert "--with 'wrapt<2'" in extras + for pin in ("langchain-community==0.4.2", "langchain-core==1.5.2", "langgraph==1.2.2"): + assert "--with " + pin in extras, pin + + +def test_plain_pytest_collects_from_the_package_directory(): + """``pytest`` run in the package directory, as for the other traceAI Python + packages, collects every module without a PYTHONPATH that names + ``python/tests``: the modules that use the shared harness put it on + ``sys.path`` themselves. + """ + import fi_instrumentation + + # Only what an installed package provides: traceai_tavily and + # fi_instrumentation. The shared harness directory is left out on purpose. + fi_root = Path(fi_instrumentation.__file__).resolve().parents[1] + env = { + key: value + for key, value in os.environ.items() + if key != "PYTHONPATH" and not key.startswith("PYTEST_") + } + env["PYTHONPATH"] = os.pathsep.join([str(PACKAGE), str(fi_root)]) + harness_dir = str(PACKAGE.parents[1] / "tests") + assert harness_dir not in env["PYTHONPATH"].split(os.pathsep) + + result = subprocess.run( + [sys.executable, "-m", "pytest", "--collect-only", "-q", "-p", "no:cacheprovider"], + cwd=str(PACKAGE), + env=env, + capture_output=True, + text=True, + timeout=300, + ) + output = result.stdout + result.stderr + assert result.returncode == 0, output + assert "ERROR collecting" not in output, output + collected = {line.split("::", 1)[0] for line in result.stdout.splitlines() if "::" in line} + for module in ("test_contract_harness.py", "test_example.py", "test_measurement.py"): + assert "tests/" + module in collected, (module, sorted(collected)) + + +def test_readme_counts_the_tests_that_need_the_langchain_extras(): + gated = _tests_needing_langchain_extras() + assert gated, "no measurement test needs the LangChain extras" + match = re.search(r"skips the (\d+) measurement tests", _readme_tests_section()) + assert match, "the Tests section does not say how many tests the extras run" + assert int(match.group(1)) == len(gated), gated diff --git a/python/frameworks/tavily/tests/test_privacy.py b/python/frameworks/tavily/tests/test_privacy.py new file mode 100644 index 000000000..4e96ac8ba --- /dev/null +++ b/python/frameworks/tavily/tests/test_privacy.py @@ -0,0 +1,354 @@ +"""Privacy: the Tavily key is redacted wherever the SDK keeps it; queries are capped. + +tavily-python 0.8.4 keeps the key in different places per client: + +* TavilyClient: ``api_key``, ``headers["Authorization"]`` and the requests + session's ``Authorization`` header (a caller's own session may carry one + that ``api_key`` does not). +* AsyncTavilyClient: only the httpx client's ``Authorization`` header; there + is no ``api_key`` attribute. + +Each test puts the key into the recorded text, so a redaction that reads the +wrong place fails. +""" + +from __future__ import annotations + +import asyncio +import json +from typing import Any, Dict + +import pytest + +pytest.importorskip("tavily", reason="tavily-python must be installed to test its instrumentor") + +from _support import attrs, instrumented # noqa: E402 +from _tavily_fake import ECHO_400, TAVILY_KEY, FakeTavily # noqa: E402 + +OTHER_KEY = "tvly-session-key-must-not-be-exported" + + +@pytest.fixture() +def fake(): + with FakeTavily() as server: + yield server + + +@pytest.fixture(autouse=True) +def no_env_key(monkeypatch): + monkeypatch.delenv("TAVILY_API_KEY", raising=False) + + +def _wire(traced: Any) -> str: + return traced.wire() + + +def test_sync_query_with_the_key_is_redacted(fake): + from tavily import TavilyClient + + with instrumented() as traced: + TavilyClient(api_key=TAVILY_KEY, api_base_url=fake.origin).search( + "find {0} now".format(TAVILY_KEY) + ) + + assert attrs(traced.one())["input.value"] == "find [redacted] now" + assert TAVILY_KEY not in _wire(traced) + # The request itself is unchanged: redaction is for the span only. + assert fake.bodies()[0]["query"] == "find {0} now".format(TAVILY_KEY) + + +def test_async_query_with_the_key_is_redacted(fake): + """AsyncTavilyClient has no api_key attribute; the key is in its httpx headers.""" + from tavily import AsyncTavilyClient + + async def call() -> None: + client = AsyncTavilyClient(api_key=TAVILY_KEY, api_base_url=fake.origin) + assert not hasattr(client, "api_key") + try: + await client.search("find {0} now".format(TAVILY_KEY)) + finally: + await client.close() + + with instrumented() as traced: + asyncio.run(call()) + + assert attrs(traced.one())["input.value"] == "find [redacted] now" + assert TAVILY_KEY not in _wire(traced) + + +def test_env_key_is_redacted(fake, monkeypatch): + from tavily import TavilyClient + + monkeypatch.setenv("TAVILY_API_KEY", TAVILY_KEY) + with instrumented() as traced: + TavilyClient(api_base_url=fake.origin).search("q " + TAVILY_KEY) + + assert attrs(traced.one())["input.value"] == "q [redacted]" + + +def test_key_on_a_callers_requests_session_is_redacted(fake): + import requests + from tavily import TavilyClient + + session = requests.Session() + session.headers["Authorization"] = "Bearer " + OTHER_KEY + with instrumented() as traced: + client = TavilyClient(api_base_url=fake.origin, session=session) + assert client.api_key is None + client.search("q " + OTHER_KEY) + + assert fake.calls[0][1]["authorization"] == "Bearer " + OTHER_KEY + assert attrs(traced.one())["input.value"] == "q [redacted]" + assert OTHER_KEY not in _wire(traced) + + +def test_key_on_a_callers_httpx_client_is_redacted(fake): + import httpx + from tavily import AsyncTavilyClient + + async def call() -> None: + async with httpx.AsyncClient( + base_url=fake.origin, headers={"Authorization": "Bearer " + OTHER_KEY} + ) as http: + await AsyncTavilyClient(client=http).extract( + ["https://example.com/a"], query="q " + OTHER_KEY + ) + + with instrumented() as traced: + asyncio.run(call()) + + assert attrs(traced.one())["input.value"] == "q [redacted]" + assert OTHER_KEY not in _wire(traced) + + +@pytest.mark.parametrize("mode", ["sync", "async"]) +def test_key_in_an_error_message_is_redacted_from_status_and_event(fake, mode): + """The fake's 400 repeats the query; the caller still sees the original error.""" + from tavily import AsyncTavilyClient, BadRequestError, TavilyClient + + query = "{0} {1}".format(ECHO_400, TAVILY_KEY) + + async def call_async() -> None: + client = AsyncTavilyClient(api_key=TAVILY_KEY, api_base_url=fake.origin) + try: + await client.search(query) + finally: + await client.close() + + with instrumented() as traced: + with pytest.raises(BadRequestError) as info: + if mode == "sync": + TavilyClient(api_key=TAVILY_KEY, api_base_url=fake.origin).search(query) + else: + asyncio.run(call_async()) + + assert TAVILY_KEY in str(info.value) # unchanged for the caller + span = traced.one() + assert span.status.description == "BadRequestError: Bad query: {0} [redacted]".format( + ECHO_400 + ) + (event,) = [event for event in span.events if event.name == "exception"] + assert event.attributes["exception.message"] == "Bad query: {0} [redacted]".format(ECHO_400) + assert "[redacted]" in event.attributes["exception.stacktrace"] + assert TAVILY_KEY not in _wire(traced) + assert "tvly-" not in _wire(traced) + + +PRIVATE = "private question" + + +@pytest.mark.parametrize("mode", ["sync", "async"]) +@pytest.mark.parametrize("hide", ["config", "env"]) +@pytest.mark.parametrize("keyed", [True, False], ids=["keyed", "keyless"]) +def test_hidden_inputs_remove_the_query_from_error_text(fake, monkeypatch, mode, hide, keyed): + """The fake's 400 repeats the query; with inputs hidden it must not leave.""" + from fi_instrumentation import TraceConfig + from tavily import AsyncTavilyClient, BadRequestError, TavilyClient + + api_key = TAVILY_KEY if keyed else None + query = "{0} {1}".format(ECHO_400, PRIVATE) + (" " + TAVILY_KEY if keyed else "") + options: Dict[str, Any] = {} + if hide == "config": + options["config"] = TraceConfig(hide_inputs=True) + else: + monkeypatch.setenv("FI_HIDE_INPUTS", "true") + + async def call_async() -> None: + client = AsyncTavilyClient(api_key=api_key, api_base_url=fake.origin) + try: + await client.search(query) + finally: + await client.close() + + with instrumented(**options) as traced: + with pytest.raises(BadRequestError) as info: + if mode == "sync": + TavilyClient(api_key=api_key, api_base_url=fake.origin).search(query) + else: + asyncio.run(call_async()) + + assert str(info.value) == "Bad query: " + query # unchanged for the caller + span = traced.one() + assert attrs(span)["input.value"] == "__REDACTED__" + assert span.status.description == "BadRequestError: Bad query: __REDACTED__" + (event,) = [event for event in span.events if event.name == "exception"] + assert event.attributes["exception.message"] == "Bad query: __REDACTED__" + assert "Bad query: __REDACTED__" in event.attributes["exception.stacktrace"] + wire = _wire(traced) + assert PRIVATE not in wire + assert "tvly-" not in wire + + +@pytest.mark.parametrize("mode", ["sync", "async"]) +def test_without_hidden_inputs_error_text_keeps_the_query(fake, mode): + from tavily import AsyncTavilyClient, BadRequestError, TavilyClient + + query = "{0} {1}".format(ECHO_400, PRIVATE) + + async def call_async() -> None: + client = AsyncTavilyClient(api_key=TAVILY_KEY, api_base_url=fake.origin) + try: + await client.search(query) + finally: + await client.close() + + with instrumented() as traced: + with pytest.raises(BadRequestError): + if mode == "sync": + TavilyClient(api_key=TAVILY_KEY, api_base_url=fake.origin).search(query) + else: + asyncio.run(call_async()) + + span = traced.one() + assert attrs(span)["input.value"] == query + assert span.status.description == "BadRequestError: Bad query: " + query + (event,) = [event for event in span.events if event.name == "exception"] + assert event.attributes["exception.message"] == "Bad query: " + query + assert "Bad query: " + query in event.attributes["exception.stacktrace"] + + +def test_hidden_inputs_leave_error_text_alone_for_an_empty_query(): + """An empty query has nothing to remove; the error text stays readable.""" + import requests + from fi_instrumentation import TraceConfig + from tavily import TavilyClient + + # Nothing listens on port 1, so the call fails before any response. + client = TavilyClient(api_key=TAVILY_KEY, api_base_url="http://127.0.0.1:1") + with instrumented(config=TraceConfig(hide_inputs=True)) as traced: + with pytest.raises(requests.ConnectionError) as info: + client.search("") + + span = traced.one() + assert span.status.description.startswith("ConnectionError: ") + assert "__REDACTED__" not in span.status.description + (event,) = [event for event in span.events if event.name == "exception"] + assert "__REDACTED__" not in event.attributes["exception.message"] + assert event.attributes["exception.message"] == str(info.value)[:1024] + + +def test_query_is_capped_at_1024_utf8_bytes_on_a_character_boundary(fake): + from tavily import TavilyClient + + queries = { + "two-byte": "é" * 2000, + "three-byte": "a" + "€" * 500, + "four-byte": "ab" + "\U0001F600" * 300, + } + expected = { + "two-byte": "é" * 512, + "three-byte": "a" + "€" * 341, + "four-byte": "ab" + "\U0001F600" * 255, + } + with instrumented() as traced: + client = TavilyClient(api_key=TAVILY_KEY, api_base_url=fake.origin) + for query in queries.values(): + client.search(query) + + values = [attrs(span)["input.value"] for span in traced.spans()] + assert values == list(expected.values()) + for value in values: + assert len(value.encode("utf-8")) <= 1024 + # The vendor got the full query. + assert [body["query"] for body in fake.bodies()] == list(queries.values()) + + +def test_redaction_happens_before_the_cap(fake): + """A key that straddles the 1 KB cut must not leave a prefix behind.""" + from tavily import TavilyClient + + query = "x" * 1015 + TAVILY_KEY + with instrumented() as traced: + TavilyClient(api_key=TAVILY_KEY, api_base_url=fake.origin).search(query) + + value = attrs(traced.one())["input.value"] + assert value == "x" * 1015 + "[redacted" + assert "tvly-" not in json.dumps(value) + + +FRAME_MARKERS = ("Traceback", 'File "', "/Users/", "/home/", "site-packages", "Documents/") + + +@pytest.mark.parametrize("mode", ["sync", "async"]) +@pytest.mark.parametrize("hide", [False, True], ids=["defaults", "hide-inputs"]) +def test_error_events_never_carry_a_raw_traceback(fake, monkeypatch, mode, hide): + """The exception event must hold no frames, local paths or username. + + The in-memory provider keeps span events, so this scans what an exporter + would send for them (fi_verify does not return events). + """ + import getpass + import os + + from tavily import AsyncTavilyClient, BadRequestError, TavilyClient + + if hide: + monkeypatch.setenv("FI_HIDE_INPUTS", "true") + query = "{0} {1}".format(ECHO_400, PRIVATE) + + async def call_async() -> None: + client = AsyncTavilyClient(api_key=TAVILY_KEY, api_base_url=fake.origin) + try: + await client.search(query) + finally: + await client.close() + + with instrumented() as traced: + with pytest.raises(BadRequestError) as info: + if mode == "sync": + TavilyClient(api_key=TAVILY_KEY, api_base_url=fake.origin).search(query) + else: + asyncio.run(call_async()) + + # The caller still gets the real traceback. + assert info.value.__traceback__ is not None + span = traced.one() + (event,) = [event for event in span.events if event.name == "exception"] + stacktrace = event.attributes["exception.stacktrace"] + assert stacktrace.startswith("BadRequestError") + wire = traced.wire() + for marker in FRAME_MARKERS + (os.path.expanduser("~"), getpass.getuser(), __file__): + assert marker not in wire, marker + assert TAVILY_KEY not in wire + + +def test_error_event_with_unreadable_key_has_no_traceback(fake): + """When the key cannot be read the event is a placeholder, still frames-free.""" + from tavily import TavilyClient + + client = TavilyClient(api_key=TAVILY_KEY, api_base_url=fake.origin) + + class _Unreadable: + def get(self, *_): + raise RuntimeError("no headers") + + client.headers = _Unreadable() + with instrumented() as traced: + with pytest.raises(Exception): + client.search("fail-401") + span = traced.one() + (event,) = [event for event in span.events if event.name == "exception"] + assert event.attributes["exception.message"] == "[redacted]" + wire = traced.wire() + for marker in FRAME_MARKERS: + assert marker not in wire, marker diff --git a/python/frameworks/tavily/tests/test_spans.py b/python/frameworks/tavily/tests/test_spans.py new file mode 100644 index 000000000..84e00db2c --- /dev/null +++ b/python/frameworks/tavily/tests/test_spans.py @@ -0,0 +1,418 @@ +"""Span contract for TavilyClient / AsyncTavilyClient search and extract. + +The real tavily-python client makes real HTTP calls to the loopback fake +(``api_base_url=fake.origin``); spans go to an in-memory provider so the +wrapper's own status is visible (see _support). +""" + +from __future__ import annotations + +import asyncio +import inspect +from typing import Any, List + +import pytest + +pytest.importorskip("tavily", reason="tavily-python must be installed to test its instrumentor") + +from opentelemetry import trace as trace_api # noqa: E402 +from opentelemetry.trace import StatusCode # noqa: E402 + +from _support import attrs, instrumented, new_provider # noqa: E402 +from _tavily_fake import CONTENT_MARKERS, TAVILY_KEY, FakeTavily # noqa: E402 + +QUERY = "who maintains OpenTelemetry?" +URLS = ["https://example.com/a", "https://example.com/b", "https://example.com/fail-c"] + + +@pytest.fixture() +def fake(): + with FakeTavily() as server: + yield server + + +def _sync_client(fake: FakeTavily) -> Any: + from tavily import TavilyClient + + return TavilyClient(api_key=TAVILY_KEY, api_base_url=fake.origin) + + +def _async_client(fake: FakeTavily) -> Any: + from tavily import AsyncTavilyClient + + return AsyncTavilyClient(api_key=TAVILY_KEY, api_base_url=fake.origin) + + +async def _async_call(fake: FakeTavily, method: str, *args: Any, **kwargs: Any) -> Any: + client = _async_client(fake) + try: + return await getattr(client, method)(*args, **kwargs) + finally: + await client.close() + + +def _assert_no_content(traced: Any) -> None: + wire = traced.wire() + for marker in CONTENT_MARKERS: + assert marker not in wire, marker + assert TAVILY_KEY not in wire + for url in URLS: + assert url not in wire, url + + +# --- search ------------------------------------------------------------------- + + +def test_sync_search_emits_one_retriever_span(fake): + with instrumented() as traced: + result = _sync_client(fake).search(QUERY, max_results=3) + + assert len(result["results"]) == 3 + assert fake.paths() == ["/search"] + span = traced.one() + assert span.name == "tavily.search" + assert attrs(span) == { + "gen_ai.span.kind": "RETRIEVER", + "gen_ai.tool.name": "tavily.search", + "gen_ai.retrieval.query": QUERY, + "input.value": QUERY, + "tavily.result_count": 3, + } + assert span.status.status_code is StatusCode.OK + assert span.parent is None + _assert_no_content(traced) + + +def test_async_search_matches_sync(fake): + with instrumented() as traced: + result = asyncio.run(_async_call(fake, "search", query=QUERY, max_results=3)) + + assert len(result["results"]) == 3 + span = traced.one() + assert span.name == "tavily.search" + assert attrs(span) == { + "gen_ai.span.kind": "RETRIEVER", + "gen_ai.tool.name": "tavily.search", + "gen_ai.retrieval.query": QUERY, + "input.value": QUERY, + "tavily.result_count": 3, + } + assert span.status.status_code is StatusCode.OK + _assert_no_content(traced) + + +# --- extract ------------------------------------------------------------------ + + +def test_sync_extract_counts_urls_and_results_without_recording_urls(fake): + with instrumented() as traced: + result = _sync_client(fake).extract(URLS) + + assert len(result["results"]) == 2 and len(result["failed_results"]) == 1 + span = traced.one() + assert span.name == "tavily.extract" + # No query was given, so there is no input.value; URLs are never recorded. + assert attrs(span) == { + "gen_ai.span.kind": "TOOL", + "gen_ai.tool.name": "tavily.extract", + "tavily.url_count": 3, + "tavily.result_count": 2, + "tavily.failed_result_count": 1, + } + # Partial failures are part of a successful response. + assert span.status.status_code is StatusCode.OK + _assert_no_content(traced) + + +def test_async_extract_matches_sync(fake): + with instrumented() as traced: + asyncio.run(_async_call(fake, "extract", URLS)) + + span = traced.one() + assert span.name == "tavily.extract" + assert attrs(span) == { + "gen_ai.span.kind": "TOOL", + "gen_ai.tool.name": "tavily.extract", + "tavily.url_count": 3, + "tavily.result_count": 2, + "tavily.failed_result_count": 1, + } + assert span.status.status_code is StatusCode.OK + _assert_no_content(traced) + + +@pytest.mark.parametrize("mode", ["sync", "async"]) +def test_extract_records_its_rerank_query_as_input(fake, mode): + with instrumented() as traced: + if mode == "sync": + _sync_client(fake).extract(URLS[0], query="pricing page") + else: + asyncio.run(_async_call(fake, "extract", URLS[0], query="pricing page")) + + values = attrs(traced.one()) + assert values["input.value"] == "pricing page" + # A single URL string counts as one URL. + assert values["tavily.url_count"] == 1 + + +@pytest.mark.parametrize("mode", ["sync", "async"]) +@pytest.mark.parametrize("style", ["keyword", "positional", "class_call"]) +def test_extract_reads_its_query_by_name(fake, mode, style): + from tavily import AsyncTavilyClient, TavilyClient + + if style == "keyword": + args, kwargs = (URLS[:1],), {"query": "by name"} + else: + # extract(urls, include_images, extract_depth, format, timeout, + # include_favicon, include_usage, query) in tavily-python 0.8.4 + args, kwargs = (URLS[:1], None, None, None, 30, None, None, "by name"), {} + + async def call_async() -> None: + client = _async_client(fake) + try: + if style == "class_call": + await AsyncTavilyClient.extract(client, *args, **kwargs) + else: + await client.extract(*args, **kwargs) + finally: + await client.close() + + with instrumented() as traced: + if mode == "async": + asyncio.run(call_async()) + elif style == "class_call": + TavilyClient.extract(_sync_client(fake), *args, **kwargs) + else: + _sync_client(fake).extract(*args, **kwargs) + + assert attrs(traced.one())["input.value"] == "by name" + + +@pytest.mark.parametrize("mode", ["sync", "async"]) +def test_arguments_that_do_not_bind_record_no_query_and_keep_the_vendor_error(fake, mode): + # One positional argument more than extract takes: the arguments cannot be + # bound, and the vendor call itself raises TypeError. + args = (URLS[:1], None, None, None, 30, None, None, "too many", None, "extra") + with instrumented() as traced: + with pytest.raises(TypeError, match="positional argument"): + if mode == "sync": + _sync_client(fake).extract(*args) + else: + asyncio.run(_async_call(fake, "extract", *args)) + + assert fake.paths() == [] + span = traced.one() + assert "input.value" not in attrs(span) + assert span.status.status_code is StatusCode.ERROR + + +def _shifted_client(mode: str, tracer: Any) -> Any: + """A client whose extract moved ``query`` to second place, as a release might.""" + import wrapt + + from traceai_tavily._wrappers import AsyncWrapper, SyncWrapper + + result = {"results": [], "failed_results": []} + if mode == "sync": + + class Shifted: + def extract( + self, + urls: Any, + query: Any = None, + include_images: Any = None, + extract_depth: Any = None, + format: Any = None, + timeout: Any = 30, + include_favicon: Any = None, + include_usage: Any = None, + chunks_per_source: Any = None, + **kwargs: Any, + ) -> Any: + return result + + wrapper: Any = SyncWrapper(tracer, "extract") + else: + + class Shifted: # type: ignore[no-redef] + async def extract( + self, + urls: Any, + query: Any = None, + include_images: Any = None, + extract_depth: Any = None, + format: Any = None, + timeout: Any = 30, + include_favicon: Any = None, + include_usage: Any = None, + chunks_per_source: Any = None, + **kwargs: Any, + ) -> Any: + return result + + wrapper = AsyncWrapper(tracer, "extract") + wrapt.wrap_function_wrapper(Shifted, "extract", wrapper) + return Shifted + + +@pytest.mark.parametrize("mode", ["sync", "async"]) +@pytest.mark.parametrize("style", ["keyword", "positional", "every_positional", "class_call"]) +def test_a_shifted_extract_signature_still_finds_query_by_name(mode, style): + from fi_instrumentation import FITracer, TraceConfig + + traced = new_provider() + cls = _shifted_client(mode, FITracer(traced.provider.get_tracer("test"), TraceConfig())) + client = cls() + url = URLS[:1] + calls = { + "keyword": (client.extract, (url,), {"query": "shifted"}), + "positional": (client.extract, (url, "shifted"), {}), + # The eighth positional argument is include_usage here, not query. + "every_positional": ( + client.extract, + (url, "shifted", False, "basic", "markdown", 30, False, True, 3), + {}, + ), + "class_call": (cls.extract, (client, url, "shifted"), {}), + } + method, args, kwargs = calls[style] + result = method(*args, **kwargs) + if mode == "async": + result = asyncio.run(result) + + assert result == {"results": [], "failed_results": []} + span = traced.one() + assert attrs(span)["input.value"] == "shifted" + assert attrs(span)["tavily.url_count"] == 1 + + +def test_the_signature_is_read_once_per_wrapped_function(fake, monkeypatch): + from tavily import TavilyClient + + real = inspect.signature + read: List[Any] = [] + + def spy(obj: Any, *args: Any, **kwargs: Any) -> Any: + read.append(obj) + return real(obj, *args, **kwargs) + + with instrumented() as traced: + monkeypatch.setattr(inspect, "signature", spy) + client = _sync_client(fake) + for _ in range(3): + client.extract(URLS[:1], query="cached") + TavilyClient.extract(client, URLS[:1], query="cached") + monkeypatch.undo() + + assert [attrs(span)["input.value"] for span in traced.spans()] == ["cached"] * 4 + assert [getattr(obj, "__qualname__", None) for obj in read] == ["TavilyClient.extract"] + + +# --- context ------------------------------------------------------------------ + + +@pytest.mark.parametrize("mode", ["sync", "async"]) +def test_span_is_a_child_of_the_active_span(fake, mode): + with instrumented() as traced: + tracer = traced.provider.get_tracer("test") + with tracer.start_as_current_span("agent-step") as parent: + if mode == "sync": + _sync_client(fake).search(QUERY) + else: + asyncio.run(_async_call(fake, "search", QUERY)) + + by_name = {span.name: span for span in traced.spans()} + child = by_name["tavily.search"] + assert child.parent is not None + assert child.parent.span_id == parent.get_span_context().span_id + assert child.context.trace_id == parent.get_span_context().trace_id + + +def test_span_is_current_while_requests_sends(fake): + """An HTTP-client span started during the call would nest under tavily.search.""" + import requests + from tavily import TavilyClient + + seen: List[Any] = [] + + def hook(response: Any, *args: Any, **kwargs: Any) -> Any: + seen.append(trace_api.get_current_span().get_span_context()) + return response + + session = requests.Session() + session.hooks["response"].append(hook) + with instrumented() as traced: + client = TavilyClient(api_key=TAVILY_KEY, api_base_url=fake.origin, session=session) + client.search(QUERY) + + span = traced.one() + assert [context.span_id for context in seen] == [span.context.span_id] + + +def test_span_is_current_while_httpx_sends(fake): + import httpx + from tavily import AsyncTavilyClient + + seen: List[Any] = [] + + async def hook(request: Any) -> None: + seen.append(trace_api.get_current_span().get_span_context()) + + async def call() -> None: + async with httpx.AsyncClient( + base_url=fake.origin, + headers={"Authorization": "Bearer " + TAVILY_KEY}, + event_hooks={"request": [hook]}, + ) as http: + await AsyncTavilyClient(client=http).search(QUERY) + + with instrumented() as traced: + asyncio.run(call()) + + span = traced.one() + assert [context.span_id for context in seen] == [span.context.span_id] + # Outside the call the span is no longer current. + assert trace_api.get_current_span().get_span_context().span_id != span.context.span_id + + +def test_async_methods_stay_coroutine_functions(fake): + from tavily import AsyncTavilyClient + + with instrumented(): + assert inspect.iscoroutinefunction(AsyncTavilyClient.search) + assert inspect.iscoroutinefunction(AsyncTavilyClient.extract) + + +def test_deprecated_client_subclass_is_traced_through_tavily_client(fake): + """``tavily.Client`` inherits TavilyClient.search, so it is covered too.""" + import warnings + + from tavily import Client + + with instrumented() as traced: + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + client = Client(TAVILY_KEY) + # Client(kwargs) takes only the key; the URL is read on every call. + client.base_url = fake.origin + client.search(QUERY) + + assert fake.paths() == ["/search"] + assert [span.name for span in traced.spans()] == ["tavily.search"] + + +def test_search_keyword_renamed_between_releases_is_traced(fake): + """tavily-python 0.8.5 renamed search's ``max_hours`` to ``max_age_hours``. + + The span must not depend on which spelling the installed release has. + """ + from tavily import TavilyClient + + names = inspect.signature(TavilyClient.search.__wrapped__ if hasattr( + TavilyClient.search, "__wrapped__") else TavilyClient.search).parameters + keyword = "max_age_hours" if "max_age_hours" in names else "max_hours" + with instrumented() as traced: + _sync_client(fake).search(QUERY, **{keyword: 24}) + span = traced.one() + assert attrs(span)["input.value"] == QUERY + assert fake.bodies()[0][keyword] == 24 diff --git a/python/frameworks/tavily/traceai_tavily/__init__.py b/python/frameworks/tavily/traceai_tavily/__init__.py new file mode 100644 index 000000000..660729aac --- /dev/null +++ b/python/frameworks/tavily/traceai_tavily/__init__.py @@ -0,0 +1,102 @@ +"""OpenTelemetry instrumentation for the Tavily Python client (tavily-python).""" + +import logging +from importlib import import_module +from typing import Any, Collection, Dict, Tuple + +from fi_instrumentation import FITracer, TraceConfig +from opentelemetry import trace as trace_api +from opentelemetry.instrumentation.instrumentor import BaseInstrumentor +from wrapt import wrap_function_wrapper + +from traceai_tavily._wrappers import AsyncWrapper, SyncWrapper +from traceai_tavily.package import _instruments +from traceai_tavily.version import __version__ + +logger = logging.getLogger(__name__) +logger.addHandler(logging.NullHandler()) + +# (module, class, wrapper type). Both methods are defined in each class body in +# tavily-python 0.8.4, so wrapping the class covers every instance, including +# the deprecated ``tavily.Client`` subclass. +_CLIENTS = ( + ("tavily.tavily", "TavilyClient", SyncWrapper), + ("tavily.async_tavily", "AsyncTavilyClient", AsyncWrapper), +) +_METHODS = ("search", "extract") + + +class TavilyInstrumentor(BaseInstrumentor): # type: ignore[misc] + """Trace ``search`` and ``extract`` on TavilyClient and AsyncTavilyClient. + + ``instrument()`` accepts ``tracer_provider`` and ``config`` (a + ``fi_instrumentation.TraceConfig``; by default one read from the + ``FI_HIDE_*`` environment variables). + """ + + __slots__ = ("_originals",) + + def instrumentation_dependencies(self) -> Collection[str]: + return _instruments + + def _instrument(self, **kwargs: Any) -> None: + config = kwargs.get("config") + if config is None: + config = TraceConfig() + elif not isinstance(config, TraceConfig): + raise TypeError( + "config must be a fi_instrumentation.TraceConfig, not {0}".format( + type(config).__name__ + ) + ) + tracer_provider = kwargs.get("tracer_provider") or trace_api.get_tracer_provider() + tracer = FITracer( + trace_api.get_tracer(__name__, __version__, tracer_provider), + config=config, + ) + self._originals: Dict[Tuple[str, str, str], Any] = {} + for module_name, class_name, wrapper_type in _CLIENTS: + client = _vendor_class(module_name, class_name) + if client is None: + continue + for method in _METHODS: + original = vars(client).get(method) + if original is None: + logger.warning( + "traceAI-tavily: %s.%s.%s not found; it is not traced", + module_name, + class_name, + method, + ) + continue + # wrapt 2.x rejects the keyword form; these are positional. + wrap_function_wrapper( + module_name, + "{0}.{1}".format(class_name, method), + wrapper_type( + tracer, method, bool(config.hide_inputs), bool(config.pii_redaction) + ), + ) + self._originals[(module_name, class_name, method)] = original + + def _uninstrument(self, **kwargs: Any) -> None: + originals = getattr(self, "_originals", None) or {} + for (module_name, class_name, method), original in originals.items(): + client = _vendor_class(module_name, class_name) + if client is not None: + setattr(client, method, original) + self._originals = {} + + +def _vendor_class(module_name: str, class_name: str) -> Any: + """The vendor class, or None (logged) when a tavily-python release moved it.""" + try: + return getattr(import_module(module_name), class_name) + except (ImportError, AttributeError): + logger.warning( + "traceAI-tavily: %s.%s not found; it is not traced", module_name, class_name + ) + return None + + +__all__ = ["TavilyInstrumentor", "__version__"] diff --git a/python/frameworks/tavily/traceai_tavily/_wrappers.py b/python/frameworks/tavily/traceai_tavily/_wrappers.py new file mode 100644 index 000000000..08ee8a58d --- /dev/null +++ b/python/frameworks/tavily/traceai_tavily/_wrappers.py @@ -0,0 +1,386 @@ +"""wrapt wrappers for TavilyClient / AsyncTavilyClient search and extract. + +Everything the wrapper does for itself (reading the client, reading the +arguments, starting and ending the span, reading the result, recording an +error) is isolated from the caller: a failure there is logged at debug level +and never replaces the vendor's own result or exception. +""" + +from __future__ import annotations + +import asyncio +import concurrent.futures +import inspect +import logging +from typing import Any, Callable, Dict, List, Mapping, Optional, Sequence + +from fi_instrumentation.instrumentation.pii_redaction import redact_pii_in_string +from opentelemetry import context as context_api +from opentelemetry import trace as trace_api +from opentelemetry.trace import Span, Status, StatusCode + +logger = logging.getLogger(__name__) + +_SPAN_KIND = "gen_ai.span.kind" +_TOOL = "TOOL" +_RETRIEVER = "RETRIEVER" +_RETRIEVAL_QUERY = "gen_ai.retrieval.query" +_TOOL_NAME = "gen_ai.tool.name" +_INPUT_VALUE = "input.value" +_URL_COUNT = "tavily.url_count" +_RESULT_COUNT = "tavily.result_count" +_FAILED_RESULT_COUNT = "tavily.failed_result_count" +_CANCELLED = "tavily.cancelled" +_CANCELLATION_ERRORS = (asyncio.CancelledError, concurrent.futures.CancelledError) +_REDACTED = "[redacted]" +# What fi_instrumentation's hide_inputs writes in place of input.value. +_HIDDEN = "__REDACTED__" +_MAX_TEXT_BYTES = 1024 + + +def _bearer_token(value: Any) -> Optional[str]: + """The key in an ``Authorization`` header value (``Bearer ``).""" + if not isinstance(value, str): + return None + value = value.strip() + if value[:7].lower() == "bearer ": + value = value[7:].strip() + return value or None + + +def _api_keys(instance: Any) -> Optional[List[str]]: + """Every Tavily key the client holds, longest first; None if unknown. + + tavily-python 0.8.4 keeps it in ``api_key``, ``headers["Authorization"]`` + and the requests session's headers (TavilyClient), or only in the httpx + client's headers (AsyncTavilyClient, ``_client``). A caller's own session + or httpx client can carry a key that ``api_key`` does not. If any of those + places cannot be read, the result is None and no free text is recorded: + a key there could not be removed from it. + """ + try: + candidates: List[Any] = [getattr(instance, "api_key", None)] + for holder in ( + instance, + getattr(instance, "session", None), + getattr(instance, "_client", None), + ): + headers = getattr(holder, "headers", None) if holder is not None else None + if headers is not None: + candidates.append(_bearer_token(headers.get("Authorization"))) + except Exception: + logger.debug("Could not read the Tavily client's key", exc_info=True) + return None + keys: List[str] = [] + for key in candidates: + if isinstance(key, str) and key and key not in keys: + keys.append(key) + return sorted(keys, key=len, reverse=True) + + +def _redact(text: str, keys: Sequence[str]) -> str: + for key in keys: + text = text.replace(key, _REDACTED) + return text + + +def _scrub(text: str, keys: Sequence[str], hidden: Optional[str] = None) -> str: + """``text`` with every key redacted, then every ``hidden`` query replaced.""" + text = _redact(text, keys) + if hidden: + text = text.replace(hidden, _HIDDEN) + return text + + +def _safe_text( + text: str, keys: Optional[Sequence[str]], hidden: Optional[str] = None, pii: bool = False +) -> Optional[str]: + """``text`` scrubbed (see ``_scrub``), PII-redacted when asked, then capped. + + None when keys are unknown. + """ + if keys is None: + return None + # Redact first, then cap, so a key or an email cut at the limit leaves no prefix. + text = _scrub(text, keys, hidden) + if pii: + text = redact_pii_in_string(text) + return _cap(text) + + +def _cap(text: str, limit: int = _MAX_TEXT_BYTES) -> str: + """The longest whole-character prefix of at most ``limit`` UTF-8 bytes.""" + # A cut inside a multi-byte character leaves an invalid tail that "ignore" + # drops; "replace" keeps a lone surrogate from raising. + return text.encode("utf-8", "replace")[:limit].decode("utf-8", "ignore") + + +def _function(wrapped: Any) -> Any: + """The plain function behind what wrapt passes as ``wrapped``. + + ``client.extract(...)`` gives a bound method; ``TavilyClient.extract(client, + ...)`` gives a wrapt partial proxy whose ``__wrapped__`` is the function. + """ + function = getattr(wrapped, "__func__", None) + if function is None: + function = getattr(wrapped, "__wrapped__", wrapped) + return function + + +def _url_count(args: Sequence[Any], kwargs: Mapping[str, Any]) -> Optional[int]: + value = kwargs["urls"] if "urls" in kwargs else (args[0] if args else None) + if isinstance(value, str): + return 1 + if isinstance(value, (list, tuple)): + return len(value) + return None + + +def _count(result: Any, key: str) -> Optional[int]: + """Length of a list in the response; None (not 0) when it is not a list.""" + value = result.get(key) if isinstance(result, Mapping) else None + return len(value) if isinstance(value, list) else None + + +class _Operation: + """Span bookkeeping shared by the sync and async wrappers.""" + + def __init__( + self, tracer: Any, method: str, hide_inputs: bool = False, pii: bool = False + ) -> None: + self._tracer = tracer + self._method = method + # TraceConfig.hide_inputs: also keep the query out of error text. + self._hide_inputs = hide_inputs + # TraceConfig.pii_redaction: FITracer covers attributes; error text is done here. + self._pii = pii + # search is a retrieval; extract fetches the pages it is given. + self._kind = _RETRIEVER if method == "search" else _TOOL + self._span_name = "tavily.{0}".format(method) + # Signature of each wrapped function, read once. + self._signatures: Dict[Any, inspect.Signature] = {} + + def _query( + self, wrapped: Any, instance: Any, args: Sequence[Any], kwargs: Mapping[str, Any] + ) -> Optional[str]: + """The search query, or extract's optional rerank query, bound by name. + + None when there is none, or when the arguments cannot be bound to the + method's signature or the value cannot be printed. + """ + try: + function = _function(wrapped) + signature = self._signatures.get(function) + if signature is None: + signature = inspect.signature(function) + self._signatures[function] = signature + # The function's first parameter is ``self``. + bound = signature.bind_partial( + *((instance,) if instance is not None else ()), *args, **kwargs + ) + value = bound.arguments.get("query") + return None if value is None else str(value) + except Exception: + logger.debug("Could not read the %s query", self._span_name, exc_info=True) + return None + + def _attributes( + self, + keys: Optional[Sequence[str]], + query: Optional[str], + args: Sequence[Any], + kwargs: Mapping[str, Any], + ) -> Dict[str, Any]: + attributes: Dict[str, Any] = {_SPAN_KIND: self._kind, _TOOL_NAME: self._span_name} + if query is not None: + value = _safe_text(query, keys) + if value is not None: + attributes[_INPUT_VALUE] = value + # FITracer masks input.value when inputs are hidden, but not this key. + if self._method == "search" and not self._hide_inputs: + attributes[_RETRIEVAL_QUERY] = value + if self._method == "extract": + urls = _url_count(args, kwargs) + if urls is not None: + attributes[_URL_COUNT] = urls + return attributes + + def _start( + self, + keys: Optional[Sequence[str]], + query: Optional[str], + args: Sequence[Any], + kwargs: Mapping[str, Any], + ) -> Optional[Span]: + try: + attributes = self._attributes(keys, query, args, kwargs) + except Exception: + logger.debug("Could not read %s arguments", self._span_name, exc_info=True) + attributes = {_SPAN_KIND: self._kind, _TOOL_NAME: self._span_name} + try: + return self._tracer.start_span(self._span_name, attributes=attributes) + except Exception: + logger.debug("Could not start the %s span", self._span_name, exc_info=True) + return None + + def _finish_ok(self, span: Span, result: Any) -> None: + try: + count = _count(result, "results") + if count is not None: + span.set_attribute(_RESULT_COUNT, count) + if self._method == "extract": + failed = _count(result, "failed_results") + if failed is not None: + span.set_attribute(_FAILED_RESULT_COUNT, failed) + except Exception: + logger.debug("Could not read the %s result", self._span_name, exc_info=True) + try: + span.set_status(Status(StatusCode.OK)) + except Exception: + logger.debug("Could not set the %s status", self._span_name, exc_info=True) + _end(span) + + def _finish_error( + self, + span: Span, + error: BaseException, + keys: Optional[Sequence[str]], + query: Optional[str], + ) -> None: + try: + if isinstance(error, _CANCELLATION_ERRORS): + # Cancellation is not a failure of the call: no exception event. + span.set_attribute(_CANCELLED, True) + span.set_status(Status(StatusCode.ERROR, "cancelled")) + else: + self._record_error(span, error, keys, query) + except Exception: + logger.debug("Could not record the %s error", self._span_name, exc_info=True) + _end(span) + + def _hidden_query( + self, query: Optional[str], keys: Optional[Sequence[str]] + ) -> Optional[str]: + """The query to remove from error text: only with inputs hidden. + + It is the query as it reads after the key is redacted, because the key + is removed from the text first. None when there is nothing to remove. + """ + if not self._hide_inputs or not query or keys is None: + return None + return _redact(query, keys) or None + + def _record_error( + self, + span: Span, + error: BaseException, + keys: Optional[Sequence[str]], + query: Optional[str], + ) -> None: + name = type(error).__name__ + try: + hidden = self._hidden_query(query, keys) + message = _safe_text(str(error), keys, hidden, self._pii) + except Exception: + hidden, message = None, None + description = name if message is None else _cap("{0}: {1}".format(name, message)) + try: + span.set_status(Status(StatusCode.ERROR, description)) + except Exception: + logger.debug("Could not set the %s status", self._span_name, exc_info=True) + # The SDK would copy str(error) and the raw traceback into the event. + # A raw traceback carries local file paths, the username and code + # frames, so it is never exported: "exception.stacktrace" is always a + # frames-free summary (type and the same scrubbed message as the status), + # and "exception.message" is the scrubbed message. The key (and, with + # inputs hidden, the query) is removed from both. + if message is None: + event: Dict[str, Any] = { + "exception.message": _REDACTED, + "exception.stacktrace": name, + } + else: + event = {"exception.message": message, "exception.stacktrace": description} + try: + span.record_exception(error, attributes=event) + except Exception: + logger.debug("Could not record the %s exception", self._span_name, exc_info=True) + + +def _end(span: Span) -> None: + try: + span.end() + except Exception: + logger.debug("Could not end the Tavily span", exc_info=True) + + +def _attach(span: Span) -> Any: + """Make ``span`` current so HTTP client activity nests under it.""" + try: + return context_api.attach(trace_api.set_span_in_context(span)) + except Exception: + logger.debug("Could not make the Tavily span current", exc_info=True) + return None + + +def _detach(token: Any) -> None: + if token is None: + return + try: + context_api.detach(token) + except Exception: + logger.debug("Could not restore the context", exc_info=True) + + +class SyncWrapper(_Operation): + """Trace a TavilyClient call.""" + + def __call__( + self, + wrapped: Callable[..., Any], + instance: Any, + args: tuple, + kwargs: Mapping[str, Any], + ) -> Any: + keys = _api_keys(instance) + query = self._query(wrapped, instance, args, kwargs) + span = self._start(keys, query, args, kwargs) + if span is None: + return wrapped(*args, **kwargs) + token = _attach(span) + try: + result = wrapped(*args, **kwargs) + except BaseException as error: + _detach(token) + self._finish_error(span, error, keys, query) + raise + _detach(token) + self._finish_ok(span, result) + return result + + +class AsyncWrapper(_Operation): + """Trace an AsyncTavilyClient call.""" + + async def __call__( + self, + wrapped: Callable[..., Any], + instance: Any, + args: tuple, + kwargs: Mapping[str, Any], + ) -> Any: + keys = _api_keys(instance) + query = self._query(wrapped, instance, args, kwargs) + span = self._start(keys, query, args, kwargs) + if span is None: + return await wrapped(*args, **kwargs) + token = _attach(span) + try: + result = await wrapped(*args, **kwargs) + except BaseException as error: + _detach(token) + self._finish_error(span, error, keys, query) + raise + _detach(token) + self._finish_ok(span, result) + return result diff --git a/python/frameworks/tavily/traceai_tavily/package.py b/python/frameworks/tavily/traceai_tavily/package.py new file mode 100644 index 000000000..d19609c17 --- /dev/null +++ b/python/frameworks/tavily/traceai_tavily/package.py @@ -0,0 +1 @@ +_instruments = ("tavily-python >= 0.8.4, < 1",) diff --git a/python/frameworks/tavily/traceai_tavily/py.typed b/python/frameworks/tavily/traceai_tavily/py.typed new file mode 100644 index 000000000..e69de29bb diff --git a/python/frameworks/tavily/traceai_tavily/version.py b/python/frameworks/tavily/traceai_tavily/version.py new file mode 100644 index 000000000..3dc1f76bc --- /dev/null +++ b/python/frameworks/tavily/traceai_tavily/version.py @@ -0,0 +1 @@ +__version__ = "0.1.0" diff --git a/python/tests/harness/__init__.py b/python/tests/harness/__init__.py new file mode 100644 index 000000000..cf79047f1 --- /dev/null +++ b/python/tests/harness/__init__.py @@ -0,0 +1,238 @@ +"""Small, loopback-only utilities for OTLP contract tests.""" + +from __future__ import annotations + +import copy +import difflib +import json +import subprocess +import threading +from http import HTTPStatus +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from typing import Any, Mapping, Optional, Sequence, Union +from urllib.parse import urlsplit +from urllib.request import Request, urlopen + + +class Receiver: + """An in-memory OTLP/HTTP receiver for tests. + + The receiver only listens on loopback and never forwards received spans. + """ + + def __init__(self) -> None: + self._spans: list[dict[str, Any]] = [] + self._requests: list[dict[str, Any]] = [] + self._lock = threading.Lock() + owner = self + + class Handler(BaseHTTPRequestHandler): + def _read_body(self) -> bytes: + # The Node OTLP exporter streams with Transfer-Encoding: chunked + # and sends no Content-Length, so both framings are accepted. + if "chunked" in self.headers.get("Transfer-Encoding", "").lower(): + chunks = [] + while True: + size_line = self.rfile.readline().split(b";", 1)[0].strip() + size = int(size_line, 16) + if size == 0: + while self.rfile.readline() not in (b"\r\n", b"\n", b""): + pass + return b"".join(chunks) + chunks.append(self.rfile.read(size)) + self.rfile.readline() + length = int(self.headers.get("Content-Length", "0")) + return self.rfile.read(length) + + def do_POST(self) -> None: # noqa: N802 - required by BaseHTTPRequestHandler + path = urlsplit(self.path).path + if path not in ("/v1/traces", "/tracer/v1/traces"): + self.send_error(HTTPStatus.NOT_FOUND) + return + + try: + body = self._read_body() + content_type = self.headers.get("Content-Type", "").lower() + + if content_type.startswith("application/json"): + request = json.loads(body.decode("utf-8")) + elif content_type.startswith(("application/x-protobuf", "application/protobuf")): + from google.protobuf.json_format import MessageToDict + from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ( + ExportTraceServiceRequest, + ) + + protobuf_request = ExportTraceServiceRequest() + protobuf_request.ParseFromString(body) + request = MessageToDict(protobuf_request) + else: + self.send_error(HTTPStatus.UNSUPPORTED_MEDIA_TYPE) + return + + spans: list[dict[str, Any]] = [] + resource_attributes: list[dict[str, Any]] = [] + for resource_spans in request.get( + "resourceSpans", request.get("resource_spans", []) + ): + resource_attributes.append( + _flatten_attributes( + resource_spans.get("resource", {}).get("attributes", []) + ) + ) + for scope_spans in resource_spans.get( + "scopeSpans", resource_spans.get("scope_spans", []) + ): + spans.extend(scope_spans.get("spans", [])) + except (ImportError, TypeError, UnicodeDecodeError, ValueError) as error: + self.send_error(HTTPStatus.BAD_REQUEST, str(error)) + return + + with owner._lock: + owner._spans.extend(copy.deepcopy(spans)) + owner._requests.append( + { + "path": path, + "headers": { + key.lower(): value for key, value in self.headers.items() + }, + "resource_attributes": resource_attributes, + } + ) + self.send_response(HTTPStatus.OK) + self.send_header("Content-Length", "0") + self.end_headers() + + def log_message(self, _format: str, *args: Any) -> None: + return + + self._server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + self.origin = "http://127.0.0.1:{0}".format(self._server.server_port) + self.endpoint = "{0}/v1/traces".format(self.origin) + self.collector_endpoint = "{0}/tracer/v1/traces".format(self.origin) + self._thread = threading.Thread(target=self._server.serve_forever, daemon=True) + self._thread.start() + + def __enter__(self) -> "Receiver": + return self + + def __exit__(self, _type: Any, _value: Any, _traceback: Any) -> None: + self._server.shutdown() + self._server.server_close() + self._thread.join() + + def spans(self) -> list[dict[str, Any]]: + """Return a copy of the decoded spans received so far.""" + with self._lock: + return copy.deepcopy(self._spans) + + def requests(self) -> list[dict[str, Any]]: + """Return one record per accepted export: path, lower-cased headers, resource attributes.""" + with self._lock: + return copy.deepcopy(self._requests) + + def clear(self) -> None: + """Remove every decoded span and request record received so far.""" + with self._lock: + self._spans.clear() + self._requests.clear() + + +def _flatten_attributes(attributes: Sequence[Mapping[str, Any]]) -> dict[str, Any]: + """Turn OTLP key/value attributes into a plain dict of scalar values.""" + flat: dict[str, Any] = {} + for attribute in attributes: + value = attribute.get("value", {}) + for kind in ("stringValue", "boolValue", "intValue", "doubleValue"): + if kind in value: + flat[attribute.get("key", "")] = value[kind] + break + else: + flat[attribute.get("key", "")] = value + return flat + + +def run( + argv: Sequence[str], + env: Mapping[str, str], + stdin: Optional[bytes], + timeout: float, +) -> subprocess.CompletedProcess[bytes]: + """Run one process and return its captured output, including after a timeout.""" + process = subprocess.Popen( + argv, + env=dict(env), + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + timed_out = False + try: + stdout, stderr = process.communicate(input=stdin, timeout=timeout) + except subprocess.TimeoutExpired: + timed_out = True + process.kill() + stdout, stderr = process.communicate() + + result = subprocess.CompletedProcess(argv, process.returncode, stdout, stderr) + result.timed_out = timed_out + return result + + +def post_otlp(spans: dict[str, Any], endpoint: str) -> int: + """Post one JSON-encoded OTLP request to a loopback receiver.""" + parsed = urlsplit(endpoint) + if parsed.scheme != "http" or parsed.hostname != "127.0.0.1": + raise ValueError("post_otlp only sends to a loopback HTTP endpoint") + + request = Request( + endpoint, + data=json.dumps(spans).encode("utf-8"), + headers={"Content-Type": "application/json"}, + method="POST", + ) + with urlopen(request, timeout=5) as response: + return response.status + + +def compare(actual: Sequence[dict[str, Any]], golden_path: Union[str, Path]) -> None: + """Fail with a diff unless span names, attributes, and statuses match a golden.""" + with Path(golden_path).open(encoding="utf-8") as golden_file: + expected = json.load(golden_file) + + def canonical(spans: Sequence[dict[str, Any]]) -> list[dict[str, Any]]: + normalized = [] + for span in spans: + attributes = span.get("attributes", []) + if isinstance(attributes, dict): + attributes = [ + {"key": key, "value": value} for key, value in attributes.items() + ] + normalized.append( + { + "name": span.get("name", ""), + "attributes": sorted( + attributes, + key=lambda attribute: json.dumps( + attribute, sort_keys=True, separators=(",", ":") + ), + ), + "status": span.get("status", {}), + } + ) + return normalized + + actual_normalized = canonical(actual) + expected_normalized = canonical(expected) + if actual_normalized != expected_normalized: + diff = "\n".join( + difflib.unified_diff( + json.dumps(expected_normalized, indent=2, sort_keys=True).splitlines(), + json.dumps(actual_normalized, indent=2, sort_keys=True).splitlines(), + fromfile=str(golden_path), + tofile="actual", + lineterm="", + ) + ) + print(diff) + raise AssertionError("spans do not match golden") diff --git a/python/tests/harness/test_harness.py b/python/tests/harness/test_harness.py new file mode 100644 index 000000000..bd61d5adc --- /dev/null +++ b/python/tests/harness/test_harness.py @@ -0,0 +1,189 @@ +"""Contract tests for the shared loopback-only harness.""" + +import json +import sys + +import pytest + +from tests.harness import Receiver, compare, post_otlp, run + + +def test_receiver_stores_posted_span_and_can_clear() -> None: + payload = { + "resourceSpans": [ + { + "scopeSpans": [ + { + "spans": [ + { + "traceId": "trace-id", + "spanId": "span-id", + "name": "harness.receiver", + "attributes": [ + {"key": "component", "value": {"stringValue": "test"}} + ], + "status": {"code": 1}, + } + ] + } + ] + } + ] + } + + with Receiver() as receiver: + assert post_otlp(payload, receiver.endpoint) == 200 + assert receiver.spans() == payload["resourceSpans"][0]["scopeSpans"][0]["spans"] + receiver.clear() + assert receiver.spans() == [] + + +def test_receiver_accepts_the_collector_path() -> None: + payload = { + "resourceSpans": [ + {"scopeSpans": [{"spans": [{"name": "collector.path"}]}]} + ] + } + + with Receiver() as receiver: + assert post_otlp(payload, receiver.collector_endpoint) == 200 + assert receiver.spans() == [{"name": "collector.path"}] + + +def _post_chunked(url: str, body: bytes, headers: dict) -> int: + """Send a body with Transfer-Encoding: chunked, as the Node OTLP exporter does.""" + import http.client + from urllib.parse import urlsplit + + parts = urlsplit(url) + connection = http.client.HTTPConnection(parts.hostname, parts.port, timeout=5) + try: + chunks = [body[: len(body) // 2], body[len(body) // 2 :]] + connection.request( + "POST", + parts.path, + body=iter(chunks), + headers={**headers, "Transfer-Encoding": "chunked"}, + encode_chunked=True, + ) + return connection.getresponse().status + finally: + connection.close() + + +def test_receiver_decodes_a_chunked_body() -> None: + payload = { + "resourceSpans": [ + {"scopeSpans": [{"spans": [{"name": "chunked.body"}]}]} + ] + } + + with Receiver() as receiver: + status = _post_chunked( + receiver.collector_endpoint, + json.dumps(payload).encode("utf-8"), + {"Content-Type": "application/json"}, + ) + assert status == 200 + assert receiver.spans() == [{"name": "chunked.body"}] + + +def test_receiver_records_path_headers_and_resource_attributes() -> None: + payload = { + "resourceSpans": [ + { + "resource": { + "attributes": [ + {"key": "project_name", "value": {"stringValue": "harness"}}, + {"key": "project_type", "value": {"stringValue": "observe"}}, + ] + }, + "scopeSpans": [{"spans": [{"name": "with.resource"}]}], + } + ] + } + + with Receiver() as receiver: + status = _post_chunked( + receiver.collector_endpoint, + json.dumps(payload).encode("utf-8"), + { + "Content-Type": "application/json", + "X-Api-Key": "test-key", + "X-Secret-Key": "test-secret", + }, + ) + assert status == 200 + [request] = receiver.requests() + assert request["path"] == "/tracer/v1/traces" + assert request["headers"]["x-api-key"] == "test-key" + assert request["headers"]["x-secret-key"] == "test-secret" + assert "authorization" not in request["headers"] + assert request["resource_attributes"] == [ + {"project_name": "harness", "project_type": "observe"} + ] + receiver.clear() + assert receiver.requests() == [] + + +def test_run_captures_output_and_kills_timed_out_process() -> None: + completed = run( + [ + sys.executable, + "-c", + "import sys; print('stdout'); sys.stderr.write('stderr\\n')", + ], + env={}, + stdin=b"", + timeout=1, + ) + assert completed.returncode == 0 + assert completed.stdout == b"stdout\n" + assert completed.stderr == b"stderr\n" + assert completed.timed_out is False + + timed_out = run( + [sys.executable, "-c", "import time; time.sleep(30)"], + env={}, + stdin=b"", + timeout=0.1, + ) + assert timed_out.timed_out is True + assert timed_out.returncode != 0 + + +def test_compare_accepts_normalized_match_and_prints_mismatch_diff(tmp_path, capsys) -> None: + golden = [ + { + "traceId": "golden-trace", + "spanId": "golden-span", + "startTimeUnixNano": "1", + "endTimeUnixNano": "2", + "name": "harness.compare", + "attributes": [ + {"key": "alpha", "value": {"stringValue": "a"}}, + {"key": "beta", "value": {"intValue": "2"}}, + ], + "status": {"code": 1}, + } + ] + golden_path = tmp_path / "spans.golden.json" + golden_path.write_text(json.dumps(golden), encoding="utf-8") + + actual = [ + { + "traceId": "other-trace", + "spanId": "other-span", + "startTimeUnixNano": "3", + "endTimeUnixNano": "4", + "name": "harness.compare", + "attributes": list(reversed(golden[0]["attributes"])), + "status": {"code": 1}, + } + ] + compare(actual, golden_path) + + actual[0]["name"] = "harness.mismatch" + with pytest.raises(AssertionError, match="spans do not match golden"): + compare(actual, golden_path) + assert "---" in capsys.readouterr().out