From 5936e0960f01a23e842d3df90b7c101874ed8149 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Sat, 3 Oct 2026 18:58:31 -0700 Subject: [PATCH 01/17] test(harness): add shared OTLP harness for TH-8103 contract tests --- python/tests/harness/__init__.py | 184 +++++++++++++++++++++++++++ python/tests/harness/test_harness.py | 101 +++++++++++++++ 2 files changed, 285 insertions(+) create mode 100644 python/tests/harness/__init__.py create mode 100644 python/tests/harness/test_harness.py diff --git a/python/tests/harness/__init__.py b/python/tests/harness/__init__.py new file mode 100644 index 000000000..dfd9198e4 --- /dev/null +++ b/python/tests/harness/__init__.py @@ -0,0 +1,184 @@ +"""Small, loopback-only utilities for OTLP contract tests.""" + +from __future__ import annotations + +import copy +import difflib +import json +import subprocess +import threading +from http import HTTPStatus +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from typing import Any, Mapping, Optional, Sequence, Union +from urllib.parse import urlsplit +from urllib.request import Request, urlopen + + +class Receiver: + """An in-memory OTLP/HTTP receiver for tests. + + The receiver only listens on loopback and never forwards received spans. + """ + + def __init__(self) -> None: + self._spans: list[dict[str, Any]] = [] + self._lock = threading.Lock() + owner = self + + class Handler(BaseHTTPRequestHandler): + def do_POST(self) -> None: # noqa: N802 - required by BaseHTTPRequestHandler + if urlsplit(self.path).path != "/v1/traces": + self.send_error(HTTPStatus.NOT_FOUND) + return + + try: + length = int(self.headers.get("Content-Length", "0")) + body = self.rfile.read(length) + content_type = self.headers.get("Content-Type", "").lower() + + if content_type.startswith("application/json"): + request = json.loads(body.decode("utf-8")) + elif content_type.startswith(("application/x-protobuf", "application/protobuf")): + from google.protobuf.json_format import MessageToDict + from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ( + ExportTraceServiceRequest, + ) + + protobuf_request = ExportTraceServiceRequest() + protobuf_request.ParseFromString(body) + request = MessageToDict(protobuf_request) + else: + self.send_error(HTTPStatus.UNSUPPORTED_MEDIA_TYPE) + return + + spans: list[dict[str, Any]] = [] + for resource_spans in request.get( + "resourceSpans", request.get("resource_spans", []) + ): + for scope_spans in resource_spans.get( + "scopeSpans", resource_spans.get("scope_spans", []) + ): + spans.extend(scope_spans.get("spans", [])) + except (ImportError, TypeError, UnicodeDecodeError, ValueError) as error: + self.send_error(HTTPStatus.BAD_REQUEST, str(error)) + return + + with owner._lock: + owner._spans.extend(copy.deepcopy(spans)) + self.send_response(HTTPStatus.OK) + self.send_header("Content-Length", "0") + self.end_headers() + + def log_message(self, _format: str, *args: Any) -> None: + return + + self._server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + self.origin = "http://127.0.0.1:{0}".format(self._server.server_port) + self.endpoint = "{0}/v1/traces".format(self.origin) + self._thread = threading.Thread(target=self._server.serve_forever, daemon=True) + self._thread.start() + + def __enter__(self) -> "Receiver": + return self + + def __exit__(self, _type: Any, _value: Any, _traceback: Any) -> None: + self._server.shutdown() + self._server.server_close() + self._thread.join() + + def spans(self) -> list[dict[str, Any]]: + """Return a copy of the decoded spans received so far.""" + with self._lock: + return copy.deepcopy(self._spans) + + def clear(self) -> None: + """Remove every decoded span received so far.""" + with self._lock: + self._spans.clear() + + +def run( + argv: Sequence[str], + env: Mapping[str, str], + stdin: Optional[bytes], + timeout: float, +) -> subprocess.CompletedProcess[bytes]: + """Run one process and return its captured output, including after a timeout.""" + process = subprocess.Popen( + argv, + env=dict(env), + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + timed_out = False + try: + stdout, stderr = process.communicate(input=stdin, timeout=timeout) + except subprocess.TimeoutExpired: + timed_out = True + process.kill() + stdout, stderr = process.communicate() + + result = subprocess.CompletedProcess(argv, process.returncode, stdout, stderr) + result.timed_out = timed_out + return result + + +def post_otlp(spans: dict[str, Any], endpoint: str) -> int: + """Post one JSON-encoded OTLP request to a loopback receiver.""" + parsed = urlsplit(endpoint) + if parsed.scheme != "http" or parsed.hostname != "127.0.0.1": + raise ValueError("post_otlp only sends to a loopback HTTP endpoint") + + request = Request( + endpoint, + data=json.dumps(spans).encode("utf-8"), + headers={"Content-Type": "application/json"}, + method="POST", + ) + with urlopen(request, timeout=5) as response: + return response.status + + +def compare(actual: Sequence[dict[str, Any]], golden_path: Union[str, Path]) -> None: + """Fail with a diff unless span names, attributes, and statuses match a golden.""" + with Path(golden_path).open(encoding="utf-8") as golden_file: + expected = json.load(golden_file) + + def canonical(spans: Sequence[dict[str, Any]]) -> list[dict[str, Any]]: + normalized = [] + for span in spans: + attributes = span.get("attributes", []) + if isinstance(attributes, dict): + attributes = [ + {"key": key, "value": value} for key, value in attributes.items() + ] + normalized.append( + { + "name": span.get("name", ""), + "attributes": sorted( + attributes, + key=lambda attribute: json.dumps( + attribute, sort_keys=True, separators=(",", ":") + ), + ), + "status": span.get("status", {}), + } + ) + return normalized + + actual_normalized = canonical(actual) + expected_normalized = canonical(expected) + if actual_normalized != expected_normalized: + diff = "\n".join( + difflib.unified_diff( + json.dumps(expected_normalized, indent=2, sort_keys=True).splitlines(), + json.dumps(actual_normalized, indent=2, sort_keys=True).splitlines(), + fromfile=str(golden_path), + tofile="actual", + lineterm="", + ) + ) + print(diff) + raise AssertionError("spans do not match golden") diff --git a/python/tests/harness/test_harness.py b/python/tests/harness/test_harness.py new file mode 100644 index 000000000..4e76ceabe --- /dev/null +++ b/python/tests/harness/test_harness.py @@ -0,0 +1,101 @@ +"""Contract tests for the shared loopback-only harness.""" + +import json +import sys + +import pytest + +from tests.harness import Receiver, compare, post_otlp, run + + +def test_receiver_stores_posted_span_and_can_clear() -> None: + payload = { + "resourceSpans": [ + { + "scopeSpans": [ + { + "spans": [ + { + "traceId": "trace-id", + "spanId": "span-id", + "name": "harness.receiver", + "attributes": [ + {"key": "component", "value": {"stringValue": "test"}} + ], + "status": {"code": 1}, + } + ] + } + ] + } + ] + } + + with Receiver() as receiver: + assert post_otlp(payload, receiver.endpoint) == 200 + assert receiver.spans() == payload["resourceSpans"][0]["scopeSpans"][0]["spans"] + receiver.clear() + assert receiver.spans() == [] + + +def test_run_captures_output_and_kills_timed_out_process() -> None: + completed = run( + [ + sys.executable, + "-c", + "import sys; print('stdout'); sys.stderr.write('stderr\\n')", + ], + env={}, + stdin=b"", + timeout=1, + ) + assert completed.returncode == 0 + assert completed.stdout == b"stdout\n" + assert completed.stderr == b"stderr\n" + assert completed.timed_out is False + + timed_out = run( + [sys.executable, "-c", "import time; time.sleep(30)"], + env={}, + stdin=b"", + timeout=0.1, + ) + assert timed_out.timed_out is True + assert timed_out.returncode != 0 + + +def test_compare_accepts_normalized_match_and_prints_mismatch_diff(tmp_path, capsys) -> None: + golden = [ + { + "traceId": "golden-trace", + "spanId": "golden-span", + "startTimeUnixNano": "1", + "endTimeUnixNano": "2", + "name": "harness.compare", + "attributes": [ + {"key": "alpha", "value": {"stringValue": "a"}}, + {"key": "beta", "value": {"intValue": "2"}}, + ], + "status": {"code": 1}, + } + ] + golden_path = tmp_path / "spans.golden.json" + golden_path.write_text(json.dumps(golden), encoding="utf-8") + + actual = [ + { + "traceId": "other-trace", + "spanId": "other-span", + "startTimeUnixNano": "3", + "endTimeUnixNano": "4", + "name": "harness.compare", + "attributes": list(reversed(golden[0]["attributes"])), + "status": {"code": 1}, + } + ] + compare(actual, golden_path) + + actual[0]["name"] = "harness.mismatch" + with pytest.raises(AssertionError, match="spans do not match golden"): + compare(actual, golden_path) + assert "---" in capsys.readouterr().out From 32f5c9d2b3b80f97a70e00b3f3ca1b408b14dc5c Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Sat, 3 Oct 2026 23:21:29 -0700 Subject: [PATCH 02/17] test(harness): accept the collector /tracer/v1/traces path --- python/tests/harness/__init__.py | 3 ++- python/tests/harness/test_harness.py | 12 ++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/python/tests/harness/__init__.py b/python/tests/harness/__init__.py index dfd9198e4..cb510ad10 100644 --- a/python/tests/harness/__init__.py +++ b/python/tests/harness/__init__.py @@ -28,7 +28,7 @@ def __init__(self) -> None: class Handler(BaseHTTPRequestHandler): def do_POST(self) -> None: # noqa: N802 - required by BaseHTTPRequestHandler - if urlsplit(self.path).path != "/v1/traces": + if urlsplit(self.path).path not in ("/v1/traces", "/tracer/v1/traces"): self.send_error(HTTPStatus.NOT_FOUND) return @@ -76,6 +76,7 @@ def log_message(self, _format: str, *args: Any) -> None: self._server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) self.origin = "http://127.0.0.1:{0}".format(self._server.server_port) self.endpoint = "{0}/v1/traces".format(self.origin) + self.collector_endpoint = "{0}/tracer/v1/traces".format(self.origin) self._thread = threading.Thread(target=self._server.serve_forever, daemon=True) self._thread.start() diff --git a/python/tests/harness/test_harness.py b/python/tests/harness/test_harness.py index 4e76ceabe..6a32c2af8 100644 --- a/python/tests/harness/test_harness.py +++ b/python/tests/harness/test_harness.py @@ -38,6 +38,18 @@ def test_receiver_stores_posted_span_and_can_clear() -> None: assert receiver.spans() == [] +def test_receiver_accepts_the_collector_path() -> None: + payload = { + "resourceSpans": [ + {"scopeSpans": [{"spans": [{"name": "collector.path"}]}]} + ] + } + + with Receiver() as receiver: + assert post_otlp(payload, receiver.collector_endpoint) == 200 + assert receiver.spans() == [{"name": "collector.path"}] + + def test_run_captures_output_and_kills_timed_out_process() -> None: completed = run( [ From 3eaadc8bdbf428a472db6779091bbce3d6d89015 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Sun, 4 Oct 2026 15:59:06 -0700 Subject: [PATCH 03/17] fix(harness): decode chunked OTLP bodies and record request metadata The Node OTLP exporter streams with Transfer-Encoding: chunked and sends no Content-Length. The Receiver read Content-Length only, so every Node export came back 400 with zero spans; two TH-8103 children had to put a de-chunking relay in front of it. Receiver now accepts both framings. It also records one entry per accepted export (path, lower-cased headers, flattened resource attributes) via requests(), so a contract test can assert X-Api-Key/X-Secret-Key and project_name through the shared harness instead of a private recorder. Verified: 6 harness tests pass, and the TanStack example's real Node exporter delivered 4 chunked exports (4 spans, collector path, both auth headers, project_name/project_type) with no relay. Refs: TH-8339, TH-8103 --- python/tests/harness/__init__.py | 61 ++++++++++++++++++++-- python/tests/harness/test_harness.py | 76 ++++++++++++++++++++++++++++ 2 files changed, 133 insertions(+), 4 deletions(-) diff --git a/python/tests/harness/__init__.py b/python/tests/harness/__init__.py index cb510ad10..cf79047f1 100644 --- a/python/tests/harness/__init__.py +++ b/python/tests/harness/__init__.py @@ -23,18 +23,36 @@ class Receiver: def __init__(self) -> None: self._spans: list[dict[str, Any]] = [] + self._requests: list[dict[str, Any]] = [] self._lock = threading.Lock() owner = self class Handler(BaseHTTPRequestHandler): + def _read_body(self) -> bytes: + # The Node OTLP exporter streams with Transfer-Encoding: chunked + # and sends no Content-Length, so both framings are accepted. + if "chunked" in self.headers.get("Transfer-Encoding", "").lower(): + chunks = [] + while True: + size_line = self.rfile.readline().split(b";", 1)[0].strip() + size = int(size_line, 16) + if size == 0: + while self.rfile.readline() not in (b"\r\n", b"\n", b""): + pass + return b"".join(chunks) + chunks.append(self.rfile.read(size)) + self.rfile.readline() + length = int(self.headers.get("Content-Length", "0")) + return self.rfile.read(length) + def do_POST(self) -> None: # noqa: N802 - required by BaseHTTPRequestHandler - if urlsplit(self.path).path not in ("/v1/traces", "/tracer/v1/traces"): + path = urlsplit(self.path).path + if path not in ("/v1/traces", "/tracer/v1/traces"): self.send_error(HTTPStatus.NOT_FOUND) return try: - length = int(self.headers.get("Content-Length", "0")) - body = self.rfile.read(length) + body = self._read_body() content_type = self.headers.get("Content-Type", "").lower() if content_type.startswith("application/json"): @@ -53,9 +71,15 @@ def do_POST(self) -> None: # noqa: N802 - required by BaseHTTPRequestHandler return spans: list[dict[str, Any]] = [] + resource_attributes: list[dict[str, Any]] = [] for resource_spans in request.get( "resourceSpans", request.get("resource_spans", []) ): + resource_attributes.append( + _flatten_attributes( + resource_spans.get("resource", {}).get("attributes", []) + ) + ) for scope_spans in resource_spans.get( "scopeSpans", resource_spans.get("scope_spans", []) ): @@ -66,6 +90,15 @@ def do_POST(self) -> None: # noqa: N802 - required by BaseHTTPRequestHandler with owner._lock: owner._spans.extend(copy.deepcopy(spans)) + owner._requests.append( + { + "path": path, + "headers": { + key.lower(): value for key, value in self.headers.items() + }, + "resource_attributes": resource_attributes, + } + ) self.send_response(HTTPStatus.OK) self.send_header("Content-Length", "0") self.end_headers() @@ -93,10 +126,30 @@ def spans(self) -> list[dict[str, Any]]: with self._lock: return copy.deepcopy(self._spans) + def requests(self) -> list[dict[str, Any]]: + """Return one record per accepted export: path, lower-cased headers, resource attributes.""" + with self._lock: + return copy.deepcopy(self._requests) + def clear(self) -> None: - """Remove every decoded span received so far.""" + """Remove every decoded span and request record received so far.""" with self._lock: self._spans.clear() + self._requests.clear() + + +def _flatten_attributes(attributes: Sequence[Mapping[str, Any]]) -> dict[str, Any]: + """Turn OTLP key/value attributes into a plain dict of scalar values.""" + flat: dict[str, Any] = {} + for attribute in attributes: + value = attribute.get("value", {}) + for kind in ("stringValue", "boolValue", "intValue", "doubleValue"): + if kind in value: + flat[attribute.get("key", "")] = value[kind] + break + else: + flat[attribute.get("key", "")] = value + return flat def run( diff --git a/python/tests/harness/test_harness.py b/python/tests/harness/test_harness.py index 6a32c2af8..bd61d5adc 100644 --- a/python/tests/harness/test_harness.py +++ b/python/tests/harness/test_harness.py @@ -50,6 +50,82 @@ def test_receiver_accepts_the_collector_path() -> None: assert receiver.spans() == [{"name": "collector.path"}] +def _post_chunked(url: str, body: bytes, headers: dict) -> int: + """Send a body with Transfer-Encoding: chunked, as the Node OTLP exporter does.""" + import http.client + from urllib.parse import urlsplit + + parts = urlsplit(url) + connection = http.client.HTTPConnection(parts.hostname, parts.port, timeout=5) + try: + chunks = [body[: len(body) // 2], body[len(body) // 2 :]] + connection.request( + "POST", + parts.path, + body=iter(chunks), + headers={**headers, "Transfer-Encoding": "chunked"}, + encode_chunked=True, + ) + return connection.getresponse().status + finally: + connection.close() + + +def test_receiver_decodes_a_chunked_body() -> None: + payload = { + "resourceSpans": [ + {"scopeSpans": [{"spans": [{"name": "chunked.body"}]}]} + ] + } + + with Receiver() as receiver: + status = _post_chunked( + receiver.collector_endpoint, + json.dumps(payload).encode("utf-8"), + {"Content-Type": "application/json"}, + ) + assert status == 200 + assert receiver.spans() == [{"name": "chunked.body"}] + + +def test_receiver_records_path_headers_and_resource_attributes() -> None: + payload = { + "resourceSpans": [ + { + "resource": { + "attributes": [ + {"key": "project_name", "value": {"stringValue": "harness"}}, + {"key": "project_type", "value": {"stringValue": "observe"}}, + ] + }, + "scopeSpans": [{"spans": [{"name": "with.resource"}]}], + } + ] + } + + with Receiver() as receiver: + status = _post_chunked( + receiver.collector_endpoint, + json.dumps(payload).encode("utf-8"), + { + "Content-Type": "application/json", + "X-Api-Key": "test-key", + "X-Secret-Key": "test-secret", + }, + ) + assert status == 200 + [request] = receiver.requests() + assert request["path"] == "/tracer/v1/traces" + assert request["headers"]["x-api-key"] == "test-key" + assert request["headers"]["x-secret-key"] == "test-secret" + assert "authorization" not in request["headers"] + assert request["resource_attributes"] == [ + {"project_name": "harness", "project_type": "observe"} + ] + receiver.clear() + assert receiver.requests() == [] + + def test_run_captures_output_and_kills_timed_out_process() -> None: completed = run( [ From 9afeaefd066db2d7eb831f653331720e1219d0b1 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Mon, 5 Oct 2026 01:17:10 -0700 Subject: [PATCH 04/17] test(voyage): real-SDK and contract tests for traceAI-voyage Tests first for TH-8321. A loopback fake of the Voyage HTTP API drives the real voyageai Client and AsyncClient (embed, rerank, multimodal_embed); an in-memory provider pins the wrapper's own status, and the TH-8339 harness Receiver checks the collector contract through fi_instrumentation.register(). Covers: one EMBEDDING/RERANKER span per call, counts, dimension and promoted token keys, content off by default with an opt-in that honours TraceConfig, API-key redaction, errors/timeouts/cancellation, instrumentation isolation, span nesting, sync/async parity, uninstrument, version ranges and no base_url from the instrumentor. Fails at collection until the package lands. approved: Nikhil 2026-10-03 blanket Refs: TH-8321 --- python/frameworks/voyage/tests/_support.py | 290 ++++++++++++++++++ python/frameworks/voyage/tests/test_async.py | 99 ++++++ .../voyage/tests/test_content_capture.py | 177 +++++++++++ .../frameworks/voyage/tests/test_context.py | 153 +++++++++ .../voyage/tests/test_contract_harness.py | 164 ++++++++++ python/frameworks/voyage/tests/test_embed.py | 172 +++++++++++ python/frameworks/voyage/tests/test_errors.py | 209 +++++++++++++ .../frameworks/voyage/tests/test_example.py | 90 ++++++ .../frameworks/voyage/tests/test_packaging.py | 108 +++++++ python/frameworks/voyage/tests/test_rerank.py | 80 +++++ .../voyage/tests/test_uninstrument.py | 110 +++++++ 11 files changed, 1652 insertions(+) create mode 100644 python/frameworks/voyage/tests/_support.py create mode 100644 python/frameworks/voyage/tests/test_async.py create mode 100644 python/frameworks/voyage/tests/test_content_capture.py create mode 100644 python/frameworks/voyage/tests/test_context.py create mode 100644 python/frameworks/voyage/tests/test_contract_harness.py create mode 100644 python/frameworks/voyage/tests/test_embed.py create mode 100644 python/frameworks/voyage/tests/test_errors.py create mode 100644 python/frameworks/voyage/tests/test_example.py create mode 100644 python/frameworks/voyage/tests/test_packaging.py create mode 100644 python/frameworks/voyage/tests/test_rerank.py create mode 100644 python/frameworks/voyage/tests/test_uninstrument.py diff --git a/python/frameworks/voyage/tests/_support.py b/python/frameworks/voyage/tests/_support.py new file mode 100644 index 000000000..17c7a998c --- /dev/null +++ b/python/frameworks/voyage/tests/_support.py @@ -0,0 +1,290 @@ +"""Shared support for the real-SDK tests: a loopback Voyage API and a tracer. + +``FakeVoyage`` speaks enough of the Voyage HTTP API for voyageai 0.3.7 to +0.5.x: ``POST /v1/embeddings`` (base64 or list vectors, float and packed +binary dtypes), ``POST /v1/rerank`` and ``POST /v1/multimodalembeddings``. +The real ``voyageai.Client`` / ``AsyncClient`` reach it through +``base_url=fake.base_url``. Nothing here leaves 127.0.0.1 and every key is a +placeholder. + +Models with special behaviour: + +* ``fail-401``: HTTP 401 with a ``detail`` message. +* ``echo-key``: HTTP 400 whose ``detail`` echoes the bearer key back, so a + test can prove the key is redacted from error text. +* ``rate-limit-once``: the first request gets HTTP 429, later ones succeed. +* ``slow``: the request is held until the fake closes (timeouts, cancels). +""" + +from __future__ import annotations + +import base64 +import contextlib +import json +import struct +import threading +from dataclasses import dataclass +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from typing import Any, Dict, Iterator, List, Optional, Tuple + +from opentelemetry.sdk.resources import Resource +from opentelemetry.sdk.trace import ReadableSpan, TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter + +VOYAGE_KEY = "pa-placeholder-voyage-key-must-not-be-exported" +EMBED_MODEL = "voyage-3.5" +RERANK_MODEL = "rerank-2.5" + +# Content the tests send or the fake returns. None of it may reach a span +# unless content capture is switched on. +TEXTS = ["TEXT-ALPHA-MUST-NOT-BE-EXPORTED", "TEXT-BETA-MUST-NOT-BE-EXPORTED"] +QUERY = "QUERY-MUST-NOT-BE-EXPORTED" +DOCUMENTS = [ + "DOC-ONE-MUST-NOT-BE-EXPORTED", + "DOC-TWO-MUST-NOT-BE-EXPORTED", + "DOC-THREE-MUST-NOT-BE-EXPORTED", +] +CONTENT_MARKERS = tuple(TEXTS) + (QUERY,) + tuple(DOCUMENTS) + +# Vectors: every component is VECTOR_BASE + k * 2**-9, exact in float32, so +# the decoded values are predictable and searchable on the wire. +DIMENSION = 16 +VECTOR_BASE = 0.3359375 +VECTOR_MARKER = "0.3359375" +# Relevance score per document index. The fake returns them sorted by score +# (1, 0, 2), so a span that re-sorted or lost the order is visible. +SCORES = [0.4140625, 0.91796875, 0.16015625] +SCORE_MARKERS = ("0.4140625", "0.91796875", "0.16015625") + +TOKENS_PER_TEXT = 7 + + +def rerank_tokens(documents: int) -> int: + return 5 + 6 * documents + + +def _vector(dimension: int) -> List[float]: + return [VECTOR_BASE + k * 2**-9 for k in range(dimension)] + + +def _encode(values: List[Any], fmt: str) -> str: + return base64.b64encode(struct.pack("<{0}{1}".format(len(values), fmt), *values)).decode() + + +def _embedding(body: Dict[str, Any]) -> Any: + dimension = int(body.get("output_dimension") or DIMENSION) + dtype = body.get("output_dtype") or "float" + as_base64 = body.get("encoding_format") == "base64" + if dtype in ("binary", "ubinary"): + # Packed bits: eight dimensions per byte. + values = [(-1) ** k * (k + 1) for k in range(dimension // 8)] + fmt = "b" if dtype == "binary" else "B" + values = [abs(v) for v in values] if dtype == "ubinary" else values + return _encode(values, fmt) if as_base64 else values + if dtype in ("int8", "uint8"): + values = [k % 100 for k in range(dimension)] + return _encode(values, "b" if dtype == "int8" else "B") if as_base64 else values + values = _vector(dimension) + return _encode(values, "f") if as_base64 else values + + +class _QuietServer(ThreadingHTTPServer): + daemon_threads = True + + def handle_error(self, request: Any, client_address: Any) -> None: + # A timed-out or cancelled client leaves a broken pipe behind. + return + + +class FakeVoyage: + """Loopback stand-in for the Voyage API (api.voyageai.com / ai.mongodb.com).""" + + def __init__(self) -> None: + self.calls: List[Tuple[str, Dict[str, Any], bool]] = [] + self.received = threading.Event() + self._release = threading.Event() + self._rate_limited = False + self._lock = threading.Lock() + owner = self + + class Handler(BaseHTTPRequestHandler): + def do_POST(self) -> None: # noqa: N802 - BaseHTTPRequestHandler API + length = int(self.headers.get("Content-Length", "0") or 0) + body = json.loads(self.rfile.read(length) or b"{}") + path = self.path.split("?")[0] + authorization = self.headers.get("Authorization", "") + with owner._lock: + owner.calls.append((path, body, authorization == "Bearer " + VOYAGE_KEY)) + owner.received.set() + model = body.get("model") + if model == "slow": + owner._release.wait(30) + self._send(503, {"detail": "released"}) + elif model == "fail-401": + self._send(401, {"detail": "Provided API key is invalid."}) + elif model == "echo-key": + key = authorization[len("Bearer "):] + self._send(400, {"detail": "bad request for key {0}".format(key)}) + elif model == "rate-limit-once" and owner._take_rate_limit(): + self._send(429, {"detail": "rate limited"}) + elif path == "/v1/embeddings": + texts = body.get("input") + texts = [texts] if isinstance(texts, str) else list(texts or []) + data = [ + {"object": "embedding", "embedding": _embedding(body), "index": i} + for i in range(len(texts)) + ] + self._send( + 200, + { + "object": "list", + "data": data, + "model": model, + "usage": {"total_tokens": TOKENS_PER_TEXT * len(texts)}, + }, + ) + elif path == "/v1/rerank": + documents = list(body.get("documents") or []) + ranked = sorted( + range(len(documents)), + key=lambda i: SCORES[i % len(SCORES)], + reverse=True, + ) + top_k = body.get("top_k") + if top_k is not None: + ranked = ranked[: int(top_k)] + data = [ + {"index": i, "relevance_score": SCORES[i % len(SCORES)]} for i in ranked + ] + self._send( + 200, + { + "object": "list", + "data": data, + "model": model, + "usage": {"total_tokens": rerank_tokens(len(documents))}, + }, + ) + elif path == "/v1/multimodalembeddings": + inputs = list(body.get("inputs") or []) + data = [ + {"object": "embedding", "embedding": _vector(DIMENSION), "index": i} + for i in range(len(inputs)) + ] + usage = { + "text_tokens": 3, + "image_pixels": 0, + "video_pixels": 0, + "total_tokens": 3, + } + self._send(200, {"object": "list", "data": data, "usage": usage}) + else: + self._send(404, {"detail": "not found"}) + + def _send(self, status: int, payload: Dict[str, Any]) -> None: + data = json.dumps(payload).encode("utf-8") + try: + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(data))) + self.end_headers() + self.wfile.write(data) + except OSError: + return + + def log_message(self, *_: Any) -> None: + return + + self._server = _QuietServer(("127.0.0.1", 0), Handler) + self.origin = "http://127.0.0.1:{0}".format(self._server.server_port) + self.base_url = self.origin + "/v1" + self._thread = threading.Thread( + target=self._server.serve_forever, kwargs={"poll_interval": 0.05}, daemon=True + ) + self._thread.start() + + def _take_rate_limit(self) -> bool: + with self._lock: + if self._rate_limited: + return False + self._rate_limited = True + return True + + def paths(self) -> List[str]: + with self._lock: + return [path for path, _, _ in self.calls] + + def bodies(self) -> List[Dict[str, Any]]: + with self._lock: + return [body for _, body, _ in self.calls] + + def close(self) -> None: + self._release.set() + self._server.shutdown() + self._server.server_close() + self._thread.join() + + def __enter__(self) -> "FakeVoyage": + return self + + def __exit__(self, *_: Any) -> None: + self.close() + + +@dataclass +class Traced: + exporter: InMemorySpanExporter + provider: TracerProvider + + def spans(self) -> List[ReadableSpan]: + return list(self.exporter.get_finished_spans()) + + def one(self) -> ReadableSpan: + spans = self.spans() + assert len(spans) == 1, [span.name for span in spans] + return spans[0] + + def wire(self) -> str: + """Everything an exporter could send: attributes, events, status.""" + return "".join(span.to_json() for span in self.spans()) + + +@contextlib.contextmanager +def instrumented(**options: Any) -> Iterator[Traced]: + """Instrument Voyage against a fresh in-memory provider; uninstrument after.""" + from traceai_voyage import VoyageInstrumentor + + exporter = InMemorySpanExporter() + provider = TracerProvider(resource=Resource.create({"service.name": "test-voyage"})) + provider.add_span_processor(SimpleSpanProcessor(exporter)) + instrumentor = VoyageInstrumentor() + instrumentor.instrument(tracer_provider=provider, **options) + try: + yield Traced(exporter, provider) + finally: + instrumentor.uninstrument() + + +def attrs(span: ReadableSpan) -> Dict[str, Any]: + return dict(span.attributes or {}) + + +def event_names(span: ReadableSpan) -> List[str]: + return [event.name for event in span.events] + + +def client(fake: FakeVoyage, **options: Any) -> Any: + import voyageai + + return voyageai.Client(api_key=VOYAGE_KEY, base_url=fake.base_url, **options) + + +def async_client(fake: FakeVoyage, **options: Any) -> Any: + import voyageai + + return voyageai.AsyncClient(api_key=VOYAGE_KEY, base_url=fake.base_url, **options) + + +def status_code(span: ReadableSpan) -> Optional[str]: + return span.status.status_code.name diff --git a/python/frameworks/voyage/tests/test_async.py b/python/frameworks/voyage/tests/test_async.py new file mode 100644 index 000000000..84d712963 --- /dev/null +++ b/python/frameworks/voyage/tests/test_async.py @@ -0,0 +1,99 @@ +"""voyageai.AsyncClient: the same spans as the sync client (sync/async parity).""" + +from __future__ import annotations + +import asyncio +from typing import Any, Awaitable, Callable + +import pytest + +pytest.importorskip("voyageai", reason="voyageai must be installed to test its instrumentor") + +import voyageai # noqa: E402 + +from _support import ( # noqa: E402 + CONTENT_MARKERS, + DOCUMENTS, + EMBED_MODEL, + QUERY, + RERANK_MODEL, + SCORE_MARKERS, + TEXTS, + VECTOR_MARKER, + VOYAGE_KEY, + FakeVoyage, + async_client, + attrs, + client, + event_names, + instrumented, + status_code, +) + + +@pytest.fixture() +def fake(): + with FakeVoyage() as server: + yield server + + +def _run(call: Callable[[], Awaitable[Any]]) -> Any: + return asyncio.run(call()) + + +def test_async_embed_matches_the_sync_span(fake): + with instrumented() as traced: + client(fake).embed(TEXTS, model=EMBED_MODEL, input_type="document") + result = _run( + lambda: async_client(fake).embed(TEXTS, model=EMBED_MODEL, input_type="document") + ) + + assert len(result.embeddings) == 2 + sync_span, async_span = traced.spans() + assert async_span.name == sync_span.name == "voyage.embed" + assert attrs(async_span) == attrs(sync_span) + assert status_code(async_span) == "OK" + + +def test_async_rerank_matches_the_sync_span(fake): + with instrumented() as traced: + client(fake).rerank(QUERY, DOCUMENTS, model=RERANK_MODEL, top_k=2) + result = _run( + lambda: async_client(fake).rerank(QUERY, DOCUMENTS, model=RERANK_MODEL, top_k=2) + ) + + assert [r.index for r in result.results] == [1, 0] + sync_span, async_span = traced.spans() + assert async_span.name == sync_span.name == "voyage.rerank" + assert attrs(async_span) == attrs(sync_span) + assert status_code(async_span) == "OK" + + +def test_async_spans_carry_no_content_by_default(fake): + async def calls() -> None: + voyage = async_client(fake) + await voyage.embed(TEXTS, model=EMBED_MODEL) + await voyage.rerank(QUERY, DOCUMENTS, model=RERANK_MODEL) + + with instrumented() as traced: + _run(calls) + + assert [span.name for span in traced.spans()] == ["voyage.embed", "voyage.rerank"] + wire = traced.wire() + for marker in CONTENT_MARKERS + SCORE_MARKERS + (VECTOR_MARKER, VOYAGE_KEY): + assert marker not in wire, marker + + +def test_async_errors_match_the_sync_error_span(fake): + with instrumented() as traced: + with pytest.raises(voyageai.error.AuthenticationError): + client(fake).embed(TEXTS, model="fail-401") + with pytest.raises(voyageai.error.AuthenticationError): + _run(lambda: async_client(fake).embed(TEXTS, model="fail-401")) + + sync_span, async_span = traced.spans() + for span in (sync_span, async_span): + assert status_code(span) == "ERROR" + assert span.status.description.startswith("AuthenticationError") + assert event_names(span) == ["exception"] + assert attrs(async_span) == attrs(sync_span) diff --git a/python/frameworks/voyage/tests/test_content_capture.py b/python/frameworks/voyage/tests/test_content_capture.py new file mode 100644 index 000000000..8ae3bc46b --- /dev/null +++ b/python/frameworks/voyage/tests/test_content_capture.py @@ -0,0 +1,177 @@ +"""Content capture: off by default; opt-in honours TraceConfig and the key redaction.""" + +from __future__ import annotations + +import json + +import pytest + +pytest.importorskip("voyageai", reason="voyageai must be installed to test its instrumentor") + +import voyageai # noqa: E402 +from fi_instrumentation import TraceConfig # noqa: E402 +from traceai_voyage import VoyageInstrumentor # noqa: E402 + +from _support import ( # noqa: E402 + DOCUMENTS, + EMBED_MODEL, + QUERY, + RERANK_MODEL, + SCORE_MARKERS, + SCORES, + TEXTS, + VECTOR_MARKER, + VOYAGE_KEY, + FakeVoyage, + attrs, + client, + instrumented, +) + +OTHER_KEY = "al-placeholder-module-key-must-not-be-exported" + + +@pytest.fixture() +def fake(): + with FakeVoyage() as server: + yield server + + +def _embed_and_rerank(fake, **options): + with instrumented(**options) as traced: + voyage = client(fake) + voyage.embed(TEXTS, model=EMBED_MODEL, input_type="document") + voyage.rerank(QUERY, DOCUMENTS, model=RERANK_MODEL, top_k=2) + embed, rerank = traced.spans() + return traced, attrs(embed), attrs(rerank) + + +def test_capture_content_records_inputs_and_scores_but_never_vectors(fake): + traced, embed, rerank = _embed_and_rerank(fake, capture_content=True) + + assert json.loads(embed["input.value"]) == TEXTS + assert embed["input.mime_type"] == "application/json" + assert embed["voyage.embedding.count"] == 2 + assert rerank["reranker.query"] == QUERY + assert json.loads(rerank["input.value"]) == {"query": QUERY, "documents": DOCUMENTS} + # Scores in the client's order, with the index of the document they rank. + assert json.loads(rerank["output.value"]) == [ + {"index": 1, "relevance_score": SCORES[1]}, + {"index": 0, "relevance_score": SCORES[0]}, + ] + assert rerank["output.mime_type"] == "application/json" + # Vectors are never stored, even with capture on (PRD R-04). + assert VECTOR_MARKER not in traced.wire() + assert "output.value" not in embed + + +def test_hide_inputs_drops_texts_query_and_documents_but_keeps_counts_and_scores(fake): + traced, embed, rerank = _embed_and_rerank( + fake, capture_content=True, config=TraceConfig(hide_inputs=True) + ) + + for values in (embed, rerank): + assert "input.value" not in values + assert "input.mime_type" not in values + assert "reranker.query" not in values + assert embed["voyage.embedding.count"] == 2 + assert rerank["voyage.rerank.document_count"] == 3 + assert "output.value" in rerank + wire = traced.wire() + for marker in TEXTS + [QUERY] + DOCUMENTS: + assert marker not in wire, marker + assert SCORE_MARKERS[1] in wire + + +def test_hide_outputs_drops_scores_but_keeps_inputs(fake): + traced, _embed, rerank = _embed_and_rerank( + fake, capture_content=True, config=TraceConfig(hide_outputs=True) + ) + + assert "output.value" not in rerank + assert "output.mime_type" not in rerank + assert rerank["reranker.query"] == QUERY + assert rerank["voyage.rerank.result_count"] == 2 + for marker in SCORE_MARKERS: + assert marker not in traced.wire(), marker + + +def test_fi_hide_inputs_environment_variable_is_honoured(fake, monkeypatch): + monkeypatch.setenv("FI_HIDE_INPUTS", "true") + traced, embed, rerank = _embed_and_rerank(fake, capture_content=True) + + assert "input.value" not in embed + assert "reranker.query" not in rerank + for marker in TEXTS + [QUERY] + DOCUMENTS: + assert marker not in traced.wire(), marker + + +def test_the_api_key_is_redacted_from_captured_content(fake, monkeypatch): + # The client's own key and a module-level key both count as secrets. + monkeypatch.setattr(voyageai, "api_key", OTHER_KEY) + with instrumented(capture_content=True) as traced: + voyage = client(fake) + voyage.embed(["prefix {0} suffix".format(VOYAGE_KEY), OTHER_KEY], model=EMBED_MODEL) + voyage.rerank( + "query {0}".format(VOYAGE_KEY), ["doc {0}".format(OTHER_KEY)], model=RERANK_MODEL + ) + + embed, rerank = (attrs(span) for span in traced.spans()) + assert json.loads(embed["input.value"]) == ["prefix [redacted] suffix", "[redacted]"] + assert rerank["reranker.query"] == "query [redacted]" + assert json.loads(rerank["input.value"])["documents"] == ["doc [redacted]"] + wire = traced.wire() + assert VOYAGE_KEY not in wire + assert OTHER_KEY not in wire + + +def test_a_client_key_read_from_the_environment_is_redacted(fake, monkeypatch): + monkeypatch.setenv("VOYAGE_API_KEY", OTHER_KEY) + with instrumented(capture_content=True) as traced: + voyageai.Client(base_url=fake.base_url).embed( + ["text {0}".format(OTHER_KEY)], model=EMBED_MODEL + ) + + assert json.loads(attrs(traced.one())["input.value"]) == ["text [redacted]"] + assert OTHER_KEY not in traced.wire() + + +def test_captured_content_is_capped(fake): + texts = ["t{0}".format(i) for i in range(70)] + long_text = "é" * 3000 # 6000 UTF-8 bytes + with instrumented(capture_content=True) as traced: + voyage = client(fake) + voyage.embed(texts, model=EMBED_MODEL) + voyage.embed([long_text], model=EMBED_MODEL) + + many, long = (attrs(span) for span in traced.spans()) + # The count is exact; the captured list stops at 64 items. + assert many["voyage.embedding.count"] == 70 + assert json.loads(many["input.value"]) == texts[:64] + # Each captured string is cut to 2 KB of UTF-8 on a character boundary. + (captured,) = json.loads(long["input.value"]) + assert captured == "é" * 1024 + assert len(captured.encode("utf-8")) == 2048 + + +def test_pii_redaction_in_trace_config_applies_to_captured_content(fake): + email = "jane.doe@example.com" + with instrumented(capture_content=True, config=TraceConfig(pii_redaction=True)) as traced: + client(fake).embed(["contact {0} today".format(email)], model=EMBED_MODEL) + + assert email not in traced.wire() + + +@pytest.mark.parametrize( + "options", + [{"config": {"hide_inputs": True}}, {"capture_content": "yes"}, {"capture_content": 1}], +) +def test_invalid_options_raise_type_error_and_wrap_nothing(options): + original = voyageai.Client.__dict__["embed"] + instrumentor = VoyageInstrumentor() + with pytest.raises(TypeError): + instrumentor.instrument(**options) + try: + assert voyageai.Client.__dict__["embed"] is original + finally: + instrumentor.uninstrument() diff --git a/python/frameworks/voyage/tests/test_context.py b/python/frameworks/voyage/tests/test_context.py new file mode 100644 index 000000000..f1a93b381 --- /dev/null +++ b/python/frameworks/voyage/tests/test_context.py @@ -0,0 +1,153 @@ +"""Trace context: the Voyage span is current while the SDK sends HTTP, and nests.""" + +from __future__ import annotations + +import asyncio + +import pytest +from opentelemetry import trace as trace_api + +pytest.importorskip("voyageai", reason="voyageai must be installed to test its instrumentor") + +from voyageai.api_resources import api_requestor # noqa: E402 + +from _support import ( # noqa: E402 + DOCUMENTS, + EMBED_MODEL, + QUERY, + RERANK_MODEL, + TEXTS, + FakeVoyage, + async_client, + client, + instrumented, +) + + +@pytest.fixture() +def fake(): + with FakeVoyage() as server: + yield server + + +def _current_span_id() -> int: + return trace_api.get_current_span().get_span_context().span_id + + +@pytest.fixture() +def seen_by_http(monkeypatch): + """Record the current span id where voyageai sends HTTP. + + ``APIRequestor.request_raw`` (requests) and ``arequest_raw`` (aiohttp) are + where an HTTP client instrumentation would start its span, so the id seen + there is the parent an HTTP child span would get. + """ + seen = [] + request_raw = api_requestor.APIRequestor.request_raw + arequest_raw = api_requestor.APIRequestor.arequest_raw + + def sync_raw(self, *args, **kwargs): + seen.append(_current_span_id()) + return request_raw(self, *args, **kwargs) + + async def async_raw(self, *args, **kwargs): + seen.append(_current_span_id()) + return await arequest_raw(self, *args, **kwargs) + + monkeypatch.setattr(api_requestor.APIRequestor, "request_raw", sync_raw) + monkeypatch.setattr(api_requestor.APIRequestor, "arequest_raw", async_raw) + return seen + + +def test_sync_vendor_http_runs_inside_the_voyage_span(fake, seen_by_http): + with instrumented() as traced: + voyage = client(fake) + voyage.embed(TEXTS, model=EMBED_MODEL) + voyage.rerank(QUERY, DOCUMENTS, model=RERANK_MODEL) + + span_ids = [span.context.span_id for span in traced.spans()] + assert len(span_ids) == 2 + assert seen_by_http == span_ids + # Nothing is left current once the calls return. + assert _current_span_id() == 0 + + +def test_async_vendor_http_runs_inside_the_voyage_span(fake, seen_by_http): + async def calls() -> None: + voyage = async_client(fake) + await voyage.embed(TEXTS, model=EMBED_MODEL) + await voyage.rerank(QUERY, DOCUMENTS, model=RERANK_MODEL) + assert _current_span_id() == 0 + + with instrumented() as traced: + asyncio.run(calls()) + + span_ids = [span.context.span_id for span in traced.spans()] + assert len(span_ids) == 2 + assert seen_by_http == span_ids + + +def test_an_http_child_span_nests_under_the_voyage_span(fake, monkeypatch): + """A span started at the HTTP layer (as an HTTP instrumentor would) is a child.""" + with instrumented() as traced: + request_raw = api_requestor.APIRequestor.request_raw + http_tracer = traced.provider.get_tracer("http") + + def traced_raw(self, *args, **kwargs): + with http_tracer.start_as_current_span("POST"): + return request_raw(self, *args, **kwargs) + + monkeypatch.setattr(api_requestor.APIRequestor, "request_raw", traced_raw) + client(fake).embed(TEXTS, model=EMBED_MODEL) + + by_name = {span.name: span for span in traced.spans()} + assert by_name["POST"].parent.span_id == by_name["voyage.embed"].context.span_id + # Usage lives on the Voyage span only, once. + assert "gen_ai.usage.total_tokens" not in (by_name["POST"].attributes or {}) + assert by_name["voyage.embed"].attributes["gen_ai.usage.total_tokens"] == 14 + + +def test_voyage_spans_are_children_of_the_active_span(fake): + async def call() -> None: + await async_client(fake).rerank(QUERY, DOCUMENTS, model=RERANK_MODEL) + + with instrumented() as traced: + tracer = traced.provider.get_tracer("agent") + with tracer.start_as_current_span("agent.step") as parent: + client(fake).embed(TEXTS, model=EMBED_MODEL) + asyncio.run(call()) + + by_name = {span.name: span for span in traced.spans()} + agent = by_name["agent.step"] + assert agent.context.span_id == parent.get_span_context().span_id + for name in ("voyage.embed", "voyage.rerank"): + assert by_name[name].parent.span_id == agent.context.span_id + assert by_name[name].context.trace_id == agent.context.trace_id + + +def test_a_voyage_span_is_a_root_span_without_a_parent(fake): + with instrumented() as traced: + client(fake).embed(TEXTS, model=EMBED_MODEL) + + assert traced.one().parent is None + + +def test_session_context_attributes_are_attached(fake): + from fi_instrumentation import using_session + + with instrumented() as traced: + with using_session("session-123"): + client(fake).embed(TEXTS, model=EMBED_MODEL) + + assert traced.one().attributes["session.id"] == "session-123" + + +def test_suppress_tracing_records_no_span(fake): + from fi_instrumentation import suppress_tracing + + with instrumented() as traced: + with suppress_tracing(): + result = client(fake).embed(TEXTS, model=EMBED_MODEL) + + assert len(result.embeddings) == 2 + assert traced.spans() == [] diff --git a/python/frameworks/voyage/tests/test_contract_harness.py b/python/frameworks/voyage/tests/test_contract_harness.py new file mode 100644 index 000000000..79da146e9 --- /dev/null +++ b/python/frameworks/voyage/tests/test_contract_harness.py @@ -0,0 +1,164 @@ +"""Shared-harness contract test for traceAI-voyage (TH-8321, TH-8339 Receiver). + +The real ``voyageai`` client makes real HTTP calls to a loopback fake of the +Voyage API (``Client(base_url=...)``, a choice the test makes, never the +instrumentor). Spans leave through the real ``fi_instrumentation.register()`` +OTLP exporter into ``harness.Receiver``. Nothing calls Voyage; every key is a +placeholder. +""" + +from __future__ import annotations + +import asyncio +import json +import sys +from pathlib import Path +from typing import Any, Dict, List, Tuple + +import pytest + +pytest.importorskip("voyageai", reason="voyageai must be installed to test its instrumentor") +pytest.importorskip("opentelemetry.proto", reason="the harness decodes OTLP protobuf") + +sys.path.insert(0, str(Path(__file__).resolve().parents[3] / "tests")) + +import voyageai # noqa: E402 +from harness import Receiver, _flatten_attributes # noqa: E402 + +from _support import ( # noqa: E402 + CONTENT_MARKERS, + DIMENSION, + DOCUMENTS, + EMBED_MODEL, + QUERY, + RERANK_MODEL, + SCORE_MARKERS, + TEXTS, + TOKENS_PER_TEXT, + VECTOR_MARKER, + VOYAGE_KEY, + FakeVoyage, + async_client, + client, + rerank_tokens, +) + +FI_API_KEY = "placeholder-fi-api-key" +FI_SECRET_KEY = "placeholder-fi-secret-key" +PROJECT = "voyage-contract" + + +def _journey( + receiver: Receiver, fake: FakeVoyage, monkeypatch: pytest.MonkeyPatch, **options: Any +) -> Tuple[List[dict], List[dict]]: + from fi_instrumentation import register + from fi_instrumentation.fi_types import ProjectType + + from traceai_voyage import VoyageInstrumentor + + monkeypatch.setenv("FI_BASE_URL", receiver.origin) + monkeypatch.setenv("FI_API_KEY", FI_API_KEY) + monkeypatch.setenv("FI_SECRET_KEY", FI_SECRET_KEY) + provider = register(project_type=ProjectType.OBSERVE, project_name=PROJECT, verbose=False) + instrumentor = VoyageInstrumentor() + instrumentor.instrument(tracer_provider=provider, **options) + try: + voyage = client(fake) + assert len(voyage.embed(TEXTS, model=EMBED_MODEL, input_type="document").embeddings) == 2 + assert [r.index for r in voyage.rerank(QUERY, DOCUMENTS, RERANK_MODEL, top_k=2).results] == [ + 1, + 0, + ] + with pytest.raises(voyageai.error.AuthenticationError): + voyage.embed(TEXTS, model="fail-401") + asyncio.run(async_client(fake).rerank(QUERY, DOCUMENTS, model=RERANK_MODEL)) + assert provider.force_flush(timeout_millis=10_000) + finally: + instrumentor.uninstrument() + provider.shutdown() + return receiver.spans(), receiver.requests() + + +def _by_name(spans: List[dict]) -> Dict[str, List[dict]]: + grouped: Dict[str, List[dict]] = {} + for span in spans: + grouped.setdefault(span["name"], []).append(span) + return grouped + + +def test_real_client_calls_reach_the_collector_contract(monkeypatch): + with FakeVoyage() as fake, Receiver() as receiver: + spans, exports = _journey(receiver, fake, monkeypatch) + paths = fake.paths() + + # The real SDK made every HTTP call (no monkeypatched client methods). + assert paths == ["/v1/embeddings", "/v1/rerank", "/v1/embeddings", "/v1/rerank"] + + # Collector contract on every export. + assert exports + for export in exports: + assert export["path"] == "/tracer/v1/traces" + assert export["headers"]["x-api-key"] == FI_API_KEY + assert export["headers"]["x-secret-key"] == FI_SECRET_KEY + assert "authorization" not in export["headers"] + for resource in export["resource_attributes"]: + assert resource["project_name"] == PROJECT + assert resource["project_type"] == "observe" + + by_name = _by_name(spans) + assert sorted((name, len(group)) for name, group in by_name.items()) == [ + ("voyage.embed", 2), + ("voyage.rerank", 2), + ] + + ok_embed, failed_embed = by_name["voyage.embed"] + embed = _flatten_attributes(ok_embed["attributes"]) + assert embed["gen_ai.span.kind"] == "EMBEDDING" + assert embed["gen_ai.provider.name"] == "voyage" + assert embed["gen_ai.request.model"] == EMBED_MODEL + assert embed["embedding.model_name"] == EMBED_MODEL + assert int(embed["voyage.embedding.count"]) == 2 + assert int(embed["voyage.embedding.dimension"]) == DIMENSION + # Promoted keys the collector reads (fi-collector DeriveHotKeys aliases). + assert int(embed["gen_ai.usage.input_tokens"]) == TOKENS_PER_TEXT * 2 + assert int(embed["gen_ai.usage.total_tokens"]) == TOKENS_PER_TEXT * 2 + # register() also promotes UNSET to OK on export; the wrapper's own OK is + # pinned by the in-memory tests (test_embed, test_rerank, test_async). + assert ok_embed["status"]["code"] == "STATUS_CODE_OK" + + for rerank_span in by_name["voyage.rerank"]: + rerank = _flatten_attributes(rerank_span["attributes"]) + assert rerank["gen_ai.span.kind"] == "RERANKER" + assert rerank["reranker.model_name"] == RERANK_MODEL + assert int(rerank["voyage.rerank.document_count"]) == 3 + assert int(rerank["gen_ai.usage.total_tokens"]) == rerank_tokens(3) + assert rerank_span["status"]["code"] == "STATUS_CODE_OK" + assert int(_flatten_attributes(by_name["voyage.rerank"][0]["attributes"])["reranker.top_k"]) == 2 + + failed = _flatten_attributes(failed_embed["attributes"]) + assert failed_embed["status"]["code"] == "STATUS_CODE_ERROR" + assert any(event["name"] == "exception" for event in failed_embed.get("events", [])) + assert "gen_ai.usage.total_tokens" not in failed + + +def test_no_key_vectors_or_content_are_exported_by_default(monkeypatch): + with FakeVoyage() as fake, Receiver() as receiver: + spans, exports = _journey(receiver, fake, monkeypatch) + + wire = json.dumps(spans) + json.dumps(exports) + for secret in (VOYAGE_KEY, VECTOR_MARKER) + CONTENT_MARKERS + SCORE_MARKERS: + assert secret not in wire, secret + + +def test_capture_content_control_run_puts_the_markers_on_the_wire(monkeypatch): + # Pairs the absence check above with a run that does export content, so + # the absence cannot come from dropped spans or markers that never flowed. + with FakeVoyage() as fake, Receiver() as receiver: + spans, _exports = _journey(receiver, fake, monkeypatch, capture_content=True) + + wire = json.dumps(spans) + for marker in CONTENT_MARKERS + (SCORE_MARKERS[1],): + assert marker in wire, marker + # Vectors and the key stay off the wire even with capture on. + assert VECTOR_MARKER not in wire + assert VOYAGE_KEY not in wire diff --git a/python/frameworks/voyage/tests/test_embed.py b/python/frameworks/voyage/tests/test_embed.py new file mode 100644 index 000000000..fa9e7778c --- /dev/null +++ b/python/frameworks/voyage/tests/test_embed.py @@ -0,0 +1,172 @@ +"""voyageai.Client.embed: one EMBEDDING span per call, through the real SDK.""" + +from __future__ import annotations + +import warnings +from typing import Any, Dict + +import pytest + +pytest.importorskip("voyageai", reason="voyageai must be installed to test its instrumentor") + +import voyageai # noqa: E402 +from opentelemetry.trace import SpanKind # noqa: E402 + +from _support import ( # noqa: E402 + CONTENT_MARKERS, + DIMENSION, + EMBED_MODEL, + TEXTS, + TOKENS_PER_TEXT, + VECTOR_BASE, + VECTOR_MARKER, + VOYAGE_KEY, + FakeVoyage, + attrs, + client, + instrumented, + status_code, +) + + +@pytest.fixture() +def fake(): + with FakeVoyage() as server: + yield server + + +def _no_vector_values(values: Dict[str, Any]) -> None: + for key, value in values.items(): + if isinstance(value, (list, tuple)): + assert len(value) != DIMENSION, key + assert not any(isinstance(item, float) for item in value), key + + +def test_embed_emits_one_embedding_span_with_counts_dimension_and_tokens(fake): + with instrumented() as traced: + result = client(fake).embed(TEXTS, model=EMBED_MODEL, input_type="document") + + # The caller gets the real SDK result, unchanged. + assert len(result.embeddings) == 2 + assert result.embeddings[0][0] == pytest.approx(VECTOR_BASE) + assert result.total_tokens == TOKENS_PER_TEXT * 2 + # The real client made the HTTP call with its own key. + assert fake.paths() == ["/v1/embeddings"] + assert fake.calls[0][2] is True + + span = traced.one() + assert span.name == "voyage.embed" + assert span.kind == SpanKind.INTERNAL + values = attrs(span) + assert values["gen_ai.span.kind"] == "EMBEDDING" + assert values["gen_ai.provider.name"] == "voyage" + assert values["gen_ai.operation.name"] == "embeddings" + assert values["gen_ai.request.model"] == EMBED_MODEL + assert values["embedding.model_name"] == EMBED_MODEL + assert values["voyage.input_type"] == "document" + assert values["voyage.embedding.count"] == 2 + assert values["voyage.embedding.dimension"] == DIMENSION + assert values["gen_ai.embeddings.dimension.count"] == DIMENSION + # Voyage reports one total; for an embedding call every token is input. + assert values["gen_ai.usage.input_tokens"] == TOKENS_PER_TEXT * 2 + assert values["gen_ai.usage.total_tokens"] == TOKENS_PER_TEXT * 2 + assert "gen_ai.usage.output_tokens" not in values + assert values["server.address"] == "127.0.0.1" + # The wrapper sets OK itself (no register() processor in this provider). + assert status_code(span) == "OK" + + +def test_embed_span_carries_no_vectors_texts_or_key_by_default(fake): + with instrumented() as traced: + client(fake).embed(TEXTS, model=EMBED_MODEL, input_type="query") + + values = attrs(traced.one()) + _no_vector_values(values) + for key in ("input.value", "output.value", "embedding.embeddings"): + assert not any(name.startswith(key) for name in values), key + wire = traced.wire() + assert VECTOR_MARKER not in wire + assert VOYAGE_KEY not in wire + for marker in CONTENT_MARKERS: + assert marker not in wire, marker + + +def test_embed_reads_positional_arguments(fake): + with instrumented() as traced: + client(fake).embed(TEXTS, EMBED_MODEL, "query") + + values = attrs(traced.one()) + assert values["embedding.model_name"] == EMBED_MODEL + assert values["voyage.input_type"] == "query" + assert values["voyage.embedding.count"] == 2 + + +def test_embed_records_requested_dimension_and_dtype(fake): + with instrumented() as traced: + result = client(fake).embed(TEXTS, model=EMBED_MODEL, output_dimension=32) + + assert len(result.embeddings[0]) == 32 + values = attrs(traced.one()) + assert values["voyage.output_dimension"] == 32 + assert values["voyage.embedding.dimension"] == 32 + assert "voyage.output_dtype" not in values + assert "voyage.input_type" not in values + + +def test_packed_binary_embeddings_report_the_unpacked_dimension(fake): + with instrumented() as traced: + result = client(fake).embed( + TEXTS, model=EMBED_MODEL, output_dtype="binary", output_dimension=64 + ) + + # binary/ubinary pack eight dimensions per returned integer. + assert len(result.embeddings[0]) == 8 + values = attrs(traced.one()) + assert values["voyage.output_dtype"] == "binary" + assert values["voyage.embedding.dimension"] == 64 + _no_vector_values(values) + + +def test_a_bare_string_counts_as_one_text(fake): + with instrumented() as traced: + result = client(fake).embed("one text only", model=EMBED_MODEL) + + assert len(result.embeddings) == 1 + assert attrs(traced.one())["voyage.embedding.count"] == 1 + + +def test_the_sdk_default_model_is_recorded_when_none_is_passed(fake): + with instrumented() as traced: + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + client(fake).embed(TEXTS) + + assert fake.bodies()[0]["model"] == voyageai.VOYAGE_EMBED_DEFAULT_MODEL + values = attrs(traced.one()) + assert values["embedding.model_name"] == voyageai.VOYAGE_EMBED_DEFAULT_MODEL + assert values["gen_ai.request.model"] == voyageai.VOYAGE_EMBED_DEFAULT_MODEL + + +def test_one_span_covers_the_clients_own_retries(fake): + with instrumented() as traced: + result = client(fake, max_retries=2).embed(TEXTS, model="rate-limit-once") + + assert len(result.embeddings) == 2 + # The SDK retried the 429 itself; the call is still one span. + assert fake.paths() == ["/v1/embeddings", "/v1/embeddings"] + span = traced.one() + assert status_code(span) == "OK" + assert attrs(span)["gen_ai.usage.total_tokens"] == TOKENS_PER_TEXT * 2 + + +def test_unwrapped_methods_produce_no_span(fake): + with instrumented() as traced: + result = client(fake).multimodal_embed(inputs=[["hello"]], model="voyage-multimodal-3") + assert len(result.embeddings) == 1 + # Neither multimodal_embed nor contextualized_embed is wrapped (PRD R-07). + for cls in (voyageai.Client, voyageai.AsyncClient): + for name in ("multimodal_embed", "contextualized_embed"): + assert not hasattr(cls.__dict__[name], "__wrapped__"), (cls, name) + + assert fake.paths() == ["/v1/multimodalembeddings"] + assert traced.spans() == [] diff --git a/python/frameworks/voyage/tests/test_errors.py b/python/frameworks/voyage/tests/test_errors.py new file mode 100644 index 000000000..ce3087fd9 --- /dev/null +++ b/python/frameworks/voyage/tests/test_errors.py @@ -0,0 +1,209 @@ +"""Errors, timeouts and cancellation: recorded on the span, never changed for the caller.""" + +from __future__ import annotations + +import asyncio + +import pytest + +pytest.importorskip("voyageai", reason="voyageai must be installed to test its instrumentor") + +import voyageai # noqa: E402 +from voyageai.api_resources import api_requestor # noqa: E402 + +from _support import ( # noqa: E402 + DOCUMENTS, + EMBED_MODEL, + QUERY, + TEXTS, + VOYAGE_KEY, + FakeVoyage, + async_client, + attrs, + client, + event_names, + instrumented, + status_code, +) + +_RESULT_KEYS = ( + "gen_ai.usage.input_tokens", + "gen_ai.usage.total_tokens", + "voyage.embedding.dimension", + "voyage.rerank.result_count", + "voyage.cancelled", +) + + +@pytest.fixture() +def fake(): + with FakeVoyage() as server: + yield server + + +def test_an_api_error_sets_error_status_and_one_exception_event(fake): + with instrumented() as traced: + with pytest.raises(voyageai.error.AuthenticationError): + client(fake).embed(TEXTS, model="fail-401") + with pytest.raises(voyageai.error.AuthenticationError): + client(fake).rerank(QUERY, DOCUMENTS, model="fail-401") + + for span in traced.spans(): + assert status_code(span) == "ERROR" + assert span.status.description.startswith("AuthenticationError: ") + assert event_names(span) == ["exception"] + (event,) = span.events + assert event.attributes["exception.type"].endswith("AuthenticationError") + values = attrs(span) + # Nothing came back, so result counts and usage are absent, not 0. + for key in _RESULT_KEYS: + assert key not in values, key + embed, rerank = traced.spans() + assert attrs(embed)["voyage.embedding.count"] == 2 + assert attrs(rerank)["voyage.rerank.document_count"] == 3 + + +def test_a_key_echoed_in_an_error_message_is_redacted(fake): + with instrumented() as traced: + with pytest.raises(voyageai.error.InvalidRequestError) as raised: + client(fake).embed(TEXTS, model="echo-key") + + # The caller still sees the vendor's message unchanged. + assert VOYAGE_KEY in str(raised.value) + span = traced.one() + assert "[redacted]" in span.status.description + assert "[redacted]" in span.events[0].attributes["exception.message"] + assert VOYAGE_KEY not in traced.wire() + + +def test_a_client_timeout_ends_the_span_as_an_error(fake): + with instrumented() as traced: + with pytest.raises(voyageai.error.Timeout): + client(fake, timeout=0.3).embed(TEXTS, model="slow") + + span = traced.one() + assert status_code(span) == "ERROR" + assert span.status.description.startswith("Timeout") + assert "voyage.cancelled" not in attrs(span) + + +def test_cancelling_an_async_call_marks_the_span_cancelled(fake): + async def cancel_mid_call() -> None: + task = asyncio.ensure_future(async_client(fake).embed(TEXTS, model="slow")) + while not fake.received.is_set(): + await asyncio.sleep(0.01) + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + + with instrumented() as traced: + asyncio.run(cancel_mid_call()) + + span = traced.one() + assert status_code(span) == "ERROR" + assert span.status.description == "cancelled" + assert attrs(span)["voyage.cancelled"] is True + # Cancellation is not an exception event, and nothing came back. + assert event_names(span) == [] + assert "gen_ai.usage.total_tokens" not in attrs(span) + + +def test_a_keyboard_interrupt_during_a_sync_call_marks_the_span_cancelled(fake, monkeypatch): + def interrupted(self, *args, **kwargs): + raise KeyboardInterrupt + + monkeypatch.setattr(api_requestor.APIRequestor, "request_raw", interrupted) + with instrumented() as traced: + with pytest.raises(KeyboardInterrupt): + client(fake).rerank(QUERY, DOCUMENTS, model="rerank-2.5") + + span = traced.one() + assert span.status.description == "cancelled" + assert attrs(span)["voyage.cancelled"] is True + assert event_names(span) == [] + + +class _Unprintable(Exception): + def __str__(self) -> str: + raise RuntimeError("str() is broken") + + +def test_the_vendor_exception_is_reraised_unchanged_even_if_unprintable(fake, monkeypatch): + raised = _Unprintable() + + def broken(self, *args, **kwargs): + raise raised + + monkeypatch.setattr(api_requestor.APIRequestor, "request_raw", broken) + with instrumented() as traced: + with pytest.raises(_Unprintable) as caught: + client(fake).embed(TEXTS, model=EMBED_MODEL) + + assert caught.value is raised + span = traced.one() + assert status_code(span) == "ERROR" + assert span.end_time is not None + + +def test_a_failing_request_attribute_never_reaches_the_caller(fake, monkeypatch): + import traceai_voyage._wrappers as wrappers + + def explode(*args, **kwargs): + raise RuntimeError("attribute extraction failed") + + monkeypatch.setattr(wrappers, "_request_attributes", explode) + with instrumented() as traced: + result = client(fake).embed(TEXTS, model=EMBED_MODEL) + + assert len(result.embeddings) == 2 + values = attrs(traced.one()) + assert values["gen_ai.span.kind"] == "EMBEDDING" + assert values["gen_ai.usage.total_tokens"] == 14 + + +def test_a_failing_result_attribute_never_reaches_the_caller(fake, monkeypatch): + import traceai_voyage._wrappers as wrappers + + def explode(*args, **kwargs): + raise RuntimeError("result extraction failed") + + monkeypatch.setattr(wrappers, "_response_attributes", explode) + with instrumented() as traced: + result = client(fake).rerank(QUERY, DOCUMENTS, model="rerank-2.5") + async_result = asyncio.run( + async_client(fake).rerank(QUERY, DOCUMENTS, model="rerank-2.5") + ) + + assert [r.index for r in result.results] == [1, 0, 2] + assert [r.index for r in async_result.results] == [1, 0, 2] + for span in traced.spans(): + assert span.end_time is not None + assert status_code(span) == "OK" + + +def test_a_tracer_that_raises_never_breaks_the_vendor_call(fake): + from opentelemetry import trace as trace_api + + from traceai_voyage import VoyageInstrumentor + + class ExplodingTracer(trace_api.Tracer): + def start_span(self, *args, **kwargs): + raise RuntimeError("tracer is broken") + + def start_as_current_span(self, *args, **kwargs): + raise RuntimeError("tracer is broken") + + class ExplodingProvider(trace_api.TracerProvider): + def get_tracer(self, *args, **kwargs): + return ExplodingTracer() + + instrumentor = VoyageInstrumentor() + instrumentor.instrument(tracer_provider=ExplodingProvider()) + try: + result = client(fake).embed(TEXTS, model=EMBED_MODEL) + async_result = asyncio.run(async_client(fake).embed(TEXTS, model=EMBED_MODEL)) + finally: + instrumentor.uninstrument() + + assert len(result.embeddings) == 2 + assert len(async_result.embeddings) == 2 diff --git a/python/frameworks/voyage/tests/test_example.py b/python/frameworks/voyage/tests/test_example.py new file mode 100644 index 000000000..c839e803b --- /dev/null +++ b/python/frameworks/voyage/tests/test_example.py @@ -0,0 +1,90 @@ +"""examples/embed_and_rerank.py runs end to end through the shared harness. + +harness.run starts the example as a subprocess with FI_BASE_URL pointed at +harness.Receiver and VOYAGE_BASE_URL pointed at the loopback Voyage fake, so +the real exporter and the real voyageai client both run. Nothing calls Voyage. +""" + +from __future__ import annotations + +import json +import os +import sys +from pathlib import Path + +import pytest + +pytest.importorskip("voyageai", reason="voyageai must be installed to run the example") +pytest.importorskip("opentelemetry.proto", reason="the harness decodes OTLP protobuf") + +PACKAGE = Path(__file__).resolve().parents[1] +PYTHON_ROOT = PACKAGE.parents[1] +sys.path.insert(0, str(PYTHON_ROOT / "tests")) + +from _support import ( # noqa: E402 + CONTENT_MARKERS, + SCORE_MARKERS, + VECTOR_MARKER, + VOYAGE_KEY, + FakeVoyage, +) +from harness import Receiver, _flatten_attributes, run # noqa: E402 + +EXAMPLE = PACKAGE / "examples" / "embed_and_rerank.py" +FI_API_KEY = "placeholder-fi-api-key" +FI_SECRET_KEY = "placeholder-fi-secret-key" + + +def test_example_exports_one_embedding_and_one_reranker_span(): + with FakeVoyage() as fake, Receiver() as receiver: + env = { + key: value + for key, value in os.environ.items() + if not key.startswith(("FI_", "VOYAGE_", "OTEL_")) + } + env.update( + { + "FI_BASE_URL": receiver.origin, + "FI_API_KEY": FI_API_KEY, + "FI_SECRET_KEY": FI_SECRET_KEY, + "VOYAGE_API_KEY": VOYAGE_KEY, + "VOYAGE_BASE_URL": fake.base_url, + "PYTHONPATH": os.pathsep.join( + [str(PACKAGE), str(PYTHON_ROOT), os.environ.get("PYTHONPATH", "")] + ), + } + ) + result = run([sys.executable, str(EXAMPLE)], env, None, timeout=120) + spans = receiver.spans() + exports = receiver.requests() + paths = fake.paths() + + assert not result.timed_out + assert result.returncode == 0, result.stderr.decode(errors="replace") + assert paths == ["/v1/embeddings", "/v1/rerank"] + + assert sorted(span["name"] for span in spans) == ["voyage.embed", "voyage.rerank"] + by_name = {span["name"]: _flatten_attributes(span["attributes"]) for span in spans} + assert by_name["voyage.embed"]["gen_ai.span.kind"] == "EMBEDDING" + assert int(by_name["voyage.embed"]["voyage.embedding.count"]) == 2 + assert int(by_name["voyage.embed"]["gen_ai.usage.total_tokens"]) > 0 + assert by_name["voyage.rerank"]["gen_ai.span.kind"] == "RERANKER" + assert int(by_name["voyage.rerank"]["reranker.top_k"]) == 2 + for span in spans: + assert span["status"]["code"] == "STATUS_CODE_OK" + + assert exports + for export in exports: + assert export["path"] == "/tracer/v1/traces" + assert export["headers"]["x-api-key"] == FI_API_KEY + assert export["headers"]["x-secret-key"] == FI_SECRET_KEY + for resource in export["resource_attributes"]: + assert resource["project_name"] == "voyage-example" + assert resource["project_type"] == "observe" + + # The example's own documents and query, its key, vectors and scores + # stay off the wire (content capture is off by default). + wire = json.dumps(spans) + example_markers = ("EXAMPLE-DOC", "EXAMPLE-QUERY") + for marker in (VOYAGE_KEY, VECTOR_MARKER) + example_markers + CONTENT_MARKERS + SCORE_MARKERS: + assert marker not in wire, marker diff --git a/python/frameworks/voyage/tests/test_packaging.py b/python/frameworks/voyage/tests/test_packaging.py new file mode 100644 index 000000000..5ec77e497 --- /dev/null +++ b/python/frameworks/voyage/tests/test_packaging.py @@ -0,0 +1,108 @@ +"""Packaging contract: ranges, declared Pythons, license, and no base_url from us.""" + +from __future__ import annotations + +import re +from pathlib import Path + +import pytest +from packaging.requirements import Requirement +from packaging.specifiers import SpecifierSet + +pytest.importorskip("voyageai", reason="voyageai must be installed to test its instrumentor") + +import voyageai # noqa: E402 +from traceai_voyage import VoyageInstrumentor # noqa: E402 +from traceai_voyage.package import _instruments # noqa: E402 + +PACKAGE = Path(__file__).resolve().parents[1] +PYPROJECT = PACKAGE / "pyproject.toml" +SOURCE = PACKAGE / "traceai_voyage" + + +def _dependency(name: str) -> SpecifierSet: + match = re.search( + r'^{0}\s*=\s*"([^"]+)"'.format(re.escape(name)), PYPROJECT.read_text(), re.M + ) + assert match, name + return SpecifierSet(match.group(1)) + + +def test_voyageai_is_a_range_from_the_atlas_floor_to_the_next_major(): + (requirement,) = VoyageInstrumentor().instrumentation_dependencies() + instruments = Requirement(requirement) + assert instruments.name == "voyageai" + # pyproject and the runtime dependency check agree. + assert instruments.specifier == _dependency("voyageai") + for version in ("0.3.7", "0.4.0", "0.4.1", "0.5.0", "0.9.0"): + assert instruments.specifier.contains(version), version + for version in ("0.3.6", "0.3.5", "1.0.0", "1.2.0"): + assert not instruments.specifier.contains(version), version + assert tuple(_instruments) == tuple(VoyageInstrumentor().instrumentation_dependencies()) + + +@pytest.mark.parametrize(("installed", "wrapped"), [("0.4.1", True), ("0.3.6", False), ("1.0.0", False)]) +def test_the_runtime_dependency_check_accepts_only_the_range(monkeypatch, installed, wrapped): + import opentelemetry.instrumentation.dependencies as dependencies + + real_version = dependencies.version + + def version(name): + return installed if name == "voyageai" else real_version(name) + + monkeypatch.setattr(dependencies, "version", version) + instrumentor = VoyageInstrumentor() + instrumentor.instrument() + try: + assert hasattr(voyageai.Client.__dict__["embed"], "__wrapped__") is wrapped + finally: + instrumentor.uninstrument() + assert not hasattr(voyageai.Client.__dict__["embed"], "__wrapped__") + + +def test_fi_instrumentation_floor_is_1_1_0(): + # 0.1.x export to the retired create_otel_span path and the published + # 1.0.0 fails while building its headers; 1.1.0 is the first good release. + spec = _dependency("fi-instrumentation-otel") + assert spec.contains("1.1.0") + assert spec.contains("1.2.0") + for version in ("0.1.11", "0.1.16", "1.0.0"): + assert not spec.contains(version), version + + +def test_classifiers_list_only_the_tested_python_versions(): + text = PYPROJECT.read_text() + versions = re.findall(r'"Programming Language :: Python :: (3\.\d+)"', text) + assert versions == ["3.10", "3.11", "3.13"] + assert _dependency("python") == SpecifierSet(">=3.10,<3.14") + + +def test_distribution_name_and_license(): + text = PYPROJECT.read_text() + assert re.search(r'^name = "traceAI-voyage"$', text, re.M) + assert re.search(r'^license = "Apache-2.0"$', text, re.M) + assert '"License :: OSI Approved :: Apache Software License"' in text + + +def test_the_package_never_sets_base_url(): + # D14: the client picks the host from the key; the instrumentor never does. + for path in SOURCE.glob("*.py"): + source = path.read_text() + assert not re.search(r"base_url\s*=", source), path + assert not re.search(r"""["']base_url["']\s*\]\s*=""", source), path + for host in ("api.voyageai.com", "ai.mongodb.com"): + assert host not in source, (path, host) + + +def test_instrumenting_leaves_the_clients_host_choice_alone(): + keys = ("pa-placeholder-legacy-key", "al-placeholder-atlas-key") + before = {key: voyageai.Client(api_key=key)._params["base_url"] for key in keys} + instrumentor = VoyageInstrumentor() + instrumentor.instrument() + try: + after = {key: voyageai.Client(api_key=key)._params["base_url"] for key in keys} + explicit = voyageai.Client(api_key=keys[0], base_url="https://example.invalid/v1") + finally: + instrumentor.uninstrument() + assert after == before + assert explicit._params["base_url"] == "https://example.invalid/v1" diff --git a/python/frameworks/voyage/tests/test_rerank.py b/python/frameworks/voyage/tests/test_rerank.py new file mode 100644 index 000000000..99096398a --- /dev/null +++ b/python/frameworks/voyage/tests/test_rerank.py @@ -0,0 +1,80 @@ +"""voyageai.Client.rerank: one RERANKER span per call, through the real SDK.""" + +from __future__ import annotations + +import pytest + +pytest.importorskip("voyageai", reason="voyageai must be installed to test its instrumentor") + +from _support import ( # noqa: E402 + CONTENT_MARKERS, + DOCUMENTS, + QUERY, + RERANK_MODEL, + SCORE_MARKERS, + VOYAGE_KEY, + FakeVoyage, + attrs, + client, + instrumented, + rerank_tokens, + status_code, +) + + +@pytest.fixture() +def fake(): + with FakeVoyage() as server: + yield server + + +def test_rerank_emits_one_reranker_span_with_counts_top_k_and_tokens(fake): + with instrumented() as traced: + result = client(fake).rerank(QUERY, DOCUMENTS, model=RERANK_MODEL, top_k=2) + + # The client's order is the result order; nothing is re-sorted. + assert [r.index for r in result.results] == [1, 0] + assert fake.paths() == ["/v1/rerank"] + + span = traced.one() + assert span.name == "voyage.rerank" + values = attrs(span) + assert values["gen_ai.span.kind"] == "RERANKER" + assert values["gen_ai.provider.name"] == "voyage" + assert values["gen_ai.operation.name"] == "rerank" + assert values["gen_ai.request.model"] == RERANK_MODEL + assert values["reranker.model_name"] == RERANK_MODEL + assert values["reranker.top_k"] == 2 + assert values["voyage.rerank.document_count"] == 3 + assert values["voyage.rerank.result_count"] == 2 + assert values["gen_ai.usage.input_tokens"] == rerank_tokens(3) + assert values["gen_ai.usage.total_tokens"] == rerank_tokens(3) + assert values["server.address"] == "127.0.0.1" + assert status_code(span) == "OK" + + +def test_rerank_span_carries_no_query_documents_scores_or_key_by_default(fake): + with instrumented() as traced: + client(fake).rerank(QUERY, DOCUMENTS, model=RERANK_MODEL, top_k=2) + + values = attrs(traced.one()) + for key in ("reranker.query", "reranker.input_documents", "reranker.output_documents"): + assert not any(name.startswith(key) for name in values), key + assert "input.value" not in values + assert "output.value" not in values + wire = traced.wire() + assert VOYAGE_KEY not in wire + for marker in CONTENT_MARKERS + SCORE_MARKERS: + assert marker not in wire, marker + + +def test_rerank_reads_positional_arguments_and_omits_an_unset_top_k(fake): + with instrumented() as traced: + result = client(fake).rerank(QUERY, DOCUMENTS, RERANK_MODEL) + + assert len(result.results) == 3 + values = attrs(traced.one()) + assert values["reranker.model_name"] == RERANK_MODEL + assert values["voyage.rerank.document_count"] == 3 + assert values["voyage.rerank.result_count"] == 3 + assert "reranker.top_k" not in values diff --git a/python/frameworks/voyage/tests/test_uninstrument.py b/python/frameworks/voyage/tests/test_uninstrument.py new file mode 100644 index 000000000..cfb49d253 --- /dev/null +++ b/python/frameworks/voyage/tests/test_uninstrument.py @@ -0,0 +1,110 @@ +"""uninstrument() restores every wrapped method; construction order does not matter.""" + +from __future__ import annotations + +import asyncio + +import pytest + +pytest.importorskip("voyageai", reason="voyageai must be installed to test its instrumentor") + +import voyageai # noqa: E402 +from opentelemetry.sdk.trace import TracerProvider # noqa: E402 +from opentelemetry.sdk.trace.export import SimpleSpanProcessor # noqa: E402 +from opentelemetry.sdk.trace.export.in_memory_span_exporter import ( # noqa: E402 + InMemorySpanExporter, +) +from traceai_voyage import VoyageInstrumentor # noqa: E402 + +from _support import ( # noqa: E402 + DOCUMENTS, + EMBED_MODEL, + QUERY, + RERANK_MODEL, + TEXTS, + FakeVoyage, + async_client, + client, + instrumented, +) + +_WRAPPED = [ + (voyageai.Client, "embed"), + (voyageai.Client, "rerank"), + (voyageai.AsyncClient, "embed"), + (voyageai.AsyncClient, "rerank"), +] + + +@pytest.fixture() +def fake(): + with FakeVoyage() as server: + yield server + + +def _all_calls(fake) -> None: + voyage = client(fake) + voyage.embed(TEXTS, model=EMBED_MODEL) + voyage.rerank(QUERY, DOCUMENTS, model=RERANK_MODEL) + + async def calls() -> None: + avoyage = async_client(fake) + await avoyage.embed(TEXTS, model=EMBED_MODEL) + await avoyage.rerank(QUERY, DOCUMENTS, model=RERANK_MODEL) + + asyncio.run(calls()) + + +def test_instrument_wraps_exactly_four_methods_and_uninstrument_restores_them(fake): + originals = {(cls, name): cls.__dict__[name] for cls, name in _WRAPPED} + with instrumented() as traced: + for cls, name in _WRAPPED: + assert hasattr(cls.__dict__[name], "__wrapped__"), (cls, name) + _all_calls(fake) + assert [span.name for span in traced.spans()] == [ + "voyage.embed", + "voyage.rerank", + "voyage.embed", + "voyage.rerank", + ] + + for (cls, name), original in originals.items(): + assert cls.__dict__[name] is original, (cls, name) + + # No spans once uninstrumented. + _all_calls(fake) + assert len(traced.spans()) == 4 + + +def test_instrument_again_after_uninstrument(fake): + with instrumented(): + pass + with instrumented() as traced: + client(fake).embed(TEXTS, model=EMBED_MODEL) + assert len(traced.spans()) == 1 + + +def test_a_second_instrument_call_does_not_double_wrap(fake): + exporter = InMemorySpanExporter() + provider = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + instrumentor = VoyageInstrumentor() + instrumentor.instrument(tracer_provider=provider) + try: + VoyageInstrumentor().instrument(tracer_provider=provider) + client(fake).embed(TEXTS, model=EMBED_MODEL) + finally: + instrumentor.uninstrument() + assert len(exporter.get_finished_spans()) == 1 + assert not hasattr(voyageai.Client.__dict__["embed"], "__wrapped__") + + +def test_a_client_built_before_instrument_is_traced(fake): + # Methods are patched on the class, so construction order does not matter; + # only calls made while instrumented are traced. + early = client(fake) + early.embed(TEXTS, model=EMBED_MODEL) + with instrumented() as traced: + early.embed(TEXTS, model=EMBED_MODEL) + early.embed(TEXTS, model=EMBED_MODEL) + assert len(traced.spans()) == 1 From 8565287058cb127fcd65bcfaf580eadccffe6364 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Mon, 5 Oct 2026 01:26:13 -0700 Subject: [PATCH 05/17] test(voyage): cover in-process local models; format tests voyageai 0.5 runs some embedding models (voyage-4-nano) inside Client.embed with no request. The span must still be recorded, without a server.address the call never used. Skipped on voyageai < 0.5, which has no local models. Also applies ruff format (line length 100). approved: Nikhil 2026-10-03 blanket Refs: TH-8321 --- python/frameworks/voyage/tests/_support.py | 2 +- .../voyage/tests/test_contract_harness.py | 8 +++-- python/frameworks/voyage/tests/test_embed.py | 31 +++++++++++++++++++ python/frameworks/voyage/tests/test_errors.py | 4 +-- .../frameworks/voyage/tests/test_packaging.py | 8 ++--- 5 files changed, 43 insertions(+), 10 deletions(-) diff --git a/python/frameworks/voyage/tests/_support.py b/python/frameworks/voyage/tests/_support.py index 17c7a998c..ad6bded23 100644 --- a/python/frameworks/voyage/tests/_support.py +++ b/python/frameworks/voyage/tests/_support.py @@ -124,7 +124,7 @@ def do_POST(self) -> None: # noqa: N802 - BaseHTTPRequestHandler API elif model == "fail-401": self._send(401, {"detail": "Provided API key is invalid."}) elif model == "echo-key": - key = authorization[len("Bearer "):] + key = authorization[len("Bearer ") :] self._send(400, {"detail": "bad request for key {0}".format(key)}) elif model == "rate-limit-once" and owner._take_rate_limit(): self._send(429, {"detail": "rate limited"}) diff --git a/python/frameworks/voyage/tests/test_contract_harness.py b/python/frameworks/voyage/tests/test_contract_harness.py index 79da146e9..d16118acd 100644 --- a/python/frameworks/voyage/tests/test_contract_harness.py +++ b/python/frameworks/voyage/tests/test_contract_harness.py @@ -65,7 +65,9 @@ def _journey( try: voyage = client(fake) assert len(voyage.embed(TEXTS, model=EMBED_MODEL, input_type="document").embeddings) == 2 - assert [r.index for r in voyage.rerank(QUERY, DOCUMENTS, RERANK_MODEL, top_k=2).results] == [ + assert [ + r.index for r in voyage.rerank(QUERY, DOCUMENTS, RERANK_MODEL, top_k=2).results + ] == [ 1, 0, ] @@ -133,7 +135,9 @@ def test_real_client_calls_reach_the_collector_contract(monkeypatch): assert int(rerank["voyage.rerank.document_count"]) == 3 assert int(rerank["gen_ai.usage.total_tokens"]) == rerank_tokens(3) assert rerank_span["status"]["code"] == "STATUS_CODE_OK" - assert int(_flatten_attributes(by_name["voyage.rerank"][0]["attributes"])["reranker.top_k"]) == 2 + assert ( + int(_flatten_attributes(by_name["voyage.rerank"][0]["attributes"])["reranker.top_k"]) == 2 + ) failed = _flatten_attributes(failed_embed["attributes"]) assert failed_embed["status"]["code"] == "STATUS_CODE_ERROR" diff --git a/python/frameworks/voyage/tests/test_embed.py b/python/frameworks/voyage/tests/test_embed.py index fa9e7778c..7228133da 100644 --- a/python/frameworks/voyage/tests/test_embed.py +++ b/python/frameworks/voyage/tests/test_embed.py @@ -170,3 +170,34 @@ def test_unwrapped_methods_produce_no_span(fake): assert fake.paths() == ["/v1/multimodalembeddings"] assert traced.spans() == [] + + +def test_a_local_model_is_traced_without_a_server_address(fake, monkeypatch): + helpers = pytest.importorskip( + "voyageai.local.helpers", reason="voyageai < 0.5 has no local embedding models" + ) + import voyageai.client as client_module + from voyageai.object import EmbeddingsObject + + model = "voyage-4-nano" + assert helpers.is_local_model(model) + + def embed_local(texts, model, **kwargs): + # Stands in for the sentence-transformers backend (torch is not installed). + result = EmbeddingsObject() + result.embeddings = [[VECTOR_BASE] * DIMENSION for _ in texts] + result.total_tokens = 5 + return result + + monkeypatch.setattr(client_module, "embed_local", embed_local) + with instrumented() as traced: + # The client has a key and a base URL, but a local model sends no request. + result = client(fake).embed(TEXTS, model=model) + + assert len(result.embeddings) == 2 + assert fake.paths() == [] + values = attrs(traced.one()) + assert values["embedding.model_name"] == model + assert values["voyage.embedding.count"] == 2 + assert values["gen_ai.usage.total_tokens"] == 5 + assert "server.address" not in values diff --git a/python/frameworks/voyage/tests/test_errors.py b/python/frameworks/voyage/tests/test_errors.py index ce3087fd9..773393992 100644 --- a/python/frameworks/voyage/tests/test_errors.py +++ b/python/frameworks/voyage/tests/test_errors.py @@ -170,9 +170,7 @@ def explode(*args, **kwargs): monkeypatch.setattr(wrappers, "_response_attributes", explode) with instrumented() as traced: result = client(fake).rerank(QUERY, DOCUMENTS, model="rerank-2.5") - async_result = asyncio.run( - async_client(fake).rerank(QUERY, DOCUMENTS, model="rerank-2.5") - ) + async_result = asyncio.run(async_client(fake).rerank(QUERY, DOCUMENTS, model="rerank-2.5")) assert [r.index for r in result.results] == [1, 0, 2] assert [r.index for r in async_result.results] == [1, 0, 2] diff --git a/python/frameworks/voyage/tests/test_packaging.py b/python/frameworks/voyage/tests/test_packaging.py index 5ec77e497..e148fe036 100644 --- a/python/frameworks/voyage/tests/test_packaging.py +++ b/python/frameworks/voyage/tests/test_packaging.py @@ -21,9 +21,7 @@ def _dependency(name: str) -> SpecifierSet: - match = re.search( - r'^{0}\s*=\s*"([^"]+)"'.format(re.escape(name)), PYPROJECT.read_text(), re.M - ) + match = re.search(r'^{0}\s*=\s*"([^"]+)"'.format(re.escape(name)), PYPROJECT.read_text(), re.M) assert match, name return SpecifierSet(match.group(1)) @@ -41,7 +39,9 @@ def test_voyageai_is_a_range_from_the_atlas_floor_to_the_next_major(): assert tuple(_instruments) == tuple(VoyageInstrumentor().instrumentation_dependencies()) -@pytest.mark.parametrize(("installed", "wrapped"), [("0.4.1", True), ("0.3.6", False), ("1.0.0", False)]) +@pytest.mark.parametrize( + ("installed", "wrapped"), [("0.4.1", True), ("0.3.6", False), ("1.0.0", False)] +) def test_the_runtime_dependency_check_accepts_only_the_range(monkeypatch, installed, wrapped): import opentelemetry.instrumentation.dependencies as dependencies From 01c923e2f31c9aba42a7946f1f6ebbbecf6de3f0 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Mon, 5 Oct 2026 01:26:13 -0700 Subject: [PATCH 06/17] feat(voyage): add traceAI-voyage for voyageai embed and rerank VoyageInstrumentor wraps voyageai.Client and AsyncClient embed (one voyage.embed EMBEDDING span per call) and rerank (one voyage.rerank RERANKER span per call). Method names follow the installed SDK (voyageai 0.3.7 to 0.5.0); multimodal_embed and contextualized_embed are left unwrapped (PRD R-07). - Counts, requested dtype/dimension, returned dimension (x8 for packed binary dtypes), model, server.address host, and the client's total_tokens on gen_ai.usage.input_tokens/total_tokens, the keys the collector promotes; usage is set once, on the Voyage span only. - Content (texts, rerank query/documents, scores) only with capture_content=True, honouring TraceConfig hide_inputs/hide_outputs and pii_redaction, capped at 64 items x 2 KB. Vectors are never stored. - The client's API key (client.api_key, _params, voyageai.api_key) is redacted from captured content, error status and exception events. - The span is current during the vendor call (HTTP spans nest), covers the client's retries, marks cancellation (asyncio cancel, Ctrl-C) as ERROR 'cancelled' with voyage.cancelled=true, and never lets an instrumentation failure reach the caller. uninstrument() restores the original functions. base_url is never set. - voyageai>=0.3.7,<1, fi-instrumentation-otel>=1.1.0, Apache-2.0, classifiers for Python 3.10/3.11/3.13 only. - examples/embed_and_rerank.py, run by the suite through harness.run. approved: Nikhil 2026-10-03 blanket Refs: TH-8321 --- .../voyage/examples/embed_and_rerank.py | 50 ++ python/frameworks/voyage/pyproject.toml | 33 ++ .../voyage/traceai_voyage/__init__.py | 108 ++++ .../voyage/traceai_voyage/_wrappers.py | 560 ++++++++++++++++++ .../voyage/traceai_voyage/package.py | 1 + .../frameworks/voyage/traceai_voyage/py.typed | 0 .../voyage/traceai_voyage/version.py | 1 + 7 files changed, 753 insertions(+) create mode 100644 python/frameworks/voyage/examples/embed_and_rerank.py create mode 100644 python/frameworks/voyage/pyproject.toml create mode 100644 python/frameworks/voyage/traceai_voyage/__init__.py create mode 100644 python/frameworks/voyage/traceai_voyage/_wrappers.py create mode 100644 python/frameworks/voyage/traceai_voyage/package.py create mode 100644 python/frameworks/voyage/traceai_voyage/py.typed create mode 100644 python/frameworks/voyage/traceai_voyage/version.py diff --git a/python/frameworks/voyage/examples/embed_and_rerank.py b/python/frameworks/voyage/examples/embed_and_rerank.py new file mode 100644 index 000000000..2f3f2e23d --- /dev/null +++ b/python/frameworks/voyage/examples/embed_and_rerank.py @@ -0,0 +1,50 @@ +"""Trace one Voyage embed call and one rerank call. + +Set FI_API_KEY, FI_SECRET_KEY and VOYAGE_API_KEY, then run this file. The +Voyage client picks its host from the key (Atlas keys go to ai.mongodb.com, +Voyage platform keys to api.voyageai.com). VOYAGE_BASE_URL is only read so a +test can point the client at a local fake; leave it unset otherwise. +""" + +import os + +import voyageai +from fi_instrumentation import register +from fi_instrumentation.fi_types import ProjectType +from traceai_voyage import VoyageInstrumentor + +DOCUMENTS = [ + "EXAMPLE-DOC: Voyage embeddings turn text into vectors.", + "EXAMPLE-DOC: A reranker orders documents by relevance to a query.", +] +QUERY = "EXAMPLE-QUERY: which document explains reranking?" + + +def main() -> None: + tracer_provider = register( + project_name="voyage-example", + project_type=ProjectType.OBSERVE, + batch=False, + verbose=False, + ) + VoyageInstrumentor().instrument(tracer_provider=tracer_provider) + + # The client reads VOYAGE_API_KEY. base_url=None keeps the client's own + # host choice; the instrumentor never sets one. + client = voyageai.Client(base_url=os.getenv("VOYAGE_BASE_URL")) + + embeddings = client.embed(DOCUMENTS, model="voyage-3.5", input_type="document") + print( + "embedded {0} documents, {1} tokens".format( + len(embeddings.embeddings), embeddings.total_tokens + ) + ) + + reranking = client.rerank(QUERY, DOCUMENTS, model="rerank-2.5", top_k=2) + print("top document index: {0}".format(reranking.results[0].index)) + + tracer_provider.force_flush() + + +if __name__ == "__main__": + main() diff --git a/python/frameworks/voyage/pyproject.toml b/python/frameworks/voyage/pyproject.toml new file mode 100644 index 000000000..65b6b3d25 --- /dev/null +++ b/python/frameworks/voyage/pyproject.toml @@ -0,0 +1,33 @@ +[tool.poetry] +name = "traceAI-voyage" +version = "0.1.0" +description = "OpenTelemetry instrumentation for the Voyage AI Python client" +authors = ["Future AGI "] +license = "Apache-2.0" +readme = "README.md" +packages = [ + { include = "traceai_voyage" } +] +classifiers = [ + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "License :: OSI Approved :: Apache Software License", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.13", + "Topic :: Software Development :: Libraries :: Python Modules", +] + +[tool.poetry.dependencies] +python = ">=3.10,<3.14" +fi-instrumentation-otel = ">=1.1.0" +voyageai = ">=0.3.7,<1" +wrapt = ">=1.15.0" + +[tool.poetry.group.dev.dependencies] +pytest = ">=7.0.0" + +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" diff --git a/python/frameworks/voyage/traceai_voyage/__init__.py b/python/frameworks/voyage/traceai_voyage/__init__.py new file mode 100644 index 000000000..728d05190 --- /dev/null +++ b/python/frameworks/voyage/traceai_voyage/__init__.py @@ -0,0 +1,108 @@ +"""OpenTelemetry instrumentation for the official Voyage AI Python client. + +``VoyageInstrumentor`` wraps ``voyageai.Client`` and ``voyageai.AsyncClient`` +``embed`` (one EMBEDDING span per call) and ``rerank`` (one RERANKER span per +call). It never sets ``base_url``: the client picks its host from the key. +""" + +import inspect +import logging +from importlib import import_module +from typing import Any, Collection, Dict, Tuple + +from fi_instrumentation import FITracer, TraceConfig +from opentelemetry import trace as trace_api +from opentelemetry.instrumentation.instrumentor import BaseInstrumentor +from wrapt import wrap_function_wrapper + +from traceai_voyage._wrappers import EMBED, RERANK, AsyncWrapper, Operation, SyncWrapper +from traceai_voyage.package import _instruments +from traceai_voyage.version import __version__ + +logger = logging.getLogger(__name__) +logger.addHandler(logging.NullHandler()) + +# (module, class, method, operation). multimodal_embed and contextualized_embed +# are deliberately not wrapped in v1 (PRD R-07): they produce no span. +_TARGETS: Tuple[Tuple[str, str, str, Operation], ...] = ( + ("voyageai.client", "Client", "embed", EMBED), + ("voyageai.client", "Client", "rerank", RERANK), + ("voyageai.client_async", "AsyncClient", "embed", EMBED), + ("voyageai.client_async", "AsyncClient", "rerank", RERANK), +) + + +class VoyageInstrumentor(BaseInstrumentor): # type: ignore[misc] + """Trace Voyage ``embed`` and ``rerank`` on the sync and async clients. + + ``instrument()`` accepts: + + * ``tracer_provider``: the provider to use (default: the global one). + * ``config``: a ``fi_instrumentation.TraceConfig``. ``hide_inputs`` and + ``hide_outputs`` apply to captured content; ``pii_redaction`` applies to + every recorded value. + * ``capture_content``: ``False`` by default. When ``True``, spans also + record the texts, the rerank query and documents, and the rerank scores, + subject to ``config``. Embedding vectors are never recorded. + """ + + def instrumentation_dependencies(self) -> Collection[str]: + return _instruments + + def _instrument(self, **kwargs: Any) -> None: + config = kwargs.get("config") + if config is None: + config = TraceConfig() + elif not isinstance(config, TraceConfig): + raise TypeError( + "config must be a fi_instrumentation.TraceConfig, not {0}".format( + type(config).__name__ + ) + ) + capture_content = kwargs.get("capture_content", False) + if not isinstance(capture_content, bool): + raise TypeError( + "capture_content must be a bool, not {0}".format(type(capture_content).__name__) + ) + tracer_provider = kwargs.get("tracer_provider") or trace_api.get_tracer_provider() + tracer = FITracer( + trace_api.get_tracer(__name__, __version__, tracer_provider), + config=config, + ) + + self._originals: Dict[Tuple[str, str, str], Any] = {} + for module_name, class_name, method_name, operation in _TARGETS: + try: + module = import_module(module_name) + original = getattr(module, class_name).__dict__[method_name] + except (ImportError, AttributeError, KeyError): + logger.warning( + "voyageai %s.%s.%s was not found; it is not traced", + module_name, + class_name, + method_name, + ) + continue + wrapper_class = AsyncWrapper if inspect.iscoroutinefunction(original) else SyncWrapper + wrap_function_wrapper( + module, + "{0}.{1}".format(class_name, method_name), + wrapper_class(tracer, operation, config, capture_content), + ) + self._originals[(module_name, class_name, method_name)] = original + logger.debug("Voyage instrumentation enabled") + + def _uninstrument(self, **kwargs: Any) -> None: + originals = getattr(self, "_originals", {}) + for (module_name, class_name, method_name), original in originals.items(): + try: + setattr(getattr(import_module(module_name), class_name), method_name, original) + except Exception: + logger.warning( + "Could not restore voyageai %s.%s.%s", module_name, class_name, method_name + ) + originals.clear() + logger.debug("Voyage instrumentation disabled") + + +__all__ = ["VoyageInstrumentor", "__version__"] diff --git a/python/frameworks/voyage/traceai_voyage/_wrappers.py b/python/frameworks/voyage/traceai_voyage/_wrappers.py new file mode 100644 index 000000000..1ed82c9e0 --- /dev/null +++ b/python/frameworks/voyage/traceai_voyage/_wrappers.py @@ -0,0 +1,560 @@ +"""Span wrappers for voyageai ``Client`` / ``AsyncClient`` ``embed`` and ``rerank``. + +One span per public call. The span is current while the SDK sends HTTP, so an +HTTP client instrumentation nests under it, and it covers the client's own +retries. Content (texts, query, documents, scores) is recorded only with +``capture_content=True``; embedding vectors are never recorded. Nothing in +this module may change what the caller gets back or raises. +""" + +from __future__ import annotations + +import asyncio +import json +import logging +import sys +import traceback +from dataclasses import dataclass +from typing import Any, Callable, Dict, List, Mapping, Optional, Sequence, Tuple +from urllib.parse import urlsplit + +from fi_instrumentation import TraceConfig +from fi_instrumentation.fi_types import FiSpanKindValues, RerankerAttributes, SpanAttributes +from opentelemetry import context as context_api +from opentelemetry import trace as trace_api +from opentelemetry.trace import Span, Status, StatusCode, Tracer + +logger = logging.getLogger(__name__) + +PROVIDER = "voyage" +REDACTED = "[redacted]" +CANCELLED = "voyage.cancelled" +INPUT_TYPE = "voyage.input_type" +OUTPUT_DTYPE = "voyage.output_dtype" +OUTPUT_DIMENSION = "voyage.output_dimension" +EMBEDDING_COUNT = "voyage.embedding.count" +EMBEDDING_DIMENSION = "voyage.embedding.dimension" +RERANK_DOCUMENT_COUNT = "voyage.rerank.document_count" +RERANK_RESULT_COUNT = "voyage.rerank.result_count" +SERVER_ADDRESS = "server.address" +JSON_MIME_TYPE = "application/json" + +# Captured content limits: at most this many texts or documents, each cut to +# this many UTF-8 bytes on a character boundary. Counts stay exact. +MAX_CAPTURED_ITEMS = 64 +MAX_CAPTURED_BYTES = 2048 + +# binary / ubinary pack eight dimensions into each returned integer. +_PACKED_DTYPES = ("binary", "ubinary") + +# A Ctrl-C during a blocking call, or a cancelled asyncio task, is a +# cancellation rather than a failure of the call. +_CANCELLATIONS = (asyncio.CancelledError, KeyboardInterrupt) + +_EMBED_PARAMETERS = ( + "texts", + "model", + "input_type", + "truncation", + "output_dtype", + "output_dimension", +) +_RERANK_PARAMETERS = ("query", "documents", "model", "top_k", "truncation") + + +@dataclass(frozen=True) +class Operation: + """What one wrapped method records.""" + + span_name: str + span_kind: str + operation_name: str + parameters: Tuple[str, ...] + + +EMBED = Operation( + span_name="voyage.embed", + span_kind=FiSpanKindValues.EMBEDDING.value, + operation_name="embeddings", + parameters=_EMBED_PARAMETERS, +) +RERANK = Operation( + span_name="voyage.rerank", + span_kind=FiSpanKindValues.RERANKER.value, + operation_name="rerank", + parameters=_RERANK_PARAMETERS, +) + + +def _argument( + operation: Operation, args: Sequence[Any], kwargs: Mapping[str, Any], name: str +) -> Any: + """Read a vendor argument passed by keyword or by position (``self`` excluded).""" + if name in kwargs: + return kwargs[name] + index = operation.parameters.index(name) + return args[index] if index < len(args) else None + + +def _voyage_module() -> Any: + return sys.modules.get("voyageai") + + +def _secrets(instance: Any) -> List[str]: + """Every API key the client could send: never exported, wherever it appears. + + voyageai keeps the key on ``client.api_key`` and ``client._params["api_key"]`` + (read from the argument, ``voyageai.api_key``, ``VOYAGE_API_KEY`` or a key + file); a module-level ``voyageai.api_key`` is included too. + """ + candidates: List[Any] = [getattr(instance, "api_key", None)] + params = getattr(instance, "_params", None) + if isinstance(params, Mapping): + candidates.append(params.get("api_key")) + module = _voyage_module() + if module is not None: + candidates.append(getattr(module, "api_key", None)) + secrets: List[str] = [] + for candidate in candidates: + if isinstance(candidate, str) and candidate and candidate not in secrets: + secrets.append(candidate) + # Longest first, so a key that contains another is removed whole. + return sorted(secrets, key=len, reverse=True) + + +def _redact(value: str, secrets: Sequence[str]) -> str: + for secret in secrets: + value = value.replace(secret, REDACTED) + return value + + +def _cap(value: str) -> str: + """The longest whole-character prefix of at most MAX_CAPTURED_BYTES of UTF-8.""" + return value.encode("utf-8", "replace")[:MAX_CAPTURED_BYTES].decode("utf-8", "ignore") + + +def _captured_text(value: Any, secrets: Sequence[str]) -> str: + return _cap(_redact(value if isinstance(value, str) else str(value), secrets)) + + +def _captured_list(values: Any, secrets: Sequence[str]) -> Optional[List[str]]: + if isinstance(values, str): + values = [values] + if not isinstance(values, (list, tuple)): + return None + return [_captured_text(value, secrets) for value in values[:MAX_CAPTURED_ITEMS]] + + +def _count(values: Any) -> Optional[int]: + """Number of texts or documents; a bare string is one. None when unknown.""" + if isinstance(values, str): + return 1 + if isinstance(values, (list, tuple)): + return len(values) + return None + + +def _model(operation: Operation, args: Sequence[Any], kwargs: Mapping[str, Any]) -> Optional[str]: + model = _argument(operation, args, kwargs, "model") + if model is None and operation is EMBED: + # Client.embed falls back to this default (and warns) when model is None. + model = getattr(_voyage_module(), "VOYAGE_EMBED_DEFAULT_MODEL", None) + return model if isinstance(model, str) and model else None + + +_UNRESOLVED = object() +_local_model_check: Any = _UNRESOLVED + + +def _is_local_model(model: Optional[str]) -> bool: + """voyageai >= 0.5 runs some embedding models in process, with no request.""" + global _local_model_check + if not model: + return False + if _local_model_check is _UNRESOLVED: + # Resolved once: a failed import is not cached by Python, and older + # releases would otherwise search sys.path on every call. + try: + from voyageai.local.helpers import is_local_model + except Exception: # voyageai < 0.5 has no local models + is_local_model = None + _local_model_check = is_local_model + if _local_model_check is None: + return False + try: + return bool(_local_model_check(model)) + except Exception: + return False + + +def _server_address(instance: Any) -> Optional[str]: + """The host the client chose (from its key or the caller's base_url), never a URL.""" + params = getattr(instance, "_params", None) + url = params.get("base_url") if isinstance(params, Mapping) else None + if not isinstance(url, str) or not url: + return None + try: + return urlsplit(url).hostname or None + except ValueError: + return None + + +def _base_attributes(operation: Operation) -> Dict[str, Any]: + return { + SpanAttributes.GEN_AI_SPAN_KIND: operation.span_kind, + SpanAttributes.GEN_AI_PROVIDER_NAME: PROVIDER, + SpanAttributes.GEN_AI_OPERATION_NAME: operation.operation_name, + } + + +def _request_attributes( + operation: Operation, instance: Any, args: Sequence[Any], kwargs: Mapping[str, Any] +) -> Dict[str, Any]: + """Attributes known before the call. Never content.""" + attributes: Dict[str, Any] = {} + model = _model(operation, args, kwargs) + if model: + attributes[SpanAttributes.GEN_AI_REQUEST_MODEL] = model + if operation is EMBED: + if model: + attributes[SpanAttributes.EMBEDDING_MODEL_NAME] = model + count = _count(_argument(operation, args, kwargs, "texts")) + if count is not None: + attributes[EMBEDDING_COUNT] = count + for name, key in (("input_type", INPUT_TYPE), ("output_dtype", OUTPUT_DTYPE)): + value = _argument(operation, args, kwargs, name) + if isinstance(value, str) and value: + attributes[key] = value + dimension = _argument(operation, args, kwargs, "output_dimension") + if isinstance(dimension, int) and not isinstance(dimension, bool): + attributes[OUTPUT_DIMENSION] = dimension + else: + if model: + attributes[RerankerAttributes.RERANKER_MODEL_NAME] = model + count = _count(_argument(operation, args, kwargs, "documents")) + if count is not None: + attributes[RERANK_DOCUMENT_COUNT] = count + top_k = _argument(operation, args, kwargs, "top_k") + if isinstance(top_k, int) and not isinstance(top_k, bool): + attributes[RerankerAttributes.RERANKER_TOP_K] = top_k + if not (operation is EMBED and _is_local_model(model)): + host = _server_address(instance) + if host: + attributes[SERVER_ADDRESS] = host + return attributes + + +def _hides_inputs(config: TraceConfig) -> bool: + return bool(config.hide_inputs or config.hide_input_text) + + +def _input_content( + operation: Operation, + args: Sequence[Any], + kwargs: Mapping[str, Any], + secrets: Sequence[str], + config: TraceConfig, +) -> Dict[str, Any]: + """Texts, query and documents, only with capture on and inputs not hidden.""" + if _hides_inputs(config): + return {} + attributes: Dict[str, Any] = {} + if operation is EMBED: + texts = _captured_list(_argument(operation, args, kwargs, "texts"), secrets) + if texts is not None: + attributes[SpanAttributes.INPUT_VALUE] = json.dumps(texts, ensure_ascii=False) + attributes[SpanAttributes.INPUT_MIME_TYPE] = JSON_MIME_TYPE + return attributes + query = _argument(operation, args, kwargs, "query") + documents = _captured_list(_argument(operation, args, kwargs, "documents"), secrets) + if query is not None: + attributes[RerankerAttributes.RERANKER_QUERY] = _captured_text(query, secrets) + payload: Dict[str, Any] = {} + if query is not None: + payload["query"] = attributes[RerankerAttributes.RERANKER_QUERY] + if documents is not None: + payload["documents"] = documents + if payload: + attributes[SpanAttributes.INPUT_VALUE] = json.dumps(payload, ensure_ascii=False) + attributes[SpanAttributes.INPUT_MIME_TYPE] = JSON_MIME_TYPE + return attributes + + +def _usage(result: Any) -> Dict[str, Any]: + """Voyage reports one total; every embedding or rerank token is input. + + Written on the promoted keys the collector reads (gen_ai.usage.*), once, + on this span only. Never estimated. + """ + total = getattr(result, "total_tokens", None) + if not isinstance(total, int) or isinstance(total, bool) or total < 0: + return {} + return { + SpanAttributes.GEN_AI_USAGE_INPUT_TOKENS: total, + SpanAttributes.GEN_AI_USAGE_TOTAL_TOKENS: total, + } + + +def _response_attributes( + operation: Operation, result: Any, args: Sequence[Any], kwargs: Mapping[str, Any] +) -> Dict[str, Any]: + """Counts, dimension and usage from the returned object. Never vectors.""" + attributes = _usage(result) + if operation is EMBED: + embeddings = getattr(result, "embeddings", None) + if isinstance(embeddings, (list, tuple)) and embeddings: + first = embeddings[0] + if hasattr(first, "__len__") and not isinstance(first, (str, bytes)): + dimension = len(first) + if _argument(operation, args, kwargs, "output_dtype") in _PACKED_DTYPES: + dimension *= 8 + attributes[EMBEDDING_DIMENSION] = dimension + attributes[SpanAttributes.GEN_AI_EMBEDDINGS_DIMENSION_COUNT] = dimension + else: + results = getattr(result, "results", None) + if isinstance(results, (list, tuple)): + attributes[RERANK_RESULT_COUNT] = len(results) + return attributes + + +def _output_content(operation: Operation, result: Any, config: TraceConfig) -> Dict[str, Any]: + """Rerank scores in the client's order, only with capture on and outputs not hidden.""" + if operation is not RERANK or config.hide_outputs: + return {} + results = getattr(result, "results", None) + if not isinstance(results, (list, tuple)): + return {} + scores = [ + { + "index": getattr(item, "index", None), + "relevance_score": getattr(item, "relevance_score", None), + } + for item in results + ] + return { + SpanAttributes.OUTPUT_VALUE: json.dumps(scores), + SpanAttributes.OUTPUT_MIME_TYPE: JSON_MIME_TYPE, + } + + +def _set_attributes(span: Span, attributes: Mapping[str, Any]) -> None: + for key, value in attributes.items(): + try: + span.set_attribute(key, value) + except Exception: + logger.debug("Could not set %s on the Voyage span", key, exc_info=True) + + +def _describe(error: BaseException, secrets: Sequence[str]) -> Tuple[str, str]: + try: + message = str(error) + except Exception: + message = "".format(type(error).__name__) + return type(error).__name__, _redact(message, secrets) + + +def _record_error(span: Span, error: BaseException, secrets: Sequence[str]) -> None: + name, message = _describe(error, secrets) + try: + stacktrace = "".join(traceback.format_exception(type(error), error, error.__traceback__)) + except Exception: + stacktrace = "" + # The SDK's record_exception would copy str(error) verbatim; the key may + # be in it (a server echoing it back), so the event is built here. + span.add_event( + "exception", + { + "exception.type": "{0}.{1}".format(type(error).__module__, type(error).__qualname__), + "exception.message": message, + "exception.stacktrace": _redact(stacktrace, secrets), + }, + ) + span.set_status(Status(StatusCode.ERROR, "{0}: {1}".format(name, message))) + + +def _end(span: Span) -> None: + try: + span.end() + except Exception: + logger.debug("Could not end the Voyage span", exc_info=True) + + +class _Call: + """The tracing state for one wrapped call. Every step swallows its own errors.""" + + __slots__ = ("_wrapper", "_args", "_kwargs", "_span", "_secrets", "_token") + + def __init__( + self, wrapper: "_BaseWrapper", instance: Any, args: Sequence[Any], kwargs: Mapping[str, Any] + ) -> None: + self._wrapper = wrapper + self._args = args + self._kwargs = kwargs + self._span: Optional[Span] = None + self._secrets: List[str] = [] + self._token: Any = None + operation = wrapper.operation + attributes = _base_attributes(operation) + try: + self._secrets = _secrets(instance) + except Exception: + logger.debug("Could not read the Voyage client's key", exc_info=True) + try: + attributes.update(_request_attributes(operation, instance, args, kwargs)) + except Exception: + logger.debug("Could not read Voyage request attributes", exc_info=True) + if wrapper.capture_content: + try: + attributes.update( + _input_content(operation, args, kwargs, self._secrets, wrapper.config) + ) + except Exception: + logger.debug("Could not capture Voyage request content", exc_info=True) + try: + self._span = wrapper.tracer.start_span(operation.span_name, attributes=attributes) + except Exception: + logger.debug("Could not start the Voyage span", exc_info=True) + self._span = None + + def enter(self) -> None: + """Make the span current so HTTP spans started by the SDK nest under it.""" + if self._span is None: + return + try: + self._token = context_api.attach(trace_api.set_span_in_context(self._span)) + except Exception: + logger.debug("Could not make the Voyage span current", exc_info=True) + self._token = None + + def exit(self) -> None: + if self._token is None: + return + try: + context_api.detach(self._token) + except Exception: + logger.debug("Could not restore the trace context", exc_info=True) + self._token = None + + def ok(self, result: Any) -> None: + span = self._span + if span is None: + return + try: + operation = self._wrapper.operation + try: + _set_attributes( + span, _response_attributes(operation, result, self._args, self._kwargs) + ) + except Exception: + logger.debug("Could not read Voyage response attributes", exc_info=True) + if self._wrapper.capture_content: + try: + _set_attributes(span, _output_content(operation, result, self._wrapper.config)) + except Exception: + logger.debug("Could not capture Voyage response content", exc_info=True) + span.set_status(Status(StatusCode.OK)) + except Exception: + logger.debug("Could not finish the Voyage span", exc_info=True) + finally: + _end(span) + + def error(self, error: BaseException) -> None: + span = self._span + if span is None: + return + try: + _record_error(span, error, self._secrets) + except Exception: + logger.debug("Could not record the Voyage error", exc_info=True) + finally: + _end(span) + + def cancelled(self) -> None: + # Cancellation is not an exception: no event, no result attributes. + span = self._span + if span is None: + return + try: + span.set_attribute(CANCELLED, True) + span.set_status(Status(StatusCode.ERROR, "cancelled")) + except Exception: + logger.debug("Could not mark the Voyage span cancelled", exc_info=True) + finally: + _end(span) + + +class _BaseWrapper: + def __init__( + self, tracer: Tracer, operation: Operation, config: TraceConfig, capture_content: bool + ) -> None: + self.tracer = tracer + self.operation = operation + self.config = config + self.capture_content = capture_content + + def _begin( + self, instance: Any, args: Sequence[Any], kwargs: Mapping[str, Any] + ) -> Optional[_Call]: + try: + return _Call(self, instance, args, kwargs) + except Exception: + logger.debug("Could not trace the Voyage call", exc_info=True) + return None + + +class SyncWrapper(_BaseWrapper): + """Trace ``Client.embed`` / ``Client.rerank``.""" + + def __call__( + self, + wrapped: Callable[..., Any], + instance: Any, + args: Tuple[Any, ...], + kwargs: Dict[str, Any], + ) -> Any: + call = self._begin(instance, args, kwargs) + if call is None: + return wrapped(*args, **kwargs) + call.enter() + try: + result = wrapped(*args, **kwargs) + except _CANCELLATIONS: + call.exit() + call.cancelled() + raise + except BaseException as error: + call.exit() + call.error(error) + raise + call.exit() + call.ok(result) + return result + + +class AsyncWrapper(_BaseWrapper): + """Trace ``AsyncClient.embed`` / ``AsyncClient.rerank``.""" + + async def __call__( + self, + wrapped: Callable[..., Any], + instance: Any, + args: Tuple[Any, ...], + kwargs: Dict[str, Any], + ) -> Any: + call = self._begin(instance, args, kwargs) + if call is None: + return await wrapped(*args, **kwargs) + call.enter() + try: + result = await wrapped(*args, **kwargs) + except _CANCELLATIONS: + call.exit() + call.cancelled() + raise + except BaseException as error: + call.exit() + call.error(error) + raise + call.exit() + call.ok(result) + return result diff --git a/python/frameworks/voyage/traceai_voyage/package.py b/python/frameworks/voyage/traceai_voyage/package.py new file mode 100644 index 000000000..9eb434332 --- /dev/null +++ b/python/frameworks/voyage/traceai_voyage/package.py @@ -0,0 +1 @@ +_instruments = ("voyageai >= 0.3.7, < 1",) diff --git a/python/frameworks/voyage/traceai_voyage/py.typed b/python/frameworks/voyage/traceai_voyage/py.typed new file mode 100644 index 000000000..e69de29bb diff --git a/python/frameworks/voyage/traceai_voyage/version.py b/python/frameworks/voyage/traceai_voyage/version.py new file mode 100644 index 000000000..3dc1f76bc --- /dev/null +++ b/python/frameworks/voyage/traceai_voyage/version.py @@ -0,0 +1 @@ +__version__ = "0.1.0" From 7ead20812eaf63f5d371ad9f36f21472f844102f Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Mon, 5 Oct 2026 01:26:22 -0700 Subject: [PATCH 07/17] docs(voyage): README with attributes, privacy, hosts and limits Documents the wrapped methods and span names, every attribute, the content-capture opt-in and TraceConfig flags, key redaction, the client's own host choice (the instrumentor never sets base_url), errors and cancellation, and what is not traced (multimodal_embed, contextualized_embed, deprecated get_embedding(s), TypeScript). approved: Nikhil 2026-10-03 blanket Refs: TH-8321 --- python/frameworks/voyage/README.md | 165 +++++++++++++++++++++++++++++ 1 file changed, 165 insertions(+) create mode 100644 python/frameworks/voyage/README.md diff --git a/python/frameworks/voyage/README.md b/python/frameworks/voyage/README.md new file mode 100644 index 000000000..4942e66e7 --- /dev/null +++ b/python/frameworks/voyage/README.md @@ -0,0 +1,165 @@ +# traceAI-voyage + +OpenTelemetry instrumentation for the official +[Voyage AI Python client](https://github.com/voyage-ai/voyageai-python) +(`voyageai`). + +`traceAI-openai` does not trace Voyage: it wraps the OpenAI client only and +never sees `voyageai.Client`. Use this package for Voyage embeddings and +reranking. + +## Installation + +```bash +pip install traceAI-voyage +``` + +It accepts `voyageai>=0.3.7,<1` (0.3.7 is the first release that routes Atlas +keys) and `fi-instrumentation-otel>=1.1.0`. The suite runs on Python 3.10, +3.11 and 3.13 against `voyageai` 0.3.7 and 0.5.0. + +## Usage + +```python +import voyageai +from fi_instrumentation import register +from fi_instrumentation.fi_types import ProjectType +from traceai_voyage import VoyageInstrumentor + +trace_provider = register(project_type=ProjectType.OBSERVE, project_name="my-rag-app") +VoyageInstrumentor().instrument(tracer_provider=trace_provider) + +client = voyageai.Client() # reads VOYAGE_API_KEY +client.embed(["What is the capital of France?"], model="voyage-3.5", input_type="query") +client.rerank("capital of France", ["Paris is...", "Berlin is..."], model="rerank-2.5", top_k=1) +``` + +`examples/embed_and_rerank.py` is a runnable version of this. + +The methods are patched on the client classes, so a client built before +`instrument()` is traced too; only calls made while instrumented are traced. +`uninstrument()` restores the original methods. + +## Hosts and `base_url` + +The instrumentor never sets or changes `base_url`. The Voyage client picks the +host from the API key, and a `base_url` you pass to the client yourself wins. +In `voyageai` 0.3.7 to 0.5.0, keys that start with `al-` (created in MongoDB +Atlas) go to `https://ai.mongodb.com/v1` and other keys (created on the Voyage +platform) to `https://api.voyageai.com/v1`. MongoDB also documents geography +hosts such as `eu.ai.mongodb.com` and `us.ai.mongodb.com`; to use one, pass it +as `base_url` when you construct the client. + +The span records only the host the client is using, as `server.address`. A +key sent to the wrong host fails in the client; the span records that error +and nothing retries against another host. + +## What is traced + +| Call | Span name | `gen_ai.span.kind` | +|---|---|---| +| `Client.embed`, `AsyncClient.embed` | `voyage.embed` | `EMBEDDING` | +| `Client.rerank`, `AsyncClient.rerank` | `voyage.rerank` | `RERANKER` | + +One span per call: a list of texts is one span, and the client's own retries +(`max_retries`) happen inside that one span. The span is a child of the active +span when there is one, and it is current while the SDK sends the request, so +a span from an HTTP client instrumentation nests under it. Local models that +`embed` runs in process (`voyageai` 0.5+, for example `voyage-4-nano`) are +traced the same way, without `server.address`; their token count is the one the +client computes locally. + +Not traced (no span, not an error): `multimodal_embed`, +`contextualized_embed`, the deprecated module functions +`voyageai.get_embedding(s)` / `aget_embedding(s)`, and the TypeScript client. + +## Span attributes + +Always recorded when known (never content): + +| Attribute | Calls | Value | +|---|---|---| +| `gen_ai.provider.name` | all | `voyage` | +| `gen_ai.operation.name` | all | `embeddings` or `rerank` | +| `gen_ai.request.model` | all | The model; for `embed` without `model`, the SDK default it falls back to. | +| `embedding.model_name` / `reranker.model_name` | embed / rerank | Same model. | +| `server.address` | all | Host from the client's base URL (no path, query or credentials). | +| `voyage.embedding.count` | embed | Number of texts in the call (a bare string is 1). | +| `voyage.input_type`, `voyage.output_dtype`, `voyage.output_dimension` | embed | As passed; absent when not passed. | +| `voyage.embedding.dimension`, `gen_ai.embeddings.dimension.count` | embed, success | Length of the first returned vector; times 8 for the packed `binary` / `ubinary` dtypes. | +| `voyage.rerank.document_count` | rerank | Number of documents sent. | +| `reranker.top_k` | rerank | As passed; absent when not passed. | +| `voyage.rerank.result_count` | rerank, success | Number of results returned. | +| `gen_ai.usage.input_tokens`, `gen_ai.usage.total_tokens` | success | The client's `total_tokens`. Voyage reports one total and every embedding or rerank token is input, so both keys carry it. Never estimated. No cost is set. | +| `voyage.cancelled` | cancelled calls | `true`. | + +Usage is written only on the Voyage span, once, on the `gen_ai.usage.*` keys +the Future AGI collector reads. If another instrumentation in your app (for +example a framework that wraps the same embedding call) also records usage for +it, the trace counts those tokens twice. + +## Privacy and content capture + +By default a span carries no content: no texts, no rerank query or documents, +no relevance scores, no vectors. To record content, opt in: + +```python +from fi_instrumentation import TraceConfig + +VoyageInstrumentor().instrument( + tracer_provider=trace_provider, + capture_content=True, + config=TraceConfig(), # optional; honours hide_inputs / hide_outputs / pii_redaction +) +``` + +With `capture_content=True`: + +| Attribute | Calls | Value | Dropped by | +|---|---|---|---| +| `input.value` (`input.mime_type` = `application/json`) | embed | JSON list of the texts. | `hide_inputs`, `hide_input_text` | +| `reranker.query` | rerank | The query. | `hide_inputs`, `hide_input_text` | +| `input.value` (`application/json`) | rerank | JSON `{"query": ..., "documents": [...]}`. | `hide_inputs`, `hide_input_text` | +| `output.value` (`application/json`) | rerank | JSON list of `{"index", "relevance_score"}` in the client's order. | `hide_outputs` | + +Captured content holds at most 64 texts or documents, each cut to 2 KB of +UTF-8 on a character boundary; the counts above stay exact. `TraceConfig` +reads `FI_HIDE_INPUTS` / `FI_HIDE_OUTPUTS` from the environment, and +`pii_redaction=True` applies to every recorded value. + +Embedding vectors are never recorded, with or without capture, and +`hide_embedding_vectors` has nothing to hide here. + +The Voyage API key is never recorded. The instrumentor does not read it from +the environment itself; it takes the key the client holds (`client.api_key`, +`client._params["api_key"]`) and a module-level `voyageai.api_key`, and +replaces any occurrence in captured content, error messages and stack traces +with `[redacted]`. + +## Errors and cancellation + +An error raised by the client (for example `voyageai.error.AuthenticationError` +for HTTP 401, or `voyageai.error.Timeout`) sets the span status to ERROR with +`: ` and records one `exception` event; the key is +redacted from both. The exception reaches your code unchanged. + +Cancelling an `AsyncClient` call (`asyncio.CancelledError`) or interrupting a +blocking call (`KeyboardInterrupt`) ends the span with status ERROR, +description `cancelled` and `voyage.cancelled` = `true`, with no exception +event. Result attributes and usage are absent on error and cancellation, never +written as 0. + +The instrumentation never changes what a call returns or raises: a failure +while building attributes or starting the span is logged at debug level and the +Voyage call runs as if uninstrumented. + +## Configuration errors + +`instrument(config=...)` must be a `fi_instrumentation.TraceConfig` and +`capture_content` a `bool`; anything else raises `TypeError` and wraps +nothing. On an installed `voyageai` outside `>=0.3.7,<1`, `instrument()` logs +an error and wraps nothing (OpenTelemetry's dependency check). + +## License + +Apache-2.0. From 5c01dc0dcf8780768aca525434918c82c06e6754 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Mon, 5 Oct 2026 01:26:22 -0700 Subject: [PATCH 08/17] docs: list traceAI-voyage in the README package tables approved: Nikhil 2026-10-03 blanket Refs: TH-8321 --- README.md | 1 + python/README.md | 1 + 2 files changed, 2 insertions(+) diff --git a/README.md b/README.md index 214680868..2b670adb1 100644 --- a/README.md +++ b/README.md @@ -245,6 +245,7 @@ var tracer = TraceAI.Register(opts => | [`traceAI-huggingface`](https://pypi.org/project/traceAI-huggingface/) | HuggingFace | [![PyPI](https://img.shields.io/pypi/v/traceAI-huggingface)](https://pypi.org/project/traceAI-huggingface/) | | [`traceAI-xai`](https://pypi.org/project/traceAI-xai/) | xAI (Grok) | [![PyPI](https://img.shields.io/pypi/v/traceAI-xai)](https://pypi.org/project/traceAI-xai/) | | [`traceAI-vllm`](https://pypi.org/project/traceAI-vllm/) | vLLM | [![PyPI](https://img.shields.io/pypi/v/traceAI-vllm)](https://pypi.org/project/traceAI-vllm/) | +| [`traceAI-voyage`](https://pypi.org/project/traceAI-voyage/) | Voyage AI (embeddings, rerank) | [![PyPI](https://img.shields.io/pypi/v/traceAI-voyage)](https://pypi.org/project/traceAI-voyage/) | #### Agent Frameworks diff --git a/python/README.md b/python/README.md index 2d5263467..6407668cf 100644 --- a/python/README.md +++ b/python/README.md @@ -165,6 +165,7 @@ trace_provider = register( | `traceai-bedrock` | AWS Bedrock | | `traceai-litellm` | LiteLLM | | `traceai-portkey` | Portkey | +| `traceai-voyage` | Voyage AI (embeddings, rerank) | ### Agent Frameworks From b2140ac7ec80d74f560874fc1245da3864dff1f0 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Mon, 5 Oct 2026 02:35:34 -0700 Subject: [PATCH 09/17] build(voyage): declare Python 3.12; the suite runs on 3.10 to 3.13 Review R1. The suite passes on Python 3.10, 3.11, 3.12 and 3.13 with voyageai 0.3.7 and 0.5.0, so the classifiers, the README and the packaging test now list exactly those four versions. approved: Nikhil 2026-10-03 blanket Refs: TH-8321 --- python/frameworks/voyage/README.md | 2 +- python/frameworks/voyage/pyproject.toml | 1 + python/frameworks/voyage/tests/test_packaging.py | 3 ++- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/python/frameworks/voyage/README.md b/python/frameworks/voyage/README.md index 4942e66e7..dcf691d71 100644 --- a/python/frameworks/voyage/README.md +++ b/python/frameworks/voyage/README.md @@ -16,7 +16,7 @@ pip install traceAI-voyage It accepts `voyageai>=0.3.7,<1` (0.3.7 is the first release that routes Atlas keys) and `fi-instrumentation-otel>=1.1.0`. The suite runs on Python 3.10, -3.11 and 3.13 against `voyageai` 0.3.7 and 0.5.0. +3.11, 3.12 and 3.13 against `voyageai` 0.3.7 and 0.5.0. ## Usage diff --git a/python/frameworks/voyage/pyproject.toml b/python/frameworks/voyage/pyproject.toml index 65b6b3d25..581f50c0d 100644 --- a/python/frameworks/voyage/pyproject.toml +++ b/python/frameworks/voyage/pyproject.toml @@ -15,6 +15,7 @@ classifiers = [ "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", "Topic :: Software Development :: Libraries :: Python Modules", ] diff --git a/python/frameworks/voyage/tests/test_packaging.py b/python/frameworks/voyage/tests/test_packaging.py index e148fe036..129587f7c 100644 --- a/python/frameworks/voyage/tests/test_packaging.py +++ b/python/frameworks/voyage/tests/test_packaging.py @@ -73,7 +73,8 @@ def test_fi_instrumentation_floor_is_1_1_0(): def test_classifiers_list_only_the_tested_python_versions(): text = PYPROJECT.read_text() versions = re.findall(r'"Programming Language :: Python :: (3\.\d+)"', text) - assert versions == ["3.10", "3.11", "3.13"] + # Every version the suite runs on (3.10 to 3.13), and nothing else. + assert versions == ["3.10", "3.11", "3.12", "3.13"] assert _dependency("python") == SpecifierSet(">=3.10,<3.14") From 72c6eafa23d7f8d6a10af24c2e5f54ff17972a55 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Mon, 5 Oct 2026 02:42:59 -0700 Subject: [PATCH 10/17] feat(voyage): record the rerank query and scores by default (PRD J2/AC-03) Review R3. The RERANKER span now carries the query and the relevance scores by default, as PRD J2 step 2 and AC-03 require, so the default is no longer a deviation from the spec: - reranker.query and a plain-text input.value (text/plain) hold the query, key-redacted and cut to 2 KB on a character boundary. - output.value holds [{index, relevance_score}] in the client's order. - Embed texts and rerank documents stay off by default and are recorded only with instrument(capture_content=True); then rerank input.value is the JSON {query, documents} as before. - hide_inputs / hide_input_text / FI_HIDE_INPUTS drop the query (and the texts and documents when captured); hide_outputs / FI_HIDE_OUTPUTS drop the scores. Hiding inputs keeps the scores (PRD J2). Vectors are never recorded. Tests: default-config tests for query and scores present, embed texts and rerank documents absent, query key-redaction and cap, and the hide_inputs / hide_input_text / hide_outputs / FI_HIDE_* combinations with and without capture. Existing default-absence tests in test_rerank, test_async, test_contract_harness and test_example now assert the query and scores are present and texts, documents, vectors and the key absent. The capture_content docstring and the README content table state what is on by default. approved: Nikhil 2026-10-03 blanket Refs: TH-8321 --- python/frameworks/voyage/README.md | 43 ++++--- python/frameworks/voyage/tests/_support.py | 9 +- python/frameworks/voyage/tests/test_async.py | 15 ++- .../voyage/tests/test_content_capture.py | 109 +++++++++++++++++- .../voyage/tests/test_contract_harness.py | 11 +- .../frameworks/voyage/tests/test_example.py | 12 +- python/frameworks/voyage/tests/test_rerank.py | 20 ++-- .../voyage/traceai_voyage/__init__.py | 16 ++- .../voyage/traceai_voyage/_wrappers.py | 58 +++++++--- 9 files changed, 231 insertions(+), 62 deletions(-) diff --git a/python/frameworks/voyage/README.md b/python/frameworks/voyage/README.md index dcf691d71..abc32c347 100644 --- a/python/frameworks/voyage/README.md +++ b/python/frameworks/voyage/README.md @@ -100,8 +100,27 @@ it, the trace counts those tokens twice. ## Privacy and content capture -By default a span carries no content: no texts, no rerank query or documents, -no relevance scores, no vectors. To record content, opt in: +By default the rerank span records the query and the relevance scores, as PRD +J2 / AC-03 require, so this default is not a deviation from the spec. Embed +texts and rerank documents are recorded only with `capture_content=True`. +Embedding vectors are never recorded. + +| Attribute | Calls | Value | Recorded | Dropped by | +|---|---|---|---|---| +| `reranker.query` | rerank | The query. | By default | `hide_inputs`, `hide_input_text` | +| `input.value` (`input.mime_type` = `text/plain`) | rerank | The query. | By default | `hide_inputs`, `hide_input_text` | +| `output.value` (`output.mime_type` = `application/json`) | rerank | JSON list of `{"index", "relevance_score"}` in the client's order. | By default | `hide_outputs` | +| `input.value` (`application/json`) | rerank | JSON `{"query": ..., "documents": [...]}`, in place of the plain query. | Only with `capture_content=True` | `hide_inputs`, `hide_input_text` | +| `input.value` (`application/json`) | embed | JSON list of the texts. | Only with `capture_content=True` | `hide_inputs`, `hide_input_text` | + +Hiding inputs keeps the scores (they are numbers, not input text), and hiding +outputs keeps the query. The counts (`voyage.embedding.count`, +`voyage.rerank.document_count`, `voyage.rerank.result_count`) stay whatever is +hidden. `TraceConfig` reads `FI_HIDE_INPUTS` / `FI_HIDE_OUTPUTS` from the +environment, so `FI_HIDE_INPUTS=true FI_HIDE_OUTPUTS=true` records no content +at all. + +To record embed texts and rerank documents too, opt in: ```python from fi_instrumentation import TraceConfig @@ -113,19 +132,9 @@ VoyageInstrumentor().instrument( ) ``` -With `capture_content=True`: - -| Attribute | Calls | Value | Dropped by | -|---|---|---|---| -| `input.value` (`input.mime_type` = `application/json`) | embed | JSON list of the texts. | `hide_inputs`, `hide_input_text` | -| `reranker.query` | rerank | The query. | `hide_inputs`, `hide_input_text` | -| `input.value` (`application/json`) | rerank | JSON `{"query": ..., "documents": [...]}`. | `hide_inputs`, `hide_input_text` | -| `output.value` (`application/json`) | rerank | JSON list of `{"index", "relevance_score"}` in the client's order. | `hide_outputs` | - -Captured content holds at most 64 texts or documents, each cut to 2 KB of -UTF-8 on a character boundary; the counts above stay exact. `TraceConfig` -reads `FI_HIDE_INPUTS` / `FI_HIDE_OUTPUTS` from the environment, and -`pii_redaction=True` applies to every recorded value. +Captured content holds at most 64 texts or documents, and the query and each +text or document are cut to 2 KB of UTF-8 on a character boundary; the counts +stay exact. `pii_redaction=True` applies to every recorded value. Embedding vectors are never recorded, with or without capture, and `hide_embedding_vectors` has nothing to hide here. @@ -133,8 +142,8 @@ Embedding vectors are never recorded, with or without capture, and The Voyage API key is never recorded. The instrumentor does not read it from the environment itself; it takes the key the client holds (`client.api_key`, `client._params["api_key"]`) and a module-level `voyageai.api_key`, and -replaces any occurrence in captured content, error messages and stack traces -with `[redacted]`. +replaces any occurrence in the recorded query, captured content, error +messages and stack traces with `[redacted]`. ## Errors and cancellation diff --git a/python/frameworks/voyage/tests/_support.py b/python/frameworks/voyage/tests/_support.py index ad6bded23..f03266878 100644 --- a/python/frameworks/voyage/tests/_support.py +++ b/python/frameworks/voyage/tests/_support.py @@ -36,16 +36,19 @@ EMBED_MODEL = "voyage-3.5" RERANK_MODEL = "rerank-2.5" -# Content the tests send or the fake returns. None of it may reach a span -# unless content capture is switched on. +# Content the tests send or the fake returns. By default only the rerank +# query (and the scores) may reach a span; texts and documents need +# capture_content=True, and the TraceConfig hide flags drop each kind. TEXTS = ["TEXT-ALPHA-MUST-NOT-BE-EXPORTED", "TEXT-BETA-MUST-NOT-BE-EXPORTED"] -QUERY = "QUERY-MUST-NOT-BE-EXPORTED" +QUERY = "QUERY-RECORDED-UNLESS-INPUTS-ARE-HIDDEN" DOCUMENTS = [ "DOC-ONE-MUST-NOT-BE-EXPORTED", "DOC-TWO-MUST-NOT-BE-EXPORTED", "DOC-THREE-MUST-NOT-BE-EXPORTED", ] CONTENT_MARKERS = tuple(TEXTS) + (QUERY,) + tuple(DOCUMENTS) +# Recorded only with capture_content=True. +OPT_IN_MARKERS = tuple(TEXTS) + tuple(DOCUMENTS) # Vectors: every component is VECTOR_BASE + k * 2**-9, exact in float32, so # the decoded values are predictable and searchable on the wire. diff --git a/python/frameworks/voyage/tests/test_async.py b/python/frameworks/voyage/tests/test_async.py index 84d712963..3964a9b24 100644 --- a/python/frameworks/voyage/tests/test_async.py +++ b/python/frameworks/voyage/tests/test_async.py @@ -12,9 +12,9 @@ import voyageai # noqa: E402 from _support import ( # noqa: E402 - CONTENT_MARKERS, DOCUMENTS, EMBED_MODEL, + OPT_IN_MARKERS, QUERY, RERANK_MODEL, SCORE_MARKERS, @@ -69,7 +69,7 @@ def test_async_rerank_matches_the_sync_span(fake): assert status_code(async_span) == "OK" -def test_async_spans_carry_no_content_by_default(fake): +def test_async_spans_carry_the_same_default_content_as_sync(fake): async def calls() -> None: voyage = async_client(fake) await voyage.embed(TEXTS, model=EMBED_MODEL) @@ -78,10 +78,17 @@ async def calls() -> None: with instrumented() as traced: _run(calls) - assert [span.name for span in traced.spans()] == ["voyage.embed", "voyage.rerank"] + embed, rerank = traced.spans() + assert [embed.name, rerank.name] == ["voyage.embed", "voyage.rerank"] + # Query and scores by default (PRD J2 / AC-03); texts, documents, + # vectors and the key never without capture_content. + assert attrs(rerank)["reranker.query"] == QUERY + assert "output.value" in attrs(rerank) + assert "input.value" not in attrs(embed) wire = traced.wire() - for marker in CONTENT_MARKERS + SCORE_MARKERS + (VECTOR_MARKER, VOYAGE_KEY): + for marker in OPT_IN_MARKERS + (VECTOR_MARKER, VOYAGE_KEY): assert marker not in wire, marker + assert SCORE_MARKERS[1] in wire def test_async_errors_match_the_sync_error_span(fake): diff --git a/python/frameworks/voyage/tests/test_content_capture.py b/python/frameworks/voyage/tests/test_content_capture.py index 8ae3bc46b..56a93f2ab 100644 --- a/python/frameworks/voyage/tests/test_content_capture.py +++ b/python/frameworks/voyage/tests/test_content_capture.py @@ -1,4 +1,5 @@ -"""Content capture: off by default; opt-in honours TraceConfig and the key redaction.""" +"""Content: the rerank query and scores by default (PRD J2 / AC-03); texts and +documents only with capture_content=True; TraceConfig and key redaction apply.""" from __future__ import annotations @@ -15,6 +16,7 @@ from _support import ( # noqa: E402 DOCUMENTS, EMBED_MODEL, + OPT_IN_MARKERS, QUERY, RERANK_MODEL, SCORE_MARKERS, @@ -29,6 +31,11 @@ ) OTHER_KEY = "al-placeholder-module-key-must-not-be-exported" +# The fake returns documents 1 and 0 for top_k=2, in that order. +TOP_2_SCORES = [ + {"index": 1, "relevance_score": SCORES[1]}, + {"index": 0, "relevance_score": SCORES[0]}, +] @pytest.fixture() @@ -46,6 +53,106 @@ def _embed_and_rerank(fake, **options): return traced, attrs(embed), attrs(rerank) +def test_default_config_records_the_rerank_query_and_scores(fake): + _traced, _embed, rerank = _embed_and_rerank(fake) + + # PRD J2 step 2 / AC-03: the RERANKER span carries the query and scores. + assert rerank["reranker.query"] == QUERY + assert rerank["input.value"] == QUERY + assert rerank["input.mime_type"] == "text/plain" + assert json.loads(rerank["output.value"]) == TOP_2_SCORES + assert rerank["output.mime_type"] == "application/json" + + +def test_default_config_records_no_embed_texts_rerank_documents_or_vectors(fake): + traced, embed, rerank = _embed_and_rerank(fake) + + for key in ("input.value", "input.mime_type", "output.value", "output.mime_type"): + assert key not in embed, key + for key in ("reranker.input_documents", "reranker.output_documents"): + assert not any(name.startswith(key) for name in rerank), key + wire = traced.wire() + for marker in OPT_IN_MARKERS + (VECTOR_MARKER, VOYAGE_KEY): + assert marker not in wire, marker + # The counts stay exact. + assert embed["voyage.embedding.count"] == 2 + assert rerank["voyage.rerank.document_count"] == 3 + + +def test_the_default_query_is_key_redacted_and_capped(fake, monkeypatch): + monkeypatch.setattr(voyageai, "api_key", OTHER_KEY) + with instrumented() as traced: + voyage = client(fake) + voyage.rerank("q {0} {1}".format(VOYAGE_KEY, OTHER_KEY), DOCUMENTS, model=RERANK_MODEL) + voyage.rerank("é" * 3000, DOCUMENTS, model=RERANK_MODEL) # 6000 UTF-8 bytes + + keyed, long = (attrs(span) for span in traced.spans()) + assert keyed["reranker.query"] == keyed["input.value"] == "q [redacted] [redacted]" + # Cut to 2 KB of UTF-8 on a character boundary. + assert long["reranker.query"] == long["input.value"] == "é" * 1024 + wire = traced.wire() + assert VOYAGE_KEY not in wire + assert OTHER_KEY not in wire + + +_HIDE_CASES = [ + # (TraceConfig kwargs, environment, query kept, scores kept) + pytest.param({"hide_inputs": True}, {}, False, True, id="hide_inputs"), + pytest.param({"hide_input_text": True}, {}, False, True, id="hide_input_text"), + pytest.param({"hide_outputs": True}, {}, True, False, id="hide_outputs"), + pytest.param({"hide_inputs": True, "hide_outputs": True}, {}, False, False, id="hide_both"), + pytest.param({}, {"FI_HIDE_INPUTS": "true"}, False, True, id="FI_HIDE_INPUTS"), + pytest.param({}, {"FI_HIDE_OUTPUTS": "true"}, True, False, id="FI_HIDE_OUTPUTS"), + pytest.param( + {}, + {"FI_HIDE_INPUTS": "true", "FI_HIDE_OUTPUTS": "true"}, + False, + False, + id="FI_HIDE_both", + ), +] + + +@pytest.mark.parametrize("capture_content", [False, True], ids=["default", "capture"]) +@pytest.mark.parametrize(("config", "environment", "query_kept", "scores_kept"), _HIDE_CASES) +def test_hide_flags_drop_the_query_and_the_scores_independently( + fake, monkeypatch, capture_content, config, environment, query_kept, scores_kept +): + for name, value in environment.items(): + monkeypatch.setenv(name, value) + options = {"capture_content": capture_content} + if config: + options["config"] = TraceConfig(**config) + traced, embed, rerank = _embed_and_rerank(fake, **options) + wire = traced.wire() + + if query_kept: + assert rerank["reranker.query"] == QUERY + else: + for values in (embed, rerank): + assert "reranker.query" not in values + assert "input.value" not in values + assert "input.mime_type" not in values + assert QUERY not in wire + # Scores are numbers, not input text: hiding inputs keeps them (PRD J2). + if scores_kept: + assert json.loads(rerank["output.value"]) == TOP_2_SCORES + else: + assert "output.value" not in rerank + assert "output.mime_type" not in rerank + for marker in SCORE_MARKERS: + assert marker not in wire, marker + # Texts and documents only with capture on and inputs not hidden. + for marker in OPT_IN_MARKERS: + assert (marker in wire) is (capture_content and query_kept), marker + # Counts stay exact whatever is hidden; vectors and the key never appear. + assert embed["voyage.embedding.count"] == 2 + assert rerank["voyage.rerank.document_count"] == 3 + assert rerank["voyage.rerank.result_count"] == 2 + assert VECTOR_MARKER not in wire + assert VOYAGE_KEY not in wire + + def test_capture_content_records_inputs_and_scores_but_never_vectors(fake): traced, embed, rerank = _embed_and_rerank(fake, capture_content=True) diff --git a/python/frameworks/voyage/tests/test_contract_harness.py b/python/frameworks/voyage/tests/test_contract_harness.py index d16118acd..0708747a5 100644 --- a/python/frameworks/voyage/tests/test_contract_harness.py +++ b/python/frameworks/voyage/tests/test_contract_harness.py @@ -30,6 +30,7 @@ DIMENSION, DOCUMENTS, EMBED_MODEL, + OPT_IN_MARKERS, QUERY, RERANK_MODEL, SCORE_MARKERS, @@ -145,13 +146,19 @@ def test_real_client_calls_reach_the_collector_contract(monkeypatch): assert "gen_ai.usage.total_tokens" not in failed -def test_no_key_vectors_or_content_are_exported_by_default(monkeypatch): +def test_only_the_query_and_scores_are_exported_by_default(monkeypatch): with FakeVoyage() as fake, Receiver() as receiver: spans, exports = _journey(receiver, fake, monkeypatch) wire = json.dumps(spans) + json.dumps(exports) - for secret in (VOYAGE_KEY, VECTOR_MARKER) + CONTENT_MARKERS + SCORE_MARKERS: + # Texts, documents, vectors and the key need capture_content (or never). + for secret in (VOYAGE_KEY, VECTOR_MARKER) + OPT_IN_MARKERS: assert secret not in wire, secret + # PRD J2 / AC-03: the rerank query and scores are exported by default. + for rerank_span in _by_name(spans)["voyage.rerank"]: + rerank = _flatten_attributes(rerank_span["attributes"]) + assert rerank["reranker.query"] == QUERY + assert SCORE_MARKERS[1] in rerank["output.value"] def test_capture_content_control_run_puts_the_markers_on_the_wire(monkeypatch): diff --git a/python/frameworks/voyage/tests/test_example.py b/python/frameworks/voyage/tests/test_example.py index c839e803b..600a3081d 100644 --- a/python/frameworks/voyage/tests/test_example.py +++ b/python/frameworks/voyage/tests/test_example.py @@ -22,7 +22,7 @@ sys.path.insert(0, str(PYTHON_ROOT / "tests")) from _support import ( # noqa: E402 - CONTENT_MARKERS, + OPT_IN_MARKERS, SCORE_MARKERS, VECTOR_MARKER, VOYAGE_KEY, @@ -82,9 +82,11 @@ def test_example_exports_one_embedding_and_one_reranker_span(): assert resource["project_name"] == "voyage-example" assert resource["project_type"] == "observe" - # The example's own documents and query, its key, vectors and scores - # stay off the wire (content capture is off by default). + # The example's documents, its key and vectors stay off the wire (texts + # and documents need capture_content). The rerank query and scores are + # recorded by default (PRD J2 / AC-03). wire = json.dumps(spans) - example_markers = ("EXAMPLE-DOC", "EXAMPLE-QUERY") - for marker in (VOYAGE_KEY, VECTOR_MARKER) + example_markers + CONTENT_MARKERS + SCORE_MARKERS: + for marker in (VOYAGE_KEY, VECTOR_MARKER, "EXAMPLE-DOC") + OPT_IN_MARKERS: assert marker not in wire, marker + assert by_name["voyage.rerank"]["reranker.query"].startswith("EXAMPLE-QUERY") + assert SCORE_MARKERS[1] in by_name["voyage.rerank"]["output.value"] diff --git a/python/frameworks/voyage/tests/test_rerank.py b/python/frameworks/voyage/tests/test_rerank.py index 99096398a..18ce81c8b 100644 --- a/python/frameworks/voyage/tests/test_rerank.py +++ b/python/frameworks/voyage/tests/test_rerank.py @@ -2,16 +2,17 @@ from __future__ import annotations +import json + import pytest pytest.importorskip("voyageai", reason="voyageai must be installed to test its instrumentor") from _support import ( # noqa: E402 - CONTENT_MARKERS, DOCUMENTS, QUERY, RERANK_MODEL, - SCORE_MARKERS, + SCORES, VOYAGE_KEY, FakeVoyage, attrs, @@ -53,18 +54,23 @@ def test_rerank_emits_one_reranker_span_with_counts_top_k_and_tokens(fake): assert status_code(span) == "OK" -def test_rerank_span_carries_no_query_documents_scores_or_key_by_default(fake): +def test_rerank_span_carries_the_query_and_scores_but_no_documents_or_key_by_default(fake): with instrumented() as traced: client(fake).rerank(QUERY, DOCUMENTS, model=RERANK_MODEL, top_k=2) values = attrs(traced.one()) - for key in ("reranker.query", "reranker.input_documents", "reranker.output_documents"): + # PRD J2 / AC-03: query and scores are on by default; documents are opt-in. + assert values["reranker.query"] == QUERY + assert values["input.value"] == QUERY + assert json.loads(values["output.value"]) == [ + {"index": 1, "relevance_score": SCORES[1]}, + {"index": 0, "relevance_score": SCORES[0]}, + ] + for key in ("reranker.input_documents", "reranker.output_documents"): assert not any(name.startswith(key) for name in values), key - assert "input.value" not in values - assert "output.value" not in values wire = traced.wire() assert VOYAGE_KEY not in wire - for marker in CONTENT_MARKERS + SCORE_MARKERS: + for marker in DOCUMENTS: assert marker not in wire, marker diff --git a/python/frameworks/voyage/traceai_voyage/__init__.py b/python/frameworks/voyage/traceai_voyage/__init__.py index 728d05190..151172cdb 100644 --- a/python/frameworks/voyage/traceai_voyage/__init__.py +++ b/python/frameworks/voyage/traceai_voyage/__init__.py @@ -38,12 +38,16 @@ class VoyageInstrumentor(BaseInstrumentor): # type: ignore[misc] ``instrument()`` accepts: * ``tracer_provider``: the provider to use (default: the global one). - * ``config``: a ``fi_instrumentation.TraceConfig``. ``hide_inputs`` and - ``hide_outputs`` apply to captured content; ``pii_redaction`` applies to - every recorded value. - * ``capture_content``: ``False`` by default. When ``True``, spans also - record the texts, the rerank query and documents, and the rerank scores, - subject to ``config``. Embedding vectors are never recorded. + * ``config``: a ``fi_instrumentation.TraceConfig``. ``hide_inputs`` (or + ``hide_input_text``, or ``FI_HIDE_INPUTS``) drops the rerank query and + any captured texts and documents; ``hide_outputs`` (or + ``FI_HIDE_OUTPUTS``) drops the rerank scores. Hiding inputs keeps the + scores. ``pii_redaction`` applies to every recorded value. + * ``capture_content``: ``False`` by default. By default only the rerank + span records content: the query (``reranker.query`` and a plain-text + ``input.value``) and the scores (``output.value``), as PRD J2 / AC-03 + require; embed spans record none. ``True`` also records the embed texts + and the rerank documents. Embedding vectors are never recorded. """ def instrumentation_dependencies(self) -> Collection[str]: diff --git a/python/frameworks/voyage/traceai_voyage/_wrappers.py b/python/frameworks/voyage/traceai_voyage/_wrappers.py index 1ed82c9e0..11c81454c 100644 --- a/python/frameworks/voyage/traceai_voyage/_wrappers.py +++ b/python/frameworks/voyage/traceai_voyage/_wrappers.py @@ -2,9 +2,11 @@ One span per public call. The span is current while the SDK sends HTTP, so an HTTP client instrumentation nests under it, and it covers the client's own -retries. Content (texts, query, documents, scores) is recorded only with -``capture_content=True``; embedding vectors are never recorded. Nothing in -this module may change what the caller gets back or raises. +retries. By default the rerank span records the query and the scores (PRD J2, +AC-03); embed texts and rerank documents are recorded only with +``capture_content=True``. TraceConfig ``hide_inputs`` / ``hide_outputs`` drop +them. Embedding vectors are never recorded. Nothing in this module may change +what the caller gets back or raises. """ from __future__ import annotations @@ -38,6 +40,7 @@ RERANK_RESULT_COUNT = "voyage.rerank.result_count" SERVER_ADDRESS = "server.address" JSON_MIME_TYPE = "application/json" +TEXT_MIME_TYPE = "text/plain" # Captured content limits: at most this many texts or documents, each cut to # this many UTF-8 bytes on a character boundary. Counts stay exact. @@ -254,21 +257,34 @@ def _input_content( kwargs: Mapping[str, Any], secrets: Sequence[str], config: TraceConfig, + capture_content: bool, ) -> Dict[str, Any]: - """Texts, query and documents, only with capture on and inputs not hidden.""" + """Input content, unless TraceConfig hides inputs. + + By default only the rerank query (PRD J2, AC-03): ``reranker.query`` and a + plain-text ``input.value``. With ``capture_content`` the embed texts, and + the rerank query with its documents as JSON in ``input.value``. + """ if _hides_inputs(config): return {} attributes: Dict[str, Any] = {} if operation is EMBED: + if not capture_content: + return attributes texts = _captured_list(_argument(operation, args, kwargs, "texts"), secrets) if texts is not None: attributes[SpanAttributes.INPUT_VALUE] = json.dumps(texts, ensure_ascii=False) attributes[SpanAttributes.INPUT_MIME_TYPE] = JSON_MIME_TYPE return attributes query = _argument(operation, args, kwargs, "query") - documents = _captured_list(_argument(operation, args, kwargs, "documents"), secrets) if query is not None: attributes[RerankerAttributes.RERANKER_QUERY] = _captured_text(query, secrets) + if not capture_content: + if query is not None: + attributes[SpanAttributes.INPUT_VALUE] = attributes[RerankerAttributes.RERANKER_QUERY] + attributes[SpanAttributes.INPUT_MIME_TYPE] = TEXT_MIME_TYPE + return attributes + documents = _captured_list(_argument(operation, args, kwargs, "documents"), secrets) payload: Dict[str, Any] = {} if query is not None: payload["query"] = attributes[RerankerAttributes.RERANKER_QUERY] @@ -318,7 +334,10 @@ def _response_attributes( def _output_content(operation: Operation, result: Any, config: TraceConfig) -> Dict[str, Any]: - """Rerank scores in the client's order, only with capture on and outputs not hidden.""" + """Rerank scores in the client's order, by default, unless outputs are hidden. + + Scores are numbers, not input text, so hiding inputs keeps them (PRD J2). + """ if operation is not RERANK or config.hide_outputs: return {} results = getattr(result, "results", None) @@ -403,13 +422,19 @@ def __init__( attributes.update(_request_attributes(operation, instance, args, kwargs)) except Exception: logger.debug("Could not read Voyage request attributes", exc_info=True) - if wrapper.capture_content: - try: - attributes.update( - _input_content(operation, args, kwargs, self._secrets, wrapper.config) + try: + attributes.update( + _input_content( + operation, + args, + kwargs, + self._secrets, + wrapper.config, + wrapper.capture_content, ) - except Exception: - logger.debug("Could not capture Voyage request content", exc_info=True) + ) + except Exception: + logger.debug("Could not capture Voyage request content", exc_info=True) try: self._span = wrapper.tracer.start_span(operation.span_name, attributes=attributes) except Exception: @@ -447,11 +472,10 @@ def ok(self, result: Any) -> None: ) except Exception: logger.debug("Could not read Voyage response attributes", exc_info=True) - if self._wrapper.capture_content: - try: - _set_attributes(span, _output_content(operation, result, self._wrapper.config)) - except Exception: - logger.debug("Could not capture Voyage response content", exc_info=True) + try: + _set_attributes(span, _output_content(operation, result, self._wrapper.config)) + except Exception: + logger.debug("Could not capture Voyage response content", exc_info=True) span.set_status(Status(StatusCode.OK)) except Exception: logger.debug("Could not finish the Voyage span", exc_info=True) From 998d61ae43688c649c847a3367f7a0f5cce59d0e Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Mon, 5 Oct 2026 02:44:19 -0700 Subject: [PATCH 11/17] fix(voyage): cap recorded rerank scores at 64 results Review R5. output.value now holds at most MAX_CAPTURED_ITEMS (64) scores, the first 64 results in the client's order, matching the cap on texts and documents. voyage.rerank.result_count stays exact. A rerank over up to 1000 documents without top_k no longer writes a ~1000-entry attribute. The README caps sentence names the score cap. approved: Nikhil 2026-10-03 blanket Refs: TH-8321 --- python/frameworks/voyage/README.md | 8 +++++--- .../voyage/tests/test_content_capture.py | 17 +++++++++++++++++ .../voyage/traceai_voyage/_wrappers.py | 9 ++++++--- 3 files changed, 28 insertions(+), 6 deletions(-) diff --git a/python/frameworks/voyage/README.md b/python/frameworks/voyage/README.md index abc32c347..ed420a324 100644 --- a/python/frameworks/voyage/README.md +++ b/python/frameworks/voyage/README.md @@ -132,9 +132,11 @@ VoyageInstrumentor().instrument( ) ``` -Captured content holds at most 64 texts or documents, and the query and each -text or document are cut to 2 KB of UTF-8 on a character boundary; the counts -stay exact. `pii_redaction=True` applies to every recorded value. +At most 64 texts, 64 documents and 64 scores (the first 64 results, in the +client's order) are recorded, and the query and each text or document are +cut to 2 KB of UTF-8 on a character boundary; `voyage.embedding.count`, +`voyage.rerank.document_count` and `voyage.rerank.result_count` stay exact. +`pii_redaction=True` applies to every recorded value. Embedding vectors are never recorded, with or without capture, and `hide_embedding_vectors` has nothing to hide here. diff --git a/python/frameworks/voyage/tests/test_content_capture.py b/python/frameworks/voyage/tests/test_content_capture.py index 56a93f2ab..9b0a3954d 100644 --- a/python/frameworks/voyage/tests/test_content_capture.py +++ b/python/frameworks/voyage/tests/test_content_capture.py @@ -261,6 +261,23 @@ def test_captured_content_is_capped(fake): assert len(captured.encode("utf-8")) == 2048 +@pytest.mark.parametrize("capture_content", [False, True], ids=["default", "capture"]) +def test_rerank_scores_are_capped_and_the_result_count_stays_exact(fake, capture_content): + documents = ["d{0}".format(i) for i in range(70)] + with instrumented(capture_content=capture_content) as traced: + result = client(fake).rerank(QUERY, documents, model=RERANK_MODEL) + + assert len(result.results) == 70 + values = attrs(traced.one()) + assert values["voyage.rerank.document_count"] == 70 + assert values["voyage.rerank.result_count"] == 70 + # The first 64 results, in the client's order. + assert json.loads(values["output.value"]) == [ + {"index": item.index, "relevance_score": item.relevance_score} + for item in result.results[:64] + ] + + def test_pii_redaction_in_trace_config_applies_to_captured_content(fake): email = "jane.doe@example.com" with instrumented(capture_content=True, config=TraceConfig(pii_redaction=True)) as traced: diff --git a/python/frameworks/voyage/traceai_voyage/_wrappers.py b/python/frameworks/voyage/traceai_voyage/_wrappers.py index 11c81454c..287f938f3 100644 --- a/python/frameworks/voyage/traceai_voyage/_wrappers.py +++ b/python/frameworks/voyage/traceai_voyage/_wrappers.py @@ -42,8 +42,9 @@ JSON_MIME_TYPE = "application/json" TEXT_MIME_TYPE = "text/plain" -# Captured content limits: at most this many texts or documents, each cut to -# this many UTF-8 bytes on a character boundary. Counts stay exact. +# Recorded content limits: at most this many texts, documents or rerank +# scores, and each text, document or query cut to this many UTF-8 bytes on a +# character boundary. Counts stay exact. MAX_CAPTURED_ITEMS = 64 MAX_CAPTURED_BYTES = 2048 @@ -343,12 +344,14 @@ def _output_content(operation: Operation, result: Any, config: TraceConfig) -> D results = getattr(result, "results", None) if not isinstance(results, (list, tuple)): return {} + # At most MAX_CAPTURED_ITEMS, like texts and documents; result_count + # stays exact. scores = [ { "index": getattr(item, "index", None), "relevance_score": getattr(item, "relevance_score", None), } - for item in results + for item in results[:MAX_CAPTURED_ITEMS] ] return { SpanAttributes.OUTPUT_VALUE: json.dumps(scores), From 58c6bd2f7d85b7713b2160e703ee7805e6c391fe Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Mon, 5 Oct 2026 02:54:47 -0700 Subject: [PATCH 12/17] fix(voyage): apply pii_redaction to the error status and exception event Review R4. FiSpan masks span attributes only; the error status description and the exception event's exception.message and exception.stacktrace go through set_status / add_event unmasked. With TraceConfig pii_redaction on, the wrapper now runs them through fi_instrumentation's redact_pii_in_value (the function TraceConfig.mask uses), after the API key is removed, so a key that looks like PII is still removed whole. With pii_redaction off nothing changes. The README now says exactly what pii_redaction covers: every string attribute (query, captured texts and documents, scores JSON, other attributes) and the error status and exception event, not span names, event names or exception.type, and that a score with 10 or more decimal digits is rewritten by the digit patterns. Tests: a fake echo-input model returns an error that echoes user text; embed and rerank error spans carry instead of the email in the status, message and stacktrace; pii off keeps the text; a phone-number-like key is redacted before PII redaction runs (fails if the order is swapped). approved: Nikhil 2026-10-03 blanket Refs: TH-8321 --- python/frameworks/voyage/README.md | 21 ++++++- python/frameworks/voyage/tests/_support.py | 6 ++ .../voyage/tests/test_content_capture.py | 12 ++++ python/frameworks/voyage/tests/test_errors.py | 57 +++++++++++++++++++ .../voyage/traceai_voyage/__init__.py | 4 +- .../voyage/traceai_voyage/_wrappers.py | 32 ++++++++++- 6 files changed, 126 insertions(+), 6 deletions(-) diff --git a/python/frameworks/voyage/README.md b/python/frameworks/voyage/README.md index ed420a324..85a23cd41 100644 --- a/python/frameworks/voyage/README.md +++ b/python/frameworks/voyage/README.md @@ -136,7 +136,23 @@ At most 64 texts, 64 documents and 64 scores (the first 64 results, in the client's order) are recorded, and the query and each text or document are cut to 2 KB of UTF-8 on a character boundary; `voyage.embedding.count`, `voyage.rerank.document_count` and `voyage.rerank.result_count` stay exact. -`pii_redaction=True` applies to every recorded value. + +`TraceConfig(pii_redaction=True)` (or `FI_PII_REDACTION=true`) applies +`fi_instrumentation`'s regex PII redaction, which replaces email addresses, +SSNs, card numbers, `sk-`/`pk-` live/test/prod API keys, IPv4 addresses and +phone numbers with tokens such as ``. It covers: + +- every string span attribute: the query, captured texts and documents, the + scores JSON, and the other attributes above (an IP `server.address` becomes + ``); +- the error status description and the `exception` event's + `exception.message` and `exception.stacktrace`, after the API key is + removed from them. + +It does not change span names, event names or `exception.type`. The patterns +match digits, not meaning: a relevance score with 10 or more digits after the +decimal point is rewritten too (for example `0.`), which leaves +that `output.value` as invalid JSON. Embedding vectors are never recorded, with or without capture, and `hide_embedding_vectors` has nothing to hide here. @@ -152,7 +168,8 @@ messages and stack traces with `[redacted]`. An error raised by the client (for example `voyageai.error.AuthenticationError` for HTTP 401, or `voyageai.error.Timeout`) sets the span status to ERROR with `: ` and records one `exception` event; the key is -redacted from both. The exception reaches your code unchanged. +redacted from both, and with `pii_redaction=True` so is PII (see above). The +exception reaches your code unchanged. Cancelling an `AsyncClient` call (`asyncio.CancelledError`) or interrupting a blocking call (`KeyboardInterrupt`) ends the span with status ERROR, diff --git a/python/frameworks/voyage/tests/_support.py b/python/frameworks/voyage/tests/_support.py index f03266878..e3ba1063a 100644 --- a/python/frameworks/voyage/tests/_support.py +++ b/python/frameworks/voyage/tests/_support.py @@ -12,6 +12,8 @@ * ``fail-401``: HTTP 401 with a ``detail`` message. * ``echo-key``: HTTP 400 whose ``detail`` echoes the bearer key back, so a test can prove the key is redacted from error text. +* ``echo-input``: HTTP 400 whose ``detail`` echoes the rerank query or the + first text back, so a test can prove PII redaction covers error text. * ``rate-limit-once``: the first request gets HTTP 429, later ones succeed. * ``slow``: the request is held until the fake closes (timeouts, cancels). """ @@ -129,6 +131,10 @@ def do_POST(self) -> None: # noqa: N802 - BaseHTTPRequestHandler API elif model == "echo-key": key = authorization[len("Bearer ") :] self._send(400, {"detail": "bad request for key {0}".format(key)}) + elif model == "echo-input": + echoed = body.get("query") or body.get("input") or "" + echoed = echoed if isinstance(echoed, str) else echoed[0] + self._send(400, {"detail": "cannot process {0}".format(echoed)}) elif model == "rate-limit-once" and owner._take_rate_limit(): self._send(429, {"detail": "rate limited"}) elif path == "/v1/embeddings": diff --git a/python/frameworks/voyage/tests/test_content_capture.py b/python/frameworks/voyage/tests/test_content_capture.py index 9b0a3954d..1e14ca044 100644 --- a/python/frameworks/voyage/tests/test_content_capture.py +++ b/python/frameworks/voyage/tests/test_content_capture.py @@ -286,6 +286,18 @@ def test_pii_redaction_in_trace_config_applies_to_captured_content(fake): assert email not in traced.wire() +def test_pii_redaction_applies_to_the_default_query_and_keeps_the_scores(fake): + email = "jane.doe@example.com" + with instrumented(config=TraceConfig(pii_redaction=True)) as traced: + client(fake).rerank("mail {0}".format(email), DOCUMENTS, model=RERANK_MODEL, top_k=2) + + values = attrs(traced.one()) + assert values["reranker.query"] == values["input.value"] == "mail " + # The fake's scores are short enough that no PII pattern matches them. + assert json.loads(values["output.value"]) == TOP_2_SCORES + assert email not in traced.wire() + + @pytest.mark.parametrize( "options", [{"config": {"hide_inputs": True}}, {"capture_content": "yes"}, {"capture_content": 1}], diff --git a/python/frameworks/voyage/tests/test_errors.py b/python/frameworks/voyage/tests/test_errors.py index 773393992..ce695f89e 100644 --- a/python/frameworks/voyage/tests/test_errors.py +++ b/python/frameworks/voyage/tests/test_errors.py @@ -9,6 +9,7 @@ pytest.importorskip("voyageai", reason="voyageai must be installed to test its instrumentor") import voyageai # noqa: E402 +from fi_instrumentation import TraceConfig # noqa: E402 from voyageai.api_resources import api_requestor # noqa: E402 from _support import ( # noqa: E402 @@ -76,6 +77,62 @@ def test_a_key_echoed_in_an_error_message_is_redacted(fake): assert VOYAGE_KEY not in traced.wire() +EMAIL = "jane.doe@example.com" + + +@pytest.mark.parametrize("operation", ["embed", "rerank"]) +def test_pii_redaction_covers_the_error_status_and_exception_event(fake, operation): + text = "mail {0} now".format(EMAIL) + with instrumented(config=TraceConfig(pii_redaction=True)) as traced: + with pytest.raises(voyageai.error.InvalidRequestError) as raised: + if operation == "embed": + client(fake).embed([text], model="echo-input") + else: + client(fake).rerank(text, DOCUMENTS, model="echo-input") + + # The caller still sees the vendor's message unchanged. + assert EMAIL in str(raised.value) + span = traced.one() + (event,) = span.events + # set_status and add_event bypass FiSpan's attribute masking, so the + # wrapper applies TraceConfig's PII redaction to them itself. + for recorded in ( + span.status.description, + event.attributes["exception.message"], + event.attributes["exception.stacktrace"], + ): + assert EMAIL not in recorded + assert "" in recorded + assert EMAIL not in traced.wire() + + +def test_without_pii_redaction_error_text_is_unchanged_apart_from_the_key(fake): + with instrumented() as traced: + with pytest.raises(voyageai.error.InvalidRequestError): + client(fake).embed(["mail {0} now".format(EMAIL)], model="echo-input") + + span = traced.one() + assert EMAIL in span.status.description + assert EMAIL in span.events[0].attributes["exception.message"] + assert EMAIL in span.events[0].attributes["exception.stacktrace"] + + +def test_the_key_is_redacted_before_pii_redaction(fake): + # A key with a phone-number-like run: PII redaction first would rewrite + # part of it and leave the rest of the key behind. + key = "pa-placeholder-5551234567-PHONE-KEY-TAIL" + with instrumented(config=TraceConfig(pii_redaction=True)) as traced: + with pytest.raises(voyageai.error.InvalidRequestError): + voyageai.Client(api_key=key, base_url=fake.base_url).embed(TEXTS, model="echo-key") + + span = traced.one() + assert "[redacted]" in span.status.description + assert "[redacted]" in span.events[0].attributes["exception.message"] + wire = traced.wire() + for fragment in (key, "5551234567", "PHONE-KEY-TAIL"): + assert fragment not in wire, fragment + + def test_a_client_timeout_ends_the_span_as_an_error(fake): with instrumented() as traced: with pytest.raises(voyageai.error.Timeout): diff --git a/python/frameworks/voyage/traceai_voyage/__init__.py b/python/frameworks/voyage/traceai_voyage/__init__.py index 151172cdb..af63efef0 100644 --- a/python/frameworks/voyage/traceai_voyage/__init__.py +++ b/python/frameworks/voyage/traceai_voyage/__init__.py @@ -42,7 +42,9 @@ class VoyageInstrumentor(BaseInstrumentor): # type: ignore[misc] ``hide_input_text``, or ``FI_HIDE_INPUTS``) drops the rerank query and any captured texts and documents; ``hide_outputs`` (or ``FI_HIDE_OUTPUTS``) drops the rerank scores. Hiding inputs keeps the - scores. ``pii_redaction`` applies to every recorded value. + scores. ``pii_redaction`` applies to every string attribute and, after + the key is removed, to the error status description and the exception + event's message and stack trace. * ``capture_content``: ``False`` by default. By default only the rerank span records content: the query (``reranker.query`` and a plain-text ``input.value``) and the scores (``output.value``), as PRD J2 / AC-03 diff --git a/python/frameworks/voyage/traceai_voyage/_wrappers.py b/python/frameworks/voyage/traceai_voyage/_wrappers.py index 287f938f3..9dec9f761 100644 --- a/python/frameworks/voyage/traceai_voyage/_wrappers.py +++ b/python/frameworks/voyage/traceai_voyage/_wrappers.py @@ -26,6 +26,11 @@ from opentelemetry import trace as trace_api from opentelemetry.trace import Span, Status, StatusCode, Tracer +try: # The function TraceConfig.mask applies when pii_redaction is on. + from fi_instrumentation.instrumentation.config import redact_pii_in_value +except ImportError: # pragma: no cover - present from fi-instrumentation-otel 1.1.0 + redact_pii_in_value = None + logger = logging.getLogger(__name__) PROVIDER = "voyage" @@ -375,12 +380,33 @@ def _describe(error: BaseException, secrets: Sequence[str]) -> Tuple[str, str]: return type(error).__name__, _redact(message, secrets) -def _record_error(span: Span, error: BaseException, secrets: Sequence[str]) -> None: +def _redact_pii(value: str, config: TraceConfig) -> str: + """TraceConfig's PII redaction for text FiSpan does not mask. + + FiSpan masks attributes only; the status description and the exception + event go through set_status / add_event unmasked, so with + ``pii_redaction`` on they are redacted here. Callers remove the key first, + so a key that looks like PII is still removed whole. + """ + if not config.pii_redaction: + return value + if redact_pii_in_value is not None: + redacted = redact_pii_in_value(value) + else: # pragma: no cover - same function, reached through TraceConfig + redacted = config.mask("exception.message", value) + return redacted if isinstance(redacted, str) else value + + +def _record_error( + span: Span, error: BaseException, secrets: Sequence[str], config: TraceConfig +) -> None: name, message = _describe(error, secrets) try: stacktrace = "".join(traceback.format_exception(type(error), error, error.__traceback__)) except Exception: stacktrace = "" + message = _redact_pii(message, config) + stacktrace = _redact_pii(_redact(stacktrace, secrets), config) # The SDK's record_exception would copy str(error) verbatim; the key may # be in it (a server echoing it back), so the event is built here. span.add_event( @@ -388,7 +414,7 @@ def _record_error(span: Span, error: BaseException, secrets: Sequence[str]) -> N { "exception.type": "{0}.{1}".format(type(error).__module__, type(error).__qualname__), "exception.message": message, - "exception.stacktrace": _redact(stacktrace, secrets), + "exception.stacktrace": stacktrace, }, ) span.set_status(Status(StatusCode.ERROR, "{0}: {1}".format(name, message))) @@ -490,7 +516,7 @@ def error(self, error: BaseException) -> None: if span is None: return try: - _record_error(span, error, self._secrets) + _record_error(span, error, self._secrets, self._wrapper.config) except Exception: logger.debug("Could not record the Voyage error", exc_info=True) finally: From 27094c4cd34202d2e0348ed9fc0b9575993cd223 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Mon, 5 Oct 2026 02:55:38 -0700 Subject: [PATCH 13/17] docs(voyage): add CHANGELOG.md with the 0.1.0 entry Review R6. Same layout as the other framework packages (groq): a 0.1.0 entry describing the package, its version range, what it traces, the default content (rerank query and scores, PRD J2/AC-03), the capture_content opt-in, caps, key and PII redaction, errors and cancellation, and the two untraced methods. A packaging test checks the changelog has an entry for the pyproject version. approved: Nikhil 2026-10-03 blanket Refs: TH-8321 --- python/frameworks/voyage/CHANGELOG.md | 21 +++++++++++++++++++ .../frameworks/voyage/tests/test_packaging.py | 9 ++++++++ 2 files changed, 30 insertions(+) create mode 100644 python/frameworks/voyage/CHANGELOG.md diff --git a/python/frameworks/voyage/CHANGELOG.md b/python/frameworks/voyage/CHANGELOG.md new file mode 100644 index 000000000..91a283ce2 --- /dev/null +++ b/python/frameworks/voyage/CHANGELOG.md @@ -0,0 +1,21 @@ +## [0.1.0] - 2026-10-05 +### Feature +- First release. OpenTelemetry instrumentation for the official Voyage AI + Python client (`voyageai>=0.3.7,<1`, `fi-instrumentation-otel>=1.1.0`), + tested on Python 3.10 to 3.13. +- `VoyageInstrumentor` wraps `Client` and `AsyncClient` `embed` (one + `voyage.embed` EMBEDDING span per call) and `rerank` (one `voyage.rerank` + RERANKER span per call). The client's own retries stay inside that span. +- Spans record the model, text and document counts, the requested and returned + embedding dimensions, the host as `server.address`, and the client's + `total_tokens` on `gen_ai.usage.input_tokens` and `gen_ai.usage.total_tokens`. +- The rerank span records the query and the relevance scores by default (PRD + J2 / AC-03). Embed texts and rerank documents are recorded only with + `instrument(capture_content=True)`. TraceConfig `hide_inputs` and + `hide_outputs` drop them. Recorded content is capped at 64 items and 2 KB per + string. Embedding vectors are never recorded. +- The Voyage API key is never exported. `pii_redaction` also covers the error + status and the exception event. +- Errors set status ERROR and record one exception event. A cancelled async + call or a Ctrl-C ends the span as `cancelled`. +- `multimodal_embed` and `contextualized_embed` are not traced in this release. diff --git a/python/frameworks/voyage/tests/test_packaging.py b/python/frameworks/voyage/tests/test_packaging.py index 129587f7c..ffee8ca6e 100644 --- a/python/frameworks/voyage/tests/test_packaging.py +++ b/python/frameworks/voyage/tests/test_packaging.py @@ -17,6 +17,7 @@ PACKAGE = Path(__file__).resolve().parents[1] PYPROJECT = PACKAGE / "pyproject.toml" +CHANGELOG = PACKAGE / "CHANGELOG.md" SOURCE = PACKAGE / "traceai_voyage" @@ -85,6 +86,14 @@ def test_distribution_name_and_license(): assert '"License :: OSI Approved :: Apache Software License"' in text +def test_the_changelog_has_an_entry_for_the_package_version(): + # Same layout as the other framework packages (e.g. groq). + version = re.search(r'^version = "([^"]+)"$', PYPROJECT.read_text(), re.M).group(1) + assert CHANGELOG.is_file() + heading = r"^## \[{0}\] - \d{{4}}-\d{{2}}-\d{{2}}$".format(re.escape(version)) + assert re.search(heading, CHANGELOG.read_text(), re.M), version + + def test_the_package_never_sets_base_url(): # D14: the client picks the host from the key; the instrumentor never does. for path in SOURCE.glob("*.py"): From 1645425c3c996bd4e262af7ff4f154b4cd60e931 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Mon, 5 Oct 2026 02:56:34 -0700 Subject: [PATCH 14/17] test(voyage): pin pii_redaction on server.address, as the README states Review R4 follow-up. The README says pii_redaction scans every string attribute and that an IP server.address becomes ; the default-query PII test now asserts that on the fake's 127.0.0.1 host. approved: Nikhil 2026-10-03 blanket Refs: TH-8321 --- python/frameworks/voyage/tests/test_content_capture.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/python/frameworks/voyage/tests/test_content_capture.py b/python/frameworks/voyage/tests/test_content_capture.py index 1e14ca044..04dcf3f2b 100644 --- a/python/frameworks/voyage/tests/test_content_capture.py +++ b/python/frameworks/voyage/tests/test_content_capture.py @@ -295,6 +295,8 @@ def test_pii_redaction_applies_to_the_default_query_and_keeps_the_scores(fake): assert values["reranker.query"] == values["input.value"] == "mail " # The fake's scores are short enough that no PII pattern matches them. assert json.loads(values["output.value"]) == TOP_2_SCORES + # Every string attribute is scanned: the fake's 127.0.0.1 host too. + assert values["server.address"] == "" assert email not in traced.wire() From 7a30179bf7fed9906730c803791dc7edf122b55f Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Sun, 11 Oct 2026 21:21:27 -0700 Subject: [PATCH 15/17] docs: move the README catalog row to the TH-8103 catalog PR The catalog rows for all TH-8103 integrations now live in one PR (docs/th-8103-integration-catalog), so this PR no longer conflicts with its siblings in README.md and can merge in any order. Co-Authored-By: Claude Opus 5.5 --- README.md | 1 - python/README.md | 1 - 2 files changed, 2 deletions(-) diff --git a/README.md b/README.md index 2b670adb1..214680868 100644 --- a/README.md +++ b/README.md @@ -245,7 +245,6 @@ var tracer = TraceAI.Register(opts => | [`traceAI-huggingface`](https://pypi.org/project/traceAI-huggingface/) | HuggingFace | [![PyPI](https://img.shields.io/pypi/v/traceAI-huggingface)](https://pypi.org/project/traceAI-huggingface/) | | [`traceAI-xai`](https://pypi.org/project/traceAI-xai/) | xAI (Grok) | [![PyPI](https://img.shields.io/pypi/v/traceAI-xai)](https://pypi.org/project/traceAI-xai/) | | [`traceAI-vllm`](https://pypi.org/project/traceAI-vllm/) | vLLM | [![PyPI](https://img.shields.io/pypi/v/traceAI-vllm)](https://pypi.org/project/traceAI-vllm/) | -| [`traceAI-voyage`](https://pypi.org/project/traceAI-voyage/) | Voyage AI (embeddings, rerank) | [![PyPI](https://img.shields.io/pypi/v/traceAI-voyage)](https://pypi.org/project/traceAI-voyage/) | #### Agent Frameworks diff --git a/python/README.md b/python/README.md index 6407668cf..2d5263467 100644 --- a/python/README.md +++ b/python/README.md @@ -165,7 +165,6 @@ trace_provider = register( | `traceai-bedrock` | AWS Bedrock | | `traceai-litellm` | LiteLLM | | `traceai-portkey` | Portkey | -| `traceai-voyage` | Voyage AI (embeddings, rerank) | ### Agent Frameworks From 9b0adff2577ad9677e3c490ebfdd3d6a007c505d Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Fri, 16 Oct 2026 18:06:24 -0700 Subject: [PATCH 16/17] fix(voyage): do not export exception.stacktrace (local paths, user name) Co-Authored-By: Claude Sonnet 5.5 --- python/frameworks/voyage/README.md | 2 +- python/frameworks/voyage/tests/test_errors.py | 16 ++++++++++++++-- .../voyage/traceai_voyage/_wrappers.py | 9 ++------- 3 files changed, 17 insertions(+), 10 deletions(-) diff --git a/python/frameworks/voyage/README.md b/python/frameworks/voyage/README.md index 85a23cd41..97d9a34c9 100644 --- a/python/frameworks/voyage/README.md +++ b/python/frameworks/voyage/README.md @@ -146,7 +146,7 @@ phone numbers with tokens such as ``. It covers: scores JSON, and the other attributes above (an IP `server.address` becomes ``); - the error status description and the `exception` event's - `exception.message` and `exception.stacktrace`, after the API key is + `exception.message`, after the API key is removed from them. It does not change span names, event names or `exception.type`. The patterns diff --git a/python/frameworks/voyage/tests/test_errors.py b/python/frameworks/voyage/tests/test_errors.py index ce695f89e..217a7aad8 100644 --- a/python/frameworks/voyage/tests/test_errors.py +++ b/python/frameworks/voyage/tests/test_errors.py @@ -99,7 +99,6 @@ def test_pii_redaction_covers_the_error_status_and_exception_event(fake, operati for recorded in ( span.status.description, event.attributes["exception.message"], - event.attributes["exception.stacktrace"], ): assert EMAIL not in recorded assert "" in recorded @@ -114,7 +113,20 @@ def test_without_pii_redaction_error_text_is_unchanged_apart_from_the_key(fake): span = traced.one() assert EMAIL in span.status.description assert EMAIL in span.events[0].attributes["exception.message"] - assert EMAIL in span.events[0].attributes["exception.stacktrace"] + assert "exception.stacktrace" not in span.events[0].attributes + + +def test_no_stacktrace_or_local_path_is_exported(fake): + with instrumented() as traced: + with pytest.raises(voyageai.error.InvalidRequestError): + client(fake).embed(["x"], model="echo-input") + + span = traced.one() + assert "exception.stacktrace" not in span.events[0].attributes + wire = traced.wire() + assert "Traceback" not in wire + assert "site-packages" not in wire + assert __file__ not in wire def test_the_key_is_redacted_before_pii_redaction(fake): diff --git a/python/frameworks/voyage/traceai_voyage/_wrappers.py b/python/frameworks/voyage/traceai_voyage/_wrappers.py index 9dec9f761..f1bdf914d 100644 --- a/python/frameworks/voyage/traceai_voyage/_wrappers.py +++ b/python/frameworks/voyage/traceai_voyage/_wrappers.py @@ -15,7 +15,6 @@ import json import logging import sys -import traceback from dataclasses import dataclass from typing import Any, Callable, Dict, List, Mapping, Optional, Sequence, Tuple from urllib.parse import urlsplit @@ -401,12 +400,9 @@ def _record_error( span: Span, error: BaseException, secrets: Sequence[str], config: TraceConfig ) -> None: name, message = _describe(error, secrets) - try: - stacktrace = "".join(traceback.format_exception(type(error), error, error.__traceback__)) - except Exception: - stacktrace = "" message = _redact_pii(message, config) - stacktrace = _redact_pii(_redact(stacktrace, secrets), config) + # No exception.stacktrace: a Python traceback carries local file paths and + # the user name, which this package never exports. # The SDK's record_exception would copy str(error) verbatim; the key may # be in it (a server echoing it back), so the event is built here. span.add_event( @@ -414,7 +410,6 @@ def _record_error( { "exception.type": "{0}.{1}".format(type(error).__module__, type(error).__qualname__), "exception.message": message, - "exception.stacktrace": stacktrace, }, ) span.set_status(Status(StatusCode.ERROR, "{0}: {1}".format(name, message))) From 007824311b2f9e47675acc507c9785d9542fa6a0 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Sat, 17 Oct 2026 03:35:35 -0700 Subject: [PATCH 17/17] docs(voyage): record live vendor verification (2026-10-17) Co-Authored-By: Claude Sonnet 5.5 --- python/frameworks/voyage/README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/python/frameworks/voyage/README.md b/python/frameworks/voyage/README.md index 97d9a34c9..b65bcabf7 100644 --- a/python/frameworks/voyage/README.md +++ b/python/frameworks/voyage/README.md @@ -98,6 +98,10 @@ the Future AGI collector reads. If another instrumentation in your app (for example a framework that wraps the same embedding call) also records usage for it, the trace counts those tokens twice. +## Live vendor verification (2026-10-17) + +Run against the real Voyage API (`api.voyageai.com`, default host, no `base_url`) with `voyageai` 0.5.0, `voyage-3.5-lite` embeddings and `rerank-2.5-lite`, exported to Future AGI: an `AGENT` span with `voyage.embed` (EMBEDDING, tokens recorded), `voyage.rerank` (RERANKER, tokens recorded) and an invalid-key `voyage.embed` as an `ERROR` span (`AuthenticationError`). Default, `capture_content=True` and `FI_HIDE_*` privacy modes were checked with a marker scan. Not yet run live: the Atlas host (`ai.mongodb.com`), geography hosts and local in-process models. + ## Privacy and content capture By default the rerank span records the query and the relevance scores, as PRD