diff --git a/.github/workflows/openai-compatible-recipe-tests.yml b/.github/workflows/openai-compatible-recipe-tests.yml index 0e2587e85..0205fc2b0 100644 --- a/.github/workflows/openai-compatible-recipe-tests.yml +++ b/.github/workflows/openai-compatible-recipe-tests.yml @@ -39,7 +39,7 @@ jobs: import json, os from pathlib import Path matrix = [] - for recipe in ('databricks', 'lepton', 'orcarouter', 'anannas-ai', 'doubleword'): + for recipe in ('databricks', 'lepton', 'orcarouter', 'anannas-ai', 'doubleword', 'perplexity'): root = Path('python/examples') / recipe if not root.exists(): continue # Recipes may land independently. diff --git a/python/examples/perplexity/README.md b/python/examples/perplexity/README.md new file mode 100644 index 000000000..889049d73 --- /dev/null +++ b/python/examples/perplexity/README.md @@ -0,0 +1,169 @@ +# Perplexity Agent API (OpenAI SDK, Responses) with traceAI + +This recipe traces Perplexity's Agent API through the `openai` Python SDK's Responses interface at `https://api.perplexity.ai/v1`. It uses the existing `traceai-openai` instrumentor without a provider-specific package. + +Search results are not traced as citations or retriever spans; they can appear inside raw `output.value` on non-streamed calls and on streams without answer text. + +## Install + +Install the pinned dependencies: + +```bash +pip install -r requirements.txt +``` + +Or install the packages directly: + +```bash +pip install traceAI-openai fi-instrumentation-otel openai +``` + +## Configure + +| Environment variable | Required | Purpose | +|---|---|---| +| `FI_API_KEY` | Yes | Future AGI API key. Goes to the tracer only. | +| `FI_SECRET_KEY` | Yes | Future AGI secret key. Goes to the tracer only. | +| `PERPLEXITY_API_KEY` | Yes | Perplexity key. Goes to the OpenAI client only. | +| `PERPLEXITY_MODEL` | Yes, unless using `--model` | An ID from the [Agent API models page](https://docs.perplexity.ai/docs/agent-api/models). This example uses `openai/gpt-6-luna` ($0.10/M input, $0.50/M output for small contexts, checked 2026-10-28). | +| `FI_PROJECT_NAME` | No | Future AGI project; defaults to `perplexity-agent-api`. | +| `PERPLEXITY_BASE_URL` | No | SDK base URL override. Defaults to `https://api.perplexity.ai/v1`. | +| `FI_BASE_URL` | No | Future AGI collector origin. Defaults to `https://api.futureagi.com`. | +| `FI_HIDE_INPUTS` | No | Set to `true` to hide prompt text in exported spans. | +| `FI_HIDE_OUTPUTS` | No | Set to `true` to hide answer text and raw response output in exported spans. | + +The SDK represents the documented base URL as `https://api.perplexity.ai/v1/`, with a trailing slash. Requests go to `https://api.perplexity.ai/v1/responses`. The app validates the URL without rewriting it and checks the model and Perplexity key before setting up tracing. + +## Run + +From this example directory, replace the placeholders with your keys: + +```bash +export FI_API_KEY="placeholder-fi-key" +export FI_SECRET_KEY="placeholder-fi-secret" +export PERPLEXITY_API_KEY="placeholder-perplexity-key" +export FI_PROJECT_NAME="${FI_PROJECT_NAME:-perplexity-agent-api}" +export PERPLEXITY_MODEL="openai/gpt-6-luna" +python src/app.py +python src/app.py --stream +``` + +Use `--prompt "What is a solar eclipse?"` to change the question. `--model` takes precedence over `PERPLEXITY_MODEL`. + +## Code + +Instrument before creating the client: + +```python +import os +from fi_instrumentation import register +from fi_instrumentation.fi_types import ProjectType +from openai import OpenAI +from traceai_openai import OpenAIInstrumentor + +provider = register(project_name=os.environ.get("FI_PROJECT_NAME") or "perplexity-agent-api", + project_type=ProjectType.OBSERVE, + set_global_tracer_provider=False, verbose=False) +OpenAIInstrumentor().instrument(tracer_provider=provider) +client = OpenAI(api_key=os.environ["PERPLEXITY_API_KEY"], + base_url="https://api.perplexity.ai/v1") +response = client.responses.create(model=os.environ["PERPLEXITY_MODEL"], + input="What is a solar eclipse?") +print(response.output_text) +provider.force_flush() +``` + +The full runnable version is [src/app.py](src/app.py). Streaming prints `response.output_text.delta` events and consumes the stream through completion so the instrumentor can finish the span. + +## What you see in Future AGI + +The fixtures produce one span named `Response` per call, with `gen_ai.span.kind=LLM`. In current `traceai-openai` behavior, `gen_ai.request.model` on successful non-streamed and streamed calls records the model returned by the response. Streams use the model in `response.completed`. Failed calls record the model you sent. The requested model remains in `gen_ai.request.parameters` on all three paths. The fixtures request `openai/gpt-5.6-sol` and return `openai/gpt-5.6-sol-2026-09-01`, a synthetic dated id used only for testing. + +Up to `traceai-openai` 0.1.10, the provider field is `openai` (`gen_ai.provider.name`), the shared instrumentor's label for calls made through the OpenAI SDK. Later versions (future-agi/traceAI#249) say `perplexity` for `api.perplexity.ai`; other base URLs keep `openai`. + +Token counts appear in `gen_ai.usage.input_tokens`, `gen_ai.usage.output_tokens` and `gen_ai.usage.total_tokens` when the response includes usage. `gen_ai.usage.output_tokens.reasoning` appears when `output_tokens_details.reasoning_tokens` is supplied. `gen_ai.usage.input_tokens.cache_read` is read from `input_tokens_details.cached_tokens`, and the shared wrapper currently requires `output_tokens_details` to be present as well. Missing usage is omitted, never filled with zero. For streaming, the instrumentor takes the answer and token counts from the full response in `response.completed`. The fixture exports 17 input tokens, 11 output tokens, 28 total tokens, 9 cache-read tokens and 4 reasoning tokens on both successful paths. HTTP 401 produces an ERROR span with an exception event. + +Search-result URLs and snippets appear inside `output.value` on non-streamed calls. Streamed `output.value` contains answer text when present; a completed response with no answer text falls back to the raw response and may include search data and instructions. Use `FI_HIDE_OUTPUTS=true` on both paths for output privacy. The non-streamed value is the shared instrumentor's raw response representation (Python repr text up to 0.1.10, JSON in later versions), which includes the `search_results` item and the response's `instructions` field. Search queries and instructions appear in non-streamed `output.value`; their markers are absent from all streamed span attributes in the fixture. Search results can therefore reach Future AGI inside raw output. There are no citation attributes or retriever spans. The tests verify that `FI_HIDE_OUTPUTS=true` removes the answer text, search queries, search-result URLs and snippets, and instructions from the exported span, with an unmasked control. The nested `usage.cost` object round-trips inside raw non-streamed output; it has no dedicated cost mapping. + +With OpenAI 3.24.0, the extra `search_results` item parses and `response.output_text` returns the message text. Serializing that item emits Pydantic warnings; it does not prevent the fixture span from exporting. + +## Provider specifics + +Perplexity documents the [Agent API through the OpenAI SDK's Responses interface](https://docs.perplexity.ai/docs/agent-api/openai-compatibility). The canonical Agent API endpoint is `/v1/agent`; `/v1/responses` is its OpenAI SDK alias. `client.responses.create` uses that alias automatically. Pass the base URL `https://api.perplexity.ai/v1`, rather than an endpoint path. + +Perplexity also documents presets through the OpenAI SDK with `extra_body={"preset": ...}`; this recipe's CLI passes a model ID rather than selecting a preset. In one MockTransport preset fixture with no request model, one span exported with the response model, while `preset` was absent from `gen_ai.request.parameters`; this does not establish preset support. With older SDKs such as `openai` 1.69.0, `responses.create` requires `model`, so a preset-only call raises `TypeError` before any request is sent. + +Sonar Chat Completions support ended on 27 September 2026. Synchronous and streaming requests keep working because they are being reformulated as Agent API requests, rolling out by model; asynchronous Sonar requests are no longer supported. New code should use the Agent API. See the [Sonar migration overview](https://docs.perplexity.ai/docs/agent-api/migrate-from-sonar/overview). + +| Refused SDK base URL | Reason and replacement | +|---|---| +| `https://api.perplexity.ai` or its root `/` | Former Sonar Chat Completions base URL. Use `https://api.perplexity.ai/v1` with `client.responses.create`. | +| `https://api.perplexity.ai/v1/sonar` | Endpoint path. The SDK appends `/responses`; use the documented base URL. | +| `https://api.perplexity.ai/v1/agent` | Canonical endpoint path. Use the documented SDK base URL for the `/v1/responses` alias. | +| `https://api.perplexity.ai/v1/responses` | Endpoint path. The SDK would post to `/v1/responses/responses`; use the documented SDK base URL. | +| `https://api.perplexity.ai/chat/completions` or `/v1/chat/completions` | Old endpoint paths. Use the documented SDK base URL with `client.responses.create`. | +| `https://api.perplexity.ai/router/v1` or `/router` | The Router API is a separate private-preview product, outside this recipe. | +| `http://api.perplexity.ai/v1` | Cleartext would expose the key. Use HTTPS. | +| Perplexity URLs with a query or fragment | Remove the query or fragment from `PERPLEXITY_BASE_URL`. | +| Any URL with embedded credentials | Remove userinfo; pass the Perplexity key to the client. | + +Host checks also catch upper-case and trailing-dot spellings. The Perplexity host allows exactly `/v1` or `/v1/`; repeated slashes, encoded separators and other paths are refused. Whitespace, control characters, Unicode or invalid IDNA hosts, malformed ports, host backslashes, percent-encoding, empty labels, and URL credentials are refused before tracing on every host. For internationalised proxy hosts, use punycode. Valid proxy and loopback URLs retain their spelling and paths; no URL is rewritten. + +## Privacy + +`FI_HIDE_INPUTS=true` masks input attributes. `FI_HIDE_OUTPUTS=true` keeps answer text and the raw response, including search-result URLs and snippets, out of the exported span. The tests check both against visible controls. You can also pass `TraceConfig(hide_inputs=True)` to the instrumentor. + +On non-streamed calls, `output.value` also includes the search queries Perplexity ran, which are derived from your prompt, and the response's `instructions` field. `FI_HIDE_INPUTS` does not remove them, so set `FI_HIDE_OUTPUTS=true` as well to keep them out. The instructions can contain Perplexity's provider system prompt. Streamed fixture spans include neither the queries nor instructions. + +Perplexity still receives the prompt. Future AGI masking does not change Perplexity's own logging. + +## Limits / not covered + +- Sonar Chat Completions through `chat.completions.create`. +- The native `perplexity` SDK (`from perplexity import Perplexity`) and its presets. +- Preset requests (`extra_body={"preset": ...}`). +- The Search API, Router API (private preview) and Embeddings API. +- Async or background runs. +- Citation attributes or retriever spans. +- `usage.cost` mapping. + +The fixtures include Perplexity's `search_results` item first, followed by the assistant message. They check SDK parsing and `response.output_text`. They do not establish compatibility with other response shapes or models. + +## Tests + +Run from the repository root. The subprocess tests use a socket guard that refuses external DNS and connections before networking. A ready marker, empty log on successful loopback calls and deliberate refused attempts verify the guard. + +Repository source: + +```bash +env -u PYTHONPATH PYTHONDONTWRITEBYTECODE=1 \ + PYTHONPATH="python/examples/perplexity/src:python:python/frameworks/openai:python/tests" \ + uv run --no-project --python 3.11 \ + --with 'openai==3.24.0' --with httpx --with 'wrapt<2' \ + --with opentelemetry-api --with opentelemetry-sdk --with opentelemetry-instrumentation \ + --with opentelemetry-exporter-otlp-proto-http --with requests --with protobuf \ + --with opentelemetry-proto --with pydantic --with jsonschema --with pytest \ + pytest python/examples/perplexity/tests -q -p no:cacheprovider --noconftest -o addopts= -rfEs +``` + +Published packages: + +```bash +env -u PYTHONPATH PYTHONDONTWRITEBYTECODE=1 \ + PYTHONPATH="python/examples/perplexity/src:python/tests" \ + uv run --no-project --python 3.11 \ + --with 'openai==3.24.0' --with httpx --with 'wrapt<2' \ + --with 'traceAI-openai==0.1.10' --with 'fi-instrumentation-otel==1.1.0' \ + --with opentelemetry-api --with opentelemetry-sdk --with opentelemetry-instrumentation \ + --with opentelemetry-exporter-otlp-proto-http --with requests --with protobuf \ + --with opentelemetry-proto --with pydantic --with jsonschema --with pytest \ + pytest python/examples/perplexity/tests -q -p no:cacheprovider --noconftest -o addopts= -rfEs +``` + +For the floor row, change `openai==3.24.0` to `openai==1.69.0` in the repository-source command. + +| Python | `openai` | traceAI packages | Result | +|---|---|---|---| +| 3.10, 3.11, 3.12, 3.13 | 3.24.0 | `traceai-openai` and `fi_instrumentation` from this repository | Full suite | +| 3.11 | 1.69.0 (the `traceai-openai` floor) | from this repository | Full suite | +| 3.11 | 3.24.0 | published `traceAI-openai==0.1.10` and `fi-instrumentation-otel==1.1.0` from PyPI | Full suite | diff --git a/python/examples/perplexity/requirements.txt b/python/examples/perplexity/requirements.txt new file mode 100644 index 000000000..c3d606746 --- /dev/null +++ b/python/examples/perplexity/requirements.txt @@ -0,0 +1,4 @@ +# Pinned dependencies; use openai==1.69.0 for the SDK floor matrix. +openai==3.24.0 +traceAI-openai==0.1.10 +fi-instrumentation-otel==1.1.0 diff --git a/python/examples/perplexity/src/app.py b/python/examples/perplexity/src/app.py new file mode 100644 index 000000000..8cd556d15 --- /dev/null +++ b/python/examples/perplexity/src/app.py @@ -0,0 +1,140 @@ +"""Trace Perplexity's Agent API through the OpenAI Responses interface.""" + +import argparse +import os +import sys +from urllib.parse import urlsplit + +from fi_instrumentation import register +from fi_instrumentation.fi_types import ProjectType +from openai import OpenAI +from traceai_openai import OpenAIInstrumentor + +DEFAULT_BASE_URL = "https://api.perplexity.ai/v1" +API_KEY_ENV = "PERPLEXITY_API_KEY" +BASE_URL_ENV = "PERPLEXITY_BASE_URL" +MODEL_ENV = "PERPLEXITY_MODEL" + + +def check_base_url(url: str) -> str: + """Reject documented endpoint mix-ups without changing an allowed URL.""" + if any(char.isspace() or not char.isprintable() for char in url): + raise ValueError(f"{BASE_URL_ENV}: remove whitespace or control characters.") + try: + parsed = urlsplit(url) + host = parsed.hostname + parsed.port # Reject malformed ports before tracing. + except ValueError: + raise ValueError(f"{BASE_URL_ENV}: use a valid SDK base URL.") from None + if parsed.username is not None or parsed.password is not None: + raise ValueError(f"{BASE_URL_ENV}: remove URL credentials; pass the key to the client.") + if not host or parsed.scheme not in ("http", "https"): + raise ValueError(f"{BASE_URL_ENV}: use an absolute HTTP(S) SDK base URL.") + if "\\" in host or "%" in host or "" in (host[:-1] if host.endswith(".") else host).split("."): + raise ValueError(f"{BASE_URL_ENV}: invalid host spelling.") + try: + host.lower().encode("ascii").decode("idna") + except UnicodeError: + raise ValueError(f"{BASE_URL_ENV}: use a plain ASCII host with valid IDNA.") from None + if host.lower().rstrip(".") == "api.perplexity.ai": + if parsed.scheme != "https": + raise ValueError( + f"{BASE_URL_ENV}: use https://api.perplexity.ai/v1 with " + "client.responses.create; HTTP would send the key in cleartext." + ) + if "?" in url or "#" in url: + raise ValueError( + f"{BASE_URL_ENV}: remove the query or fragment; use " + "https://api.perplexity.ai/v1 with client.responses.create." + ) + path = parsed.path.removesuffix("/") + if not path: + raise ValueError( + f"{BASE_URL_ENV}: that root was the Sonar Chat Completions base URL, whose support " + "ended on 27 September 2026; use https://api.perplexity.ai/v1 " + "with client.responses.create." + ) + if path in ( + "/v1/sonar", "/v1/agent", "/v1/responses", + "/chat/completions", "/v1/chat/completions", + ): + raise ValueError( + f"{BASE_URL_ENV}: Endpoint paths are not SDK base URLs: the SDK appends /responses; " + "use https://api.perplexity.ai/v1 with client.responses.create." + ) + if path in ("/router", "/router/v1"): + raise ValueError( + f"{BASE_URL_ENV}: the Router API is a separate private-preview product, not covered " + "here; use https://api.perplexity.ai/v1 with client.responses.create." + ) + if parsed.path not in ("/v1", "/v1/"): + raise ValueError( + f"{BASE_URL_ENV}: use exactly https://api.perplexity.ai/v1 or /v1/ " + "with client.responses.create; other base paths are refused." + ) + return url + + +def make_client( + base_url: str | None = None, api_key: str | None = None, http_client=None +) -> OpenAI: + if base_url is None: + base_url = os.environ.get(BASE_URL_ENV) or DEFAULT_BASE_URL + return OpenAI( + base_url=check_base_url(base_url), + api_key=os.environ[API_KEY_ENV] if api_key is None else api_key, + http_client=http_client, + ) + + +def setup_tracing(project_name: str | None = None): + provider = register( + project_name=project_name or os.environ.get("FI_PROJECT_NAME") or "perplexity-agent-api", + project_type=ProjectType.OBSERVE, + set_global_tracer_provider=False, + verbose=False, + ) + OpenAIInstrumentor().instrument(tracer_provider=provider) + return provider + + +def main(argv=None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--stream", action="store_true") + parser.add_argument("--prompt", default="What is a solar eclipse?") + parser.add_argument("--model") + args = parser.parse_args(argv) + try: + base_url = check_base_url(os.environ.get(BASE_URL_ENV) or DEFAULT_BASE_URL) + model = args.model or os.environ[MODEL_ENV] + api_key = os.environ[API_KEY_ENV] + if not model: + raise ValueError(f"Set {MODEL_ENV} or pass --model.") + if not api_key: + raise ValueError(f"Set {API_KEY_ENV}.") + except KeyError as error: + print(f"Set {error.args[0]} before running this example.", file=sys.stderr) + return 2 + except ValueError as error: + print(str(error), file=sys.stderr) + return 2 + + provider = setup_tracing() + try: + with make_client(base_url=base_url, api_key=api_key) as client: + if args.stream: + stream = client.responses.create(model=model, input=args.prompt, stream=True) + for event in stream: + if event.type == "response.output_text.delta": + print(event.delta, end="", flush=True) + print() + else: + response = client.responses.create(model=model, input=args.prompt) + print(response.output_text) + finally: + provider.force_flush() + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/python/examples/perplexity/tests/_fake_openai.py b/python/examples/perplexity/tests/_fake_openai.py new file mode 100644 index 000000000..e648c617c --- /dev/null +++ b/python/examples/perplexity/tests/_fake_openai.py @@ -0,0 +1,169 @@ +"""Local Responses fixtures, including Perplexity's search_results output item.""" + +import copy +import json +import threading +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +REQUEST_MODEL = "openai/gpt-5.6-sol" +RESPONSE_MODEL = "openai/gpt-5.6-sol-2026-09-01" # Synthetic dated id for fixtures only. +ANSWER = "A solar eclipse happens when the Moon blocks the Sun." +DELTAS = ("A solar eclipse ", "happens when the Moon ", "blocks the Sun.") +SEARCH_URL = "https://example.invalid/fixture-search-result" +SEARCH_SNIPPET = "fixture-search-snippet-marker" +QUERY_MARKER = "fixture-prompt-derived-query-marker" +INSTRUCTIONS_MARKER = "fixture-provider-instructions-marker" +USAGE = { + "input_tokens": 17, + "output_tokens": 11, + "total_tokens": 28, + "cost": { + "currency": "USD", + "input_cost": 0.00017, + "output_cost": 0.00022, + "total_cost": 0.00248, + "cache_creation_cost": None, + "cache_read_cost": 0.00009, + "tool_calls_cost": 0.002, + }, + "input_tokens_details": { + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 9, + "cached_tokens": 9, + }, + "tool_calls_details": {"search_web": {"invocation": 1}}, + "output_tokens_details": {"reasoning_tokens": 4}, +} +ERROR = {"error": {"message": "Invalid API key", "type": "invalid_request_error", "code": 401}} + + +def response_fixture(model=RESPONSE_MODEL, include_usage=True, include_usage_details=True): + response = { + "id": "resp_placeholder", + "created_at": 1790812800, + "object": "response", + "model": model, + "status": "completed", + "output": [ + { + "type": "search_results", + "results": [{ + "id": 1, + "title": "Fixture source", + "url": SEARCH_URL, + "snippet": SEARCH_SNIPPET, + "date": "2026-10-01", + "last_updated": "2026-10-02", + "source": "web", + }, { + "id": 2, + "title": "Second fixture source", + "url": SEARCH_URL + "-second", + "snippet": SEARCH_SNIPPET + "-second", + "date": None, + "last_updated": "2026-10-02", + "source": "web", + }], + "queries": [QUERY_MARKER + " solar eclipse", "fixture eclipse characteristics"], + }, + { + "id": "msg_placeholder", + "type": "message", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": ANSWER, "annotations": [], "logprobs": []}], + }, + ], + "error": None, + "background": False, + "completed_at": 1790812800, + "frequency_penalty": 0, + "incomplete_details": None, + "instructions": "Placeholder provider system prompt.\n" + INSTRUCTIONS_MARKER, + "max_output_tokens": 8192, + "max_tool_calls": None, + "metadata": {}, + "parallel_tool_calls": True, + } + if include_usage: + response["usage"] = copy.deepcopy(USAGE) + if not include_usage_details: + for key in ("input_tokens_details", "output_tokens_details", "tool_calls_details"): + response["usage"].pop(key) + return response + + +def stream_fixture(model=RESPONSE_MODEL, include_usage=True, include_usage_details=True): + completed = response_fixture(model, include_usage, include_usage_details) + created = {**completed, "status": "in_progress", "output": [], "usage": None, "completed_at": None} + events = [{"type": "response.created", "sequence_number": 0, "response": created}] + for sequence, delta in enumerate(DELTAS, 1): + events.append({ + "type": "response.output_text.delta", + "sequence_number": sequence, + "output_index": 1, + "content_index": 0, + "item_id": "msg_placeholder", + "delta": delta, + }) + events.append({ + "type": "response.completed", + "sequence_number": len(DELTAS) + 1, + "response": completed, + }) + return ("".join( + f"event: {event['type']}\ndata: {json.dumps(event)}\n\n" for event in events + ) + "data: [DONE]\n\n").encode() + + +class FakeOpenAI: + """Serve fixture responses on 127.0.0.1 and retain received requests.""" + + def __init__(self): + self._requests = [] + self._lock = threading.Lock() + owner = self + + class Handler(BaseHTTPRequestHandler): + def do_POST(self): + body = json.loads(self.rfile.read(int(self.headers["Content-Length"]))) + with owner._lock: + owner._requests.append({ + "path": self.path, + "headers": {key.lower(): value for key, value in self.headers.items()}, + "body": body, + }) + if self.path != "/v1/responses": + self.send_error(404) + return + if body.get("input") == "fixture-error": + status, content_type, payload = 401, "application/json", json.dumps(ERROR).encode() + elif body.get("stream"): + status, content_type, payload = 200, "text/event-stream", stream_fixture() + else: + status, content_type, payload = 200, "application/json", json.dumps(response_fixture()).encode() + self.send_response(status) + self.send_header("Content-Type", content_type) + self.send_header("Content-Length", str(len(payload))) + self.end_headers() + self.wfile.write(payload) + + def log_message(self, _format, *args): + return + + self._server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + self.base_url = f"http://127.0.0.1:{self._server.server_port}/v1" + self._thread = threading.Thread(target=self._server.serve_forever, daemon=True) + + def __enter__(self): + self._thread.start() + return self + + def __exit__(self, *_exc): + self._server.shutdown() + self._server.server_close() + self._thread.join() + + def requests(self): + with self._lock: + return copy.deepcopy(self._requests) diff --git a/python/examples/perplexity/tests/loopback_guard/sitecustomize.py b/python/examples/perplexity/tests/loopback_guard/sitecustomize.py new file mode 100644 index 000000000..5f7997c32 --- /dev/null +++ b/python/examples/perplexity/tests/loopback_guard/sitecustomize.py @@ -0,0 +1,49 @@ +"""Refuse non-loopback DNS and socket connections before networking occurs.""" + +import os +import socket +from pathlib import Path + +_ALLOWED = {"127.0.0.1", "localhost"} + + +def _check(host, operation): + if isinstance(host, bytes): + host = host.decode("ascii") + if str(host).lower() not in _ALLOWED: + log = os.environ.get("LOOPBACK_GUARD_LOG") + if log: + with open(log, "a", encoding="utf-8") as stream: + stream.write(f"{operation}: refused {host}\n") + raise RuntimeError(f"Loopback guard refused {operation} to {host}") + + +_getaddrinfo = socket.getaddrinfo +_connect = socket.socket.connect +_connect_ex = socket.socket.connect_ex + + +def _guarded_getaddrinfo(host, *args, **kwargs): + _check(host, "getaddrinfo") + return _getaddrinfo(host, *args, **kwargs) + + +def _guarded_connect(sock, address): + if sock.family in (socket.AF_INET, socket.AF_INET6): + _check(address[0], "connect") + return _connect(sock, address) + + +def _guarded_connect_ex(sock, address): + if sock.family in (socket.AF_INET, socket.AF_INET6): + _check(address[0], "connect_ex") + return _connect_ex(sock, address) + + +socket.getaddrinfo = _guarded_getaddrinfo +socket.socket.connect = _guarded_connect +socket.socket.connect_ex = _guarded_connect_ex +if os.environ.get("LOOPBACK_GUARD_LOG"): + Path(os.environ["LOOPBACK_GUARD_LOG"]).touch() +if os.environ.get("LOOPBACK_GUARD_READY"): + Path(os.environ["LOOPBACK_GUARD_READY"]).write_text("installed\n", encoding="utf-8") diff --git a/python/examples/perplexity/tests/test_perplexity_recipe.py b/python/examples/perplexity/tests/test_perplexity_recipe.py new file mode 100644 index 000000000..6f1e332f3 --- /dev/null +++ b/python/examples/perplexity/tests/test_perplexity_recipe.py @@ -0,0 +1,654 @@ +"""Perplexity Responses contracts using MockTransport and guarded loopback calls.""" + +import ast +from contextlib import contextmanager +import importlib.util +import json +import os +from pathlib import Path +import re +import subprocess +import sys +import uuid + +import fi_instrumentation +from harness import Receiver +import httpx +import openai +import pytest +import traceai_openai +from traceai_openai import OpenAIInstrumentor + +# traceai-openai 0.1.10 and earlier label every span "openai"; later versions +# (future-agi/traceAI#249) use the vendor's name for its documented API host. +# Accept both so this recipe passes on either version. +PROVIDER_NAMES = ("openai", "perplexity") + +TEST_DIR = Path(__file__).resolve().parent +RECIPE_DIR = TEST_DIR.parent +VENDOR_KEY = "placeholder-perplexity-key" +FI_KEY = "placeholder-fi-key" +FI_SECRET = "placeholder-fi-secret" + + +def _load_module(name, path): + spec = importlib.util.spec_from_file_location(name, path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +app = _load_module("perplexity_recipe_app", RECIPE_DIR / "src" / "app.py") +fake = _load_module("perplexity_recipe_fake", TEST_DIR / "_fake_openai.py") + + +@pytest.fixture(autouse=True) +def clean_environment(monkeypatch): + for name in tuple(os.environ): + if name.startswith(("FI_", "OPENAI_", "PERPLEXITY_")) or name.lower() in ( + "http_proxy", "https_proxy", "all_proxy", "no_proxy", + ): + monkeypatch.delenv(name) + monkeypatch.setenv("FI_API_KEY", FI_KEY) + monkeypatch.setenv("FI_SECRET_KEY", FI_SECRET) + monkeypatch.setenv(app.API_KEY_ENV, VENDOR_KEY) + monkeypatch.setenv(app.MODEL_ENV, fake.REQUEST_MODEL) + monkeypatch.setenv("NO_PROXY", "127.0.0.1,localhost") + monkeypatch.setenv("PYTHONDONTWRITEBYTECODE", "1") + + +@pytest.fixture +def tracing(monkeypatch): + @contextmanager + def start(): + project = f"perplexity-fixture-{uuid.uuid4().hex}" + with Receiver() as receiver: + monkeypatch.setenv("FI_BASE_URL", receiver.origin) + provider = app.setup_tracing(project_name=project) + try: + yield receiver, provider, project + finally: + provider.force_flush() + OpenAIInstrumentor().uninstrument() + provider.shutdown() + return start + + +@pytest.fixture(scope="module") +def validation_servers(): + """All rejected configurations must leave both loopback servers untouched.""" + with Receiver() as receiver, fake.FakeOpenAI() as server: + yield receiver, server + + +def _attributes(span): + return { + item["key"]: next(iter(item["value"].values())) + for item in span.get("attributes", []) + } + + +def _assert_export(receiver, provider, project, error=False, requested_model=fake.REQUEST_MODEL): + assert provider.force_flush() + [span] = receiver.spans() + [export] = receiver.requests() + attrs = _attributes(span) + assert span["name"] == "Response" + assert attrs["gen_ai.span.kind"] == "LLM" + assert attrs["gen_ai.provider.name"] in PROVIDER_NAMES + # A successful response overrides the request model, including on streams. + assert attrs["gen_ai.request.model"] == (requested_model if error else fake.RESPONSE_MODEL) + parameters = json.loads(attrs["gen_ai.request.parameters"]) + if requested_model is None: + assert "model" not in parameters + else: + assert parameters["model"] == requested_model + assert span["status"]["code"] == ("STATUS_CODE_ERROR" if error else "STATUS_CODE_OK") + assert export["path"] == "/tracer/v1/traces" + assert export["headers"]["x-api-key"] == FI_KEY + assert export["headers"]["x-secret-key"] == FI_SECRET + [resource] = export["resource_attributes"] + assert resource["project_name"] == project + assert resource["project_type"] == "observe" + # Positive controls: a real span and collector authentication were observed. + assert VENDOR_KEY not in json.dumps(span) + assert VENDOR_KEY not in json.dumps(export) + assert not any("citation" in key or "cost" in key for key in attrs) + return span, attrs + + +def _assert_vendor_request(request, streamed=False, prompt=None, requested_model=fake.REQUEST_MODEL): + assert str(request.url) == app.DEFAULT_BASE_URL + "/responses" + assert request.method == "POST" + assert request.headers["authorization"] == f"Bearer {VENDOR_KEY}" + assert "x-api-key" not in request.headers + assert "x-secret-key" not in request.headers + assert FI_KEY not in json.dumps(dict(request.headers)) + assert FI_SECRET not in json.dumps(dict(request.headers)) + body = json.loads(request.content) + if requested_model is None: + assert "model" not in body + else: + assert body["model"] == requested_model + assert body.get("stream", False) is streamed + if prompt is not None: + assert body["input"] == prompt + + +def _assert_usage(attrs, present=True, details=True): + usage = {key: int(value) for key, value in attrs.items() if key.startswith("gen_ai.usage.")} + expected = { + "gen_ai.usage.input_tokens": 17, + "gen_ai.usage.output_tokens": 11, + "gen_ai.usage.total_tokens": 28, + } if present else {} + if present and details: + expected.update({ + "gen_ai.usage.output_tokens.reasoning": 4, + "gen_ai.usage.input_tokens.cache_read": 9, + }) + assert usage == expected + + +def _call(handler, stream=False, prompt="What is a solar eclipse?"): + with app.make_client(http_client=httpx.Client(transport=httpx.MockTransport(handler))) as client: + if stream: + events = list(client.responses.create(model=fake.REQUEST_MODEL, input=prompt, stream=True)) + assert [event.type for event in events] == [ + "response.created", *["response.output_text.delta"] * len(fake.DELTAS), + "response.completed", + ] + assert events[-1].response.output_text == fake.ANSWER + assert events[-1].response.output[0].type == "search_results" + assert events[-1].response.model == fake.RESPONSE_MODEL + assert fake.INSTRUCTIONS_MARKER in events[-1].response.instructions + assert fake.QUERY_MARKER in json.dumps(events[-1].response.output[0].queries) + return "".join(event.delta for event in events if event.type == "response.output_text.delta") + response = client.responses.create(model=fake.REQUEST_MODEL, input=prompt) + assert response.output[0].type == "search_results" + assert response.output[1].type == "message" + assert response.model == fake.RESPONSE_MODEL + return response.output_text + + +def _handler(request, stream=False, include_usage=True, include_usage_details=True): + _assert_vendor_request(request, streamed=stream) + if stream: + return httpx.Response(200, headers={"Content-Type": "text/event-stream"}, + content=fake.stream_fixture( + include_usage=include_usage, include_usage_details=include_usage_details)) + return httpx.Response(200, json=fake.response_fixture( + include_usage=include_usage, include_usage_details=include_usage_details)) + + +def test_documented_base_url(): + assert app.DEFAULT_BASE_URL == "https://api.perplexity.ai/v1" + assert app.DEFAULT_BASE_URL in (RECIPE_DIR / "README.md").read_text() + with app.make_client() as client: + assert str(client.base_url) == "https://api.perplexity.ai/v1/" + + +def test_make_client_overrides_and_missing_key(monkeypatch): + monkeypatch.setenv(app.BASE_URL_ENV, "http://127.0.0.1:1234/v1") + with app.make_client() as client: + assert str(client.base_url) == "http://127.0.0.1:1234/v1/" + assert client.api_key == VENDOR_KEY + with app.make_client(base_url=app.DEFAULT_BASE_URL, api_key="placeholder-explicit-key") as client: + assert str(client.base_url) == app.DEFAULT_BASE_URL + "/" + assert client.api_key == "placeholder-explicit-key" + monkeypatch.delenv(app.API_KEY_ENV) + with pytest.raises(KeyError, match=app.API_KEY_ENV): + app.make_client() + + +def _raw_output(value): + # traceai-openai 0.1.10 writes the raw response as Python repr text; later versions + # (future-agi/traceAI#249) write JSON and drop None-valued fields. + try: + return json.loads(value) + except ValueError: + return ast.literal_eval(value) + + +def test_responses_at_documented_host(tracing): + assert fake.REQUEST_MODEL != fake.RESPONSE_MODEL + with tracing() as (receiver, provider, project): + with pytest.warns(UserWarning, match="search_results"): + assert _call(_handler) == fake.ANSWER + _, attrs = _assert_export(receiver, provider, project) + _assert_usage(attrs) + raw = _raw_output(attrs["output.value"]) + assert raw["output"][0]["type"] == "search_results" + assert raw["output"][0]["results"][0]["url"] == fake.SEARCH_URL + assert raw["output"][0]["results"][0]["snippet"] == fake.SEARCH_SNIPPET + assert raw["output"][0]["results"][0]["last_updated"] == "2026-10-02" + assert raw["output"][0]["results"][1].get("date") is None + assert fake.QUERY_MARKER in json.dumps(raw["output"][0]["queries"]) + assert raw["output"][1]["id"] == "msg_placeholder" + assert raw["output"][1]["content"][0]["text"] == fake.ANSWER + assert raw["usage"]["cost"] == fake.USAGE["cost"] + # SDK serialization adds optional defaults such as cache_write_tokens=None. + for key, value in fake.USAGE["input_tokens_details"].items(): + assert raw["usage"]["input_tokens_details"][key] == value + assert raw["usage"]["output_tokens_details"] == fake.USAGE["output_tokens_details"] + assert raw["usage"]["tool_calls_details"] == fake.USAGE["tool_calls_details"] + assert fake.INSTRUCTIONS_MARKER in raw["instructions"] + assert fake.SEARCH_URL in attrs["output.value"] + assert fake.SEARCH_SNIPPET in attrs["output.value"] + + +def test_preset_fixture(tracing): + prompt = "preset-fixture-input-marker" + requests = [] + + def handler(request): + requests.append(request) + _assert_vendor_request(request, prompt=prompt, requested_model=None) + assert json.loads(request.content)["preset"] == "low" + return httpx.Response(200, json=fake.response_fixture()) + + with tracing() as (receiver, provider, project): + with app.make_client(http_client=httpx.Client(transport=httpx.MockTransport(handler))) as client: + if openai.__version__ == "1.69.0": + # The pinned floor requires model; newer SDKs must exercise the request. + with pytest.raises(TypeError, match="model"): + client.responses.create(input=prompt, extra_body={"preset": "low"}) + assert requests == [] + assert provider.force_flush() + assert receiver.spans() == [] and receiver.requests() == [] + return + response = client.responses.create(input=prompt, extra_body={"preset": "low"}) + assert response.output_text == fake.ANSWER + assert response.model == fake.RESPONSE_MODEL + assert len(requests) == 1 + _, attrs = _assert_export(receiver, provider, project, requested_model=None) + _assert_usage(attrs) + assert json.loads(attrs["gen_ai.request.parameters"]) == {} + assert "preset" not in attrs["gen_ai.request.parameters"] + + +@pytest.mark.parametrize("stream", [False, True], ids=["nonstream", "stream"]) +def test_usage_absent(tracing, stream): + with tracing() as (receiver, provider, project): + assert _call(lambda request: _handler(request, stream, include_usage=False), stream) == fake.ANSWER + _, attrs = _assert_export(receiver, provider, project) + _assert_usage(attrs, present=False) + + +@pytest.mark.parametrize("stream", [False, True], ids=["nonstream", "stream"]) +def test_usage_without_details(tracing, stream): + with tracing() as (receiver, provider, project): + assert _call(lambda request: _handler(request, stream, include_usage_details=False), stream) == fake.ANSWER + _, attrs = _assert_export(receiver, provider, project) + _assert_usage(attrs, details=False) + + +def test_responses_stream(tracing): + with tracing() as (receiver, provider, project): + assert _call(lambda request: _handler(request, stream=True), stream=True) == fake.ANSWER + _, attrs = _assert_export(receiver, provider, project) + assert attrs["output.value"] == fake.ANSWER + _assert_usage(attrs) + # The non-streamed test proves these markers exist in the provider fixture. + assert fake.SEARCH_URL not in json.dumps(attrs) + assert fake.SEARCH_SNIPPET not in json.dumps(attrs) + assert fake.QUERY_MARKER not in json.dumps(attrs) + assert fake.INSTRUCTIONS_MARKER not in json.dumps(attrs) + assert json.loads(attrs["gen_ai.request.parameters"])["stream"] is True + + +def test_authentication_error(tracing): + def handler(request): + _assert_vendor_request(request) + return httpx.Response(401, json=fake.ERROR) + + with tracing() as (receiver, provider, project): + with pytest.raises(openai.AuthenticationError, match="Invalid API key"): + _call(handler) + span, attrs = _assert_export(receiver, provider, project, error=True) + assert "Invalid API key" in span["status"]["message"] + exceptions = [event for event in span["events"] if event["name"] == "exception"] + assert len(exceptions) == 1 + assert "Invalid API key" in json.dumps(exceptions) + _assert_usage(attrs, present=False) + assert "output.value" not in attrs + + +def test_hide_inputs_with_control(tracing, monkeypatch): + marker = "unique-perplexity-input-marker" + for hidden in (False, True): + monkeypatch.setenv("FI_HIDE_INPUTS", str(hidden).lower()) + + def handler(request): + _assert_vendor_request(request, prompt=marker) + return httpx.Response(200, json=fake.response_fixture()) + + with tracing() as (receiver, provider, project): + assert _call(handler, prompt=marker) == fake.ANSWER + span, _ = _assert_export(receiver, provider, project) + assert (marker in json.dumps(span)) is (not hidden) + # Output echoes remain visible even when the input itself is hidden. + for output_marker in (fake.QUERY_MARKER, fake.INSTRUCTIONS_MARKER): + assert output_marker in _attributes(span)["output.value"] + + +@pytest.mark.parametrize("stream", [False, True], ids=["nonstream", "stream"]) +def test_hide_outputs_with_control(tracing, monkeypatch, stream): + for hidden in (False, True): + monkeypatch.setenv("FI_HIDE_OUTPUTS", str(hidden).lower()) + with tracing() as (receiver, provider, project): + assert _call(lambda request: _handler(request, stream), stream) == fake.ANSWER + span, attrs = _assert_export(receiver, provider, project) + _assert_usage(attrs) + assert (fake.ANSWER in json.dumps(span)) is (not hidden) + for marker in (fake.SEARCH_URL, fake.SEARCH_SNIPPET, fake.QUERY_MARKER, fake.INSTRUCTIONS_MARKER): + assert (marker in json.dumps(span)) is (not hidden and not stream) + + +@pytest.mark.parametrize("hidden", [False, True], ids=["visible", "hidden"]) +def test_empty_text_stream_raw_fallback(tracing, monkeypatch, hidden): + monkeypatch.setenv("FI_HIDE_OUTPUTS", str(hidden).lower()) + completed = fake.response_fixture() + completed["output"] = completed["output"][:1] # Search data, but no assistant text. + event = {"type": "response.completed", "sequence_number": 0, "response": completed} + payload = f"event: response.completed\ndata: {json.dumps(event)}\n\ndata: [DONE]\n\n".encode() + with tracing() as (receiver, provider, project): + with app.make_client(http_client=httpx.Client(transport=httpx.MockTransport( + lambda request: httpx.Response(200, headers={"Content-Type": "text/event-stream"}, content=payload) + ))) as client: + list(client.responses.create(model=fake.REQUEST_MODEL, input="empty stream", stream=True)) + span, attrs = _assert_export(receiver, provider, project) + _assert_usage(attrs) + for marker in (fake.SEARCH_URL, fake.QUERY_MARKER, fake.INSTRUCTIONS_MARKER): + assert (marker in json.dumps(span)) is (not hidden) + + +def test_cache_details_require_output_details(tracing): + response = fake.response_fixture() + response["usage"].pop("output_tokens_details") + with tracing() as (receiver, provider, project): + assert _call(lambda request: httpx.Response(200, json=response)) == fake.ANSWER + _, attrs = _assert_export(receiver, provider, project) + _assert_usage(attrs, details=False) + + +def _child_environment(receiver, fake_server, tmp_path): + environment = os.environ.copy() + # These follow the actual imports, so published wheels work too. + roots = [ + TEST_DIR / "loopback_guard", RECIPE_DIR / "src", + Path(fi_instrumentation.__file__).resolve().parent.parent, + Path(traceai_openai.__file__).resolve().parent.parent, + Path(sys.modules[Receiver.__module__].__file__).resolve().parent.parent, + ] + environment.update({ + "FI_PROJECT_NAME": "perplexity-fixture-project-override", + "PYTHONPATH": os.pathsep.join(dict.fromkeys(str(path) for path in roots)), + "PYTHONDONTWRITEBYTECODE": "1", + "FI_BASE_URL": receiver.origin, + "FI_API_KEY": FI_KEY, + "FI_SECRET_KEY": FI_SECRET, + app.API_KEY_ENV: VENDOR_KEY, + app.BASE_URL_ENV: fake_server.base_url, + app.MODEL_ENV: fake.REQUEST_MODEL, + "LOOPBACK_GUARD_LOG": str(tmp_path / "guard.log"), + "LOOPBACK_GUARD_READY": str(tmp_path / "guard.ready"), + "NO_PROXY": "127.0.0.1,localhost", + }) + return environment + + +@pytest.mark.parametrize("stream", [False, True], ids=["nonstream", "stream"]) +@pytest.mark.parametrize("cli_model", [False, True], ids=["env-model", "cli-model"]) +def test_subprocess_app(tmp_path, stream, cli_model): + prompt = "subprocess-perplexity-prompt-marker" + with Receiver() as receiver, fake.FakeOpenAI() as server: + environment = _child_environment(receiver, server, tmp_path) + argv = [sys.executable, str(RECIPE_DIR / "src" / "app.py"), "--prompt", prompt] + if stream: + argv.append("--stream") + if cli_model: + environment.pop(app.MODEL_ENV) + argv.extend(["--model", fake.REQUEST_MODEL]) + result = subprocess.run(argv, env=environment, capture_output=True, text=True, timeout=30) + assert result.returncode == 0, result.stderr + assert result.stdout.strip() == fake.ANSWER + assert (tmp_path / "guard.ready").read_text() == "installed\n" + assert (tmp_path / "guard.log").read_text() == "" + [request] = server.requests() + assert request["path"] == "/v1/responses" + assert request["headers"]["authorization"] == f"Bearer {VENDOR_KEY}" + assert "x-api-key" not in request["headers"] + assert "x-secret-key" not in request["headers"] + assert request["body"]["input"] == prompt + assert request["body"]["model"] == fake.REQUEST_MODEL + assert request["body"].get("stream", False) is stream + [span] = receiver.spans() + attrs = _attributes(span) + assert span["name"] == "Response" + assert attrs["gen_ai.span.kind"] == "LLM" + assert attrs["gen_ai.provider.name"] in PROVIDER_NAMES + assert attrs["gen_ai.request.model"] == fake.RESPONSE_MODEL + assert json.loads(attrs["gen_ai.request.parameters"])["model"] == fake.REQUEST_MODEL + _assert_usage(attrs) + assert fake.ANSWER in attrs["output.value"] + assert (fake.SEARCH_URL in json.dumps(attrs)) is (not stream) + assert (fake.SEARCH_SNIPPET in json.dumps(attrs)) is (not stream) + assert (fake.QUERY_MARKER in json.dumps(attrs)) is (not stream) + assert (fake.INSTRUCTIONS_MARKER in json.dumps(attrs)) is (not stream) + [export] = receiver.requests() + assert export["path"] == "/tracer/v1/traces" + assert export["headers"]["x-api-key"] == FI_KEY + assert export["headers"]["x-secret-key"] == FI_SECRET + assert export["resource_attributes"][0]["project_name"] == "perplexity-fixture-project-override" + assert VENDOR_KEY not in json.dumps(span) + assert VENDOR_KEY not in json.dumps(export) + + +@pytest.mark.parametrize("operation", ["getaddrinfo", "connect", "connect_ex"]) +def test_loopback_guard_refuses_external_hosts(tmp_path, operation): + with Receiver() as receiver, fake.FakeOpenAI() as server: + environment = _child_environment(receiver, server, tmp_path) + script = """ +import socket +import sys +import sitecustomize +assert socket.getaddrinfo('127.0.0.1', 80) +def underlying_call(*args, **kwargs): + raise AssertionError('Underlying network operation was reached') +operation = sys.argv[1] +setattr(sitecustomize, '_' + operation, underlying_call) +try: + if operation == 'getaddrinfo': + socket.getaddrinfo('api.perplexity.ai', 443) + else: + with socket.socket() as connection: + getattr(connection, operation)(('api.perplexity.ai', 443)) +except RuntimeError as error: + assert 'Loopback guard refused' in str(error) + print('refused before networking') +else: + raise AssertionError('Guard did not refuse the provider host') +""" + result = subprocess.run([sys.executable, "-c", script, operation], env=environment, + capture_output=True, text=True, timeout=15) + assert result.returncode == 0, result.stderr + assert result.stdout.strip() == "refused before networking" + assert (tmp_path / "guard.ready").read_text() == "installed\n" + assert (tmp_path / "guard.log").read_text() == f"{operation}: refused api.perplexity.ai\n" + assert server.requests() == [] + assert receiver.spans() == [] + + +REFUSED_PATHS = [ + ("", "Sonar Chat Completions"), ("/", "Sonar Chat Completions"), + ("/v1/sonar", "Endpoint paths"), ("/v1/sonar/", "Endpoint paths"), + ("/v1/agent", "Endpoint paths"), ("/v1/agent/", "Endpoint paths"), + ("/v1/responses", "Endpoint paths"), ("/v1/responses/", "Endpoint paths"), + ("/chat/completions", "Endpoint paths"), ("/chat/completions/", "Endpoint paths"), + ("/v1/chat/completions", "Endpoint paths"), ("/v1/chat/completions/", "Endpoint paths"), + ("/router/v1", "private-preview"), ("/router/v1/", "private-preview"), + ("/router", "private-preview"), ("/router/", "private-preview"), +] + + +@pytest.mark.parametrize("host", ["api.perplexity.ai", "api.perplexity.ai.", "API.PERPLEXITY.AI"]) +@pytest.mark.parametrize("path,reason", REFUSED_PATHS) +def test_check_base_url_refusals(host, path, reason): + url = f"https://{host}{path}" + with pytest.raises(ValueError, match=reason) as raised: + app.check_base_url(url) + message = str(raised.value) + assert app.BASE_URL_ENV in message + assert "\n" not in message + assert "https://api.perplexity.ai/v1" in message + assert "client.responses.create" in message + if reason == "Sonar Chat Completions": + assert "27 September 2026" in message + + +@pytest.mark.parametrize("url", [ + "https://api.perplexity.ai/v1", "https://api.perplexity.ai/v1/", + "https://API.PERPLEXITY.AI./v1/", "http://127.0.0.1:1234/v1", + "http://localhost:1234/v1/", +]) +def test_check_base_url_allowed(url): + assert app.check_base_url(url) == url + + +@pytest.mark.parametrize("path,reason", REFUSED_PATHS) +@pytest.mark.parametrize("host", ["api.perplexity.ai", "api.perplexity.ai.", "API.PERPLEXITY.AI"]) +def test_main_refuses_urls_before_tracing(monkeypatch, capsys, validation_servers, host, path, reason): + _assert_main_refusal(f"https://{host}{path}", reason, monkeypatch, capsys, validation_servers) + + +def _assert_main_refusal(url, reason, monkeypatch, capsys, validation_servers): + receiver, server = validation_servers + monkeypatch.setenv("FI_BASE_URL", receiver.origin) + monkeypatch.setenv(app.BASE_URL_ENV, url) + monkeypatch.setattr(app, "setup_tracing", lambda *args, **kwargs: pytest.fail("Tracing started")) + assert app.main([]) == 2 + captured = capsys.readouterr() + assert reason in captured.err + assert app.BASE_URL_ENV in captured.err + assert captured.err.count("\n") == 1 + assert captured.out == "" + assert "placeholder-user" not in captured.err + assert "placeholder-password" not in captured.err + assert server.requests() == [] + assert receiver.spans() == [] + assert receiver.requests() == [] + + +@pytest.mark.parametrize("host", ["api.perplexity.ai", "api.perplexity.ai.", "API.PERPLEXITY.AI"]) +@pytest.mark.parametrize("url_template,reason", [ + ("http://{host}/v1", "cleartext"), + ("https://{host}/v1?fixture=query", "query or fragment"), + ("https://{host}/v1?", "query or fragment"), + ("https://{host}/v1#fixture", "query or fragment"), + ("https://{host}/v1#", "query or fragment"), + ("https://placeholder-user:placeholder-password@{host}/v1", "URL credentials"), + ("https://placeholder-user@{host}/v1", "URL credentials"), + ("https://@{host}/v1", "URL credentials"), +]) +def test_base_url_security_refusals(monkeypatch, capsys, validation_servers, host, url_template, reason): + url = url_template.format(host=host) + with pytest.raises(ValueError, match=reason) as raised: + app.check_base_url(url) + assert app.BASE_URL_ENV in str(raised.value) + assert "\n" not in str(raised.value) + _assert_main_refusal(url, reason, monkeypatch, capsys, validation_servers) + + +@pytest.mark.parametrize("url", [ + "http://placeholder-user:placeholder-password@127.0.0.1:1234/v1", + "http://placeholder-user@localhost:1234/v1", + "https://placeholder-user:placeholder-password@example.invalid/v1", +]) +def test_url_credentials_refused_on_any_host(monkeypatch, capsys, validation_servers, url): + with pytest.raises(ValueError, match="URL credentials"): + app.check_base_url(url) + _assert_main_refusal(url, "URL credentials", monkeypatch, capsys, validation_servers) + + +@pytest.mark.parametrize("variable", [app.MODEL_ENV, app.API_KEY_ENV]) +@pytest.mark.parametrize("empty", [False, True], ids=["missing", "empty"]) +def test_main_requires_configuration_before_tracing(monkeypatch, capsys, variable, empty): + with Receiver() as receiver, fake.FakeOpenAI() as server: + monkeypatch.setenv("FI_BASE_URL", receiver.origin) + monkeypatch.setenv(app.BASE_URL_ENV, server.base_url) + if empty: + monkeypatch.setenv(variable, "") + else: + monkeypatch.delenv(variable) + monkeypatch.setattr(app, "setup_tracing", lambda *args, **kwargs: pytest.fail("Tracing started")) + assert app.main([]) == 2 + captured = capsys.readouterr() + # A readable instruction, not a bare KeyError repr such as 'PERPLEXITY_MODEL'. + assert f"Set {variable}" in captured.err + assert captured.out == "" + assert server.requests() == [] + assert receiver.spans() == [] + assert receiver.requests() == [] + + +def test_readme_pins(): + readme = (RECIPE_DIR / "README.md").read_text() + assert readme.startswith("# Perplexity Agent API (OpenAI SDK, Responses) with traceAI\n") + assert app.DEFAULT_BASE_URL in readme + assert "provider field is `openai`" in readme + assert "Sonar Chat Completions support ended on 27 September 2026" in readme + assert "https://docs.perplexity.ai/docs/agent-api/migrate-from-sonar/overview" in readme + assert "client.responses.create" in readme + assert "gen_ai.request.model" in readme + assert "on successful non-streamed and streamed calls records the model returned by the response" in readme + assert "Failed calls record the model you sent" in readme + assert "requested model remains in `gen_ai.request.parameters` on all three paths" in readme + assert "Search results are not traced as citations or retriever spans" in readme + assert "a completed response with no answer text falls back to the raw response" in readme + assert 'project_name=os.environ.get("FI_PROJECT_NAME") or "perplexity-agent-api"' in readme + assert 'export FI_PROJECT_NAME="${FI_PROJECT_NAME:-perplexity-agent-api}"' in readme + assert 'export PERPLEXITY_MODEL="openai/gpt-6-luna"' in readme + assert "https://docs.perplexity.ai/docs/agent-api/models" in readme + assert "gen_ai.usage.input_tokens.cache_read" in readme + assert "gen_ai.usage.output_tokens.reasoning" in readme + assert "input_tokens_details.cached_tokens" in readme + assert "requires `output_tokens_details` to be present as well" in readme + assert "The nested `usage.cost` object round-trips inside raw non-streamed output" in readme + assert 'presets through the OpenAI SDK with `extra_body={"preset": ...}`' in readme + assert "one span exported with the response model, while `preset` was absent from `gen_ai.request.parameters`" in readme + assert '- Preset requests (`extra_body={"preset": ...}`).' in readme + assert "With older SDKs such as `openai` 1.69.0, `responses.create` requires `model`" in readme + assert "`FI_HIDE_INPUTS` does not remove them, so set `FI_HIDE_OUTPUTS=true` as well" in readme + assert "Streamed fixture spans include neither the queries nor instructions" in readme + assert "Synchronous and streaming requests keep working because they are being reformulated as Agent API requests, rolling out by model; asynchronous Sonar requests are no longer supported." in readme + assert "set_global_tracer_provider=False, verbose=False" in readme + for variable in (app.API_KEY_ENV, app.BASE_URL_ENV, app.MODEL_ENV, "FI_API_KEY", + "FI_SECRET_KEY", "FI_BASE_URL", "FI_HIDE_INPUTS", "FI_HIDE_OUTPUTS"): + assert variable in readme + assert "pip install -r requirements.txt" in readme + assert "pip install traceAI-openai fi-instrumentation-otel openai" in readme + assert "python src/app.py --stream" in readme + command = """env -u PYTHONPATH PYTHONDONTWRITEBYTECODE=1 \\ + PYTHONPATH="python/examples/perplexity/src:python:python/frameworks/openai:python/tests" \\ + uv run --no-project --python 3.11 \\ + --with 'openai==3.24.0' --with httpx --with 'wrapt<2' \\ + --with opentelemetry-api --with opentelemetry-sdk --with opentelemetry-instrumentation \\ + --with opentelemetry-exporter-otlp-proto-http --with requests --with protobuf \\ + --with opentelemetry-proto --with pydantic --with jsonschema --with pytest \\ + pytest python/examples/perplexity/tests -q -p no:cacheprovider --noconftest -o addopts= -rfEs""" + assert command in readme + assert 'PYTHONPATH="python/examples/perplexity/src:python/tests"' in readme + assert "--with 'traceAI-openai==0.1.10' --with 'fi-instrumentation-otel==1.1.0'" in readme + table = """| Python | `openai` | traceAI packages | Result | +|---|---|---|---| +| 3.10, 3.11, 3.12, 3.13 | 3.24.0 | `traceai-openai` and `fi_instrumentation` from this repository | Full suite | +| 3.11 | 1.69.0 (the `traceai-openai` floor) | from this repository | Full suite | +| 3.11 | 3.24.0 | published `traceAI-openai==0.1.10` and `fi-instrumentation-otel==1.1.0` from PyPI | Full suite |""" + assert table in readme + assert "TBD" not in readme + assert not re.search(r"\b(?:sk-|pplx-)[A-Za-z0-9_-]{12,}", readme) + requirements = (RECIPE_DIR / "requirements.txt").read_text().splitlines() + assert [line for line in requirements if line and not line.startswith("#")] == [ + "openai==3.24.0", "traceAI-openai==0.1.10", "fi-instrumentation-otel==1.1.0", + ] diff --git a/python/examples/perplexity/tests/test_validation.py b/python/examples/perplexity/tests/test_validation.py new file mode 100644 index 000000000..1c3518fee --- /dev/null +++ b/python/examples/perplexity/tests/test_validation.py @@ -0,0 +1,47 @@ +"""URL preflight and project-name regressions shared with the sibling recipes.""" + +from test_perplexity_recipe import app, _assert_main_refusal, clean_environment, validation_servers +import pytest + + +@pytest.mark.parametrize("path", [ + "/v1//", "/v1///", "//", "/v1/other", "/V1", "/v2", + "/%76%31", "/v1%2F", "/v1%3F", "/v1/../v1", +]) +def test_perplexity_only_accepts_exact_base_path(path, monkeypatch, capsys, validation_servers): + _assert_main_refusal("https://api.perplexity.ai" + path, "base paths", monkeypatch, capsys, validation_servers) + + +@pytest.mark.parametrize("url", [ + "http://api。perplexity.ai/v1", "http://api.perplexity.ai/v1", + "https://api.perplexity.ai。/v1", "https://☃.example/v1", + "https://api.perplexity.ai../v1", "https://api..perplexity.ai/v1", + "https://api%2eperplexity.ai/v1", "https://api\\perplexity.ai/v1", + "https://api.perplexity.ai:bad/v1", "https://api.perplexity.ai:70000/v1", + " https://api.perplexity.ai/v1", "https://api.perplexity.ai/v1 ", + "https://api.perplexity.ai/\nv1", "https://api.perplexity.ai/\tv1", + "/v1", "api.perplexity.ai/v1", + "file:///v1", "https:///v1", "https://xn--/v1", +]) +def test_malformed_urls_exit_before_tracing(url, monkeypatch, capsys, validation_servers): + _assert_main_refusal(url, app.BASE_URL_ENV, monkeypatch, capsys, validation_servers) + + +def test_null_byte_rejected_by_direct_client_preflight(): + # The OS cannot store NUL in an environment value; direct callers still can supply it. + with pytest.raises(ValueError, match="control characters"): + app.check_base_url("https://api.perplexity.ai/v1\x00") + + +def test_setup_tracing_uses_project_env_and_argument(monkeypatch): + recorded = [] + monkeypatch.setattr(app, "register", lambda **kw: recorded.append(kw) or object()) + monkeypatch.setattr(app.OpenAIInstrumentor, "instrument", lambda *a, **kw: None) + monkeypatch.delenv("FI_PROJECT_NAME", raising=False) + app.setup_tracing() + monkeypatch.setenv("FI_PROJECT_NAME", "perplexity-env-project") + app.setup_tracing() + app.setup_tracing("perplexity-explicit-project") + assert [row["project_name"] for row in recorded] == [ + "perplexity-agent-api", "perplexity-env-project", "perplexity-explicit-project", + ]