From fbaa8df70922d8b58595a06df0c0df6b5ef2cca5 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Mon, 5 Oct 2026 20:15:22 -0700 Subject: [PATCH 1/5] feat(examples): NVIDIA DGX Cloud Lepton endpoint recipe on traceai-openai Adds python/examples/lepton/: the official openai SDK pointed at the OpenAI-compatible base URL shown in a DGX Cloud Lepton endpoint's API tab, traced by the existing traceai-openai instrumentor. No provider package and no constant base URL. - LEPTON_ENDPOINT_URL and a non-empty LEPTON_API_TOKEN are required; the placeholder (literal or repeatedly percent-encoded) is refused. No path is required or added (no guessed /v1). - check_base_url refuses the legacy Lepton AI hosts (api.lepton.ai, *.lepton.run) and the console hosts, including upper-case and trailing-dot spellings; it never rewrites a URL. main() exits 2 before tracing on a refused URL or missing configuration. - Public endpoint URLs are secret-equivalent: the app never prints the URL (refusal messages name the reason; SDK request errors are reported with a generic message). Tests pin that the endpoint URL and host are absent from the exported spans, events, status and resource, with and without the FI_HIDE_* flags. - Token only on the OpenAI client; Future AGI keys only on the tracer. - Pins current traceai-openai behaviour: provider label openai; streamed and failed spans have no gen_ai.request.model (the model stays in gen_ai.request.parameters). - Loopback only: MockTransport at the reserved .invalid test host, a local OpenAI-shaped fake, the shared harness receiver and a socket guard with a negative control. approved: Nikhil 2026-10-03 blanket Refs: TH-8318 --- python/examples/lepton/README.md | 149 +++++ python/examples/lepton/requirements.txt | 4 + python/examples/lepton/src/app.py | 114 ++++ python/examples/lepton/tests/_fake_openai.py | 100 +++ .../tests/loopback_guard/sitecustomize.py | 43 ++ .../lepton/tests/test_lepton_recipe.py | 605 ++++++++++++++++++ 6 files changed, 1015 insertions(+) create mode 100644 python/examples/lepton/README.md create mode 100644 python/examples/lepton/requirements.txt create mode 100644 python/examples/lepton/src/app.py create mode 100644 python/examples/lepton/tests/_fake_openai.py create mode 100644 python/examples/lepton/tests/loopback_guard/sitecustomize.py create mode 100644 python/examples/lepton/tests/test_lepton_recipe.py diff --git a/python/examples/lepton/README.md b/python/examples/lepton/README.md new file mode 100644 index 000000000..4d2ee916a --- /dev/null +++ b/python/examples/lepton/README.md @@ -0,0 +1,149 @@ +# NVIDIA DGX Cloud Lepton (OpenAI-compatible) with traceAI + +This recipe traces Chat Completions through the official `openai` Python SDK at the OpenAI-compatible base URL shown in your endpoint's API tab. It uses `traceai-openai`; there is no separate Lepton tracing package. + +## Install + +From this directory, install the version pins: + +```bash +pip install -r requirements.txt +``` + +The plain installation form is: + +```bash +pip install traceAI-openai fi-instrumentation-otel openai +``` + +## Configure + +There is no shared public base URL. The documented form is ``. Replace this placeholder before running the app. + +| Environment variable | Required | Purpose | +|---|---|---| +| `FI_API_KEY` | Yes | Future AGI API key; goes to the tracer only. | +| `FI_SECRET_KEY` | Yes | Future AGI secret key; goes to the tracer only. | +| `LEPTON_API_TOKEN` | Yes | Non-empty token shown in the endpoint's API tab; goes to the OpenAI client only. | +| `LEPTON_ENDPOINT_URL` | Yes | OpenAI-compatible base URL from the endpoint details page, API tab. No default. | +| `LEPTON_MODEL` | Yes, unless `--model` is supplied | Model served by your endpoint. | +| `FI_BASE_URL` | No | Override the Future AGI collector origin; used by the tracer only. | + +Missing values and refused URLs exit with code 2 before tracing starts. The OpenAI SDK requires a non-empty API key. Use the token the API tab shows. + +## Run + +From this directory: + +```bash +export FI_API_KEY='placeholder-futureagi-key' +export FI_SECRET_KEY='placeholder-futureagi-secret' +export LEPTON_API_TOKEN='placeholder-lepton-token' +export LEPTON_ENDPOINT_URL='' +export LEPTON_MODEL='nvidia/Nemotron-Research-Reasoning-Qwen-1.5B' +# Replace the URL and key placeholders with your own values first. +python src/app.py +python src/app.py --stream +``` + +You can also supply `--model` and `--prompt`. The example model is the model in NVIDIA's endpoint tutorial. Set the name to the model your endpoint actually serves. + +## Code + +The essential sequence from `src/app.py` is: + +```python +import os +from fi_instrumentation import register +from fi_instrumentation.fi_types import ProjectType +from traceai_openai import OpenAIInstrumentor +from openai import OpenAI +from app import check_base_url +base_url = check_base_url(os.environ["LEPTON_ENDPOINT_URL"]) +api_key = os.environ["LEPTON_API_TOKEN"] +model = os.environ["LEPTON_MODEL"] +provider = register(project_name="lepton-openai-recipe", project_type=ProjectType.OBSERVE, set_global_tracer_provider=False, verbose=False) +OpenAIInstrumentor().instrument(tracer_provider=provider) +client = OpenAI(base_url=base_url, api_key=api_key) +response = client.chat.completions.create(model=model, messages=[{"role": "user", "content": "What is a comet?"}]) +print(response.choices[0].message.content) +provider.force_flush() +``` + +The app validates the URL, model and non-empty token before registering tracing. It instruments before creating the client and flushes after the call. It reports request failures without printing the endpoint URL. + +## What you see in Future AGI + +- One `ChatCompletion` span with `gen_ai.span.kind=LLM` per tested call. +- For a non-streamed response, `gen_ai.request.model` is the model id the provider returns in its response. +- Current `traceai-openai` leaves `gen_ai.request.model` absent on streamed and failed calls. The requested model remains in the `gen_ai.request.parameters` JSON unless `FI_HIDE_LLM_INVOCATION_PARAMETERS=true`. +- Token usage is present only when the response includes usage. Missing usage is omitted, never filled with zero. +- Streaming text is accumulated by the same instrumentor. The default fixture exports no usage attributes. With `stream_options={"include_usage": True}` and a final usage chunk, the fixture exports input, output and total token counts. Your endpoint must support this option to use it. +- Request errors are recorded as span errors with an exception event. +- The provider field says `openai`. This is the shared instrumentor's label, including for a Lepton endpoint. + +The span resource uses the project name passed to `register()`. + +## Provider specifics + +Lepton here means [NVIDIA DGX Cloud Lepton endpoints](https://docs.nvidia.com/dgx-cloud/lepton/get-started/endpoint/). An endpoint is a running model that exposes an HTTP server. + +Copy the OpenAI-compatible base URL and token shown on the endpoint details page, in its API tab. Always stop if the API tab shows no OpenAI-compatible base URL; this recipe does not apply. Do not guess `/v1`. + +`check_base_url()` returns an allowed URL exactly as given. It does not add or require a path. The tested OpenAI SDK appends a trailing slash to a non-empty base path and normalizes URL casing. For the syntactic test host `https://endpoint.example.invalid/v1`, the client base URL is `https://endpoint.example.invalid/v1/` and the request path is `/v1/chat/completions`. A base URL with no path is accepted; the tested SDK serializes it without a trailing slash. The reserved `.invalid` host is only a test shape; it is never contacted. + +Endpoints can be public and reachable by anyone with the URL. For a public endpoint, treat the URL like a secret and never commit it. The app does not print endpoint URLs in its configuration or request error messages. Legacy Lepton AI hosts and console hosts are refused even with upper-case or trailing-dot spellings. Literal and repeatedly percent-encoded placeholders are refused. + +## Privacy + +Set `FI_HIDE_INPUTS=true` to keep prompt text out of exported span attributes. Set `FI_HIDE_OUTPUTS=true` to keep response text out. You can also pass `TraceConfig(hide_inputs=True)` to the instrumentor. The endpoint still receives the prompt. Future AGI masking does not change the provider's own logging. + +In the tested chat, streaming and HTTP 401 fixtures, the endpoint URL and host are absent from exported span attributes, events, status and resource values. This holds with the `FI_HIDE_*` flags enabled or disabled. The instrumentor does not export `server.address`, `url.full` or a base-URL attribute in these cases. This observation covers this recipe's instrumentor; extra HTTP instrumentation or logging has its own behavior. + +The vendor token is also absent from the exported spans, resources and collector headers. Future AGI keys are used only for the collector. + +## Limits / not covered + +- Legacy Lepton AI hosts: `api.lepton.ai`, `llm.lepton.run` and `*.lepton.run`, including `sdxl.lepton.run`. +- Console URLs: `dashboard.dgxc-lepton.nvidia.com` and `dashboard.lepton.ai`. +- The `leptonai` photon client. +- NVIDIA's Python SDK. It is a workspace control-plane client for batch jobs, endpoints and secrets, not an inference client. +- Non-OpenAI-compatible endpoints, endpoint deployment, and APIs other than Chat Completions. +- Live endpoint calls. This recipe is not tested against the live provider (no paid call); the tests use a local fake of the OpenAI API and in-process fixtures. + +## Tests + +Run from the repository root against repository source: + +```bash +env -u PYTHONPATH PYTHONDONTWRITEBYTECODE=1 \ + PYTHONPATH="python/examples/lepton/src:python:python/frameworks/openai:python/tests" \ + uv run --no-project --python 3.11 \ + --with 'openai==3.24.0' --with httpx --with 'wrapt<2' \ + --with opentelemetry-api --with opentelemetry-sdk --with opentelemetry-instrumentation \ + --with opentelemetry-exporter-otlp-proto-http --with requests --with protobuf \ + --with opentelemetry-proto --with pydantic --with jsonschema --with pytest \ + pytest python/examples/lepton/tests -q -p no:cacheprovider --noconftest -o addopts= -rfEs +``` + +Run from the repository root against published packages: + +```bash +env -u PYTHONPATH PYTHONDONTWRITEBYTECODE=1 \ + PYTHONPATH="python/examples/lepton/src:python/tests" \ + uv run --no-project --python 3.11 \ + --with 'openai==3.24.0' --with 'traceAI-openai==0.1.10' \ + --with 'fi-instrumentation-otel==1.1.0' --with httpx --with 'wrapt<2' \ + --with opentelemetry-api --with opentelemetry-sdk --with opentelemetry-instrumentation \ + --with opentelemetry-exporter-otlp-proto-http --with requests --with protobuf \ + --with opentelemetry-proto --with pydantic --with jsonschema --with pytest \ + pytest python/examples/lepton/tests -q -p no:cacheprovider --noconftest -o addopts= -rfEs +``` + +For the floor row, change `openai==3.24.0` to `openai==1.69.0` in the repository-source command. Tests use `httpx.MockTransport` at the syntactic test host and loopback subprocesses protected by a socket guard. No provider host is called. + +| Python | `openai` | traceAI packages | Result | +|---|---|---|---| +| 3.10, 3.11, 3.12, 3.13 | 3.24.0 | `traceai-openai` and `fi_instrumentation` from this repository | Full suite | +| 3.11 | 1.69.0 (the `traceai-openai` floor) | from this repository | Full suite | +| 3.11 | 3.24.0 | published `traceAI-openai==0.1.10` and `fi-instrumentation-otel==1.1.0` from PyPI | Full suite | diff --git a/python/examples/lepton/requirements.txt b/python/examples/lepton/requirements.txt new file mode 100644 index 000000000..9ef38df9d --- /dev/null +++ b/python/examples/lepton/requirements.txt @@ -0,0 +1,4 @@ +# Tested SDK pin and published traceAI package pins. +openai==3.24.0 +traceAI-openai==0.1.10 +fi-instrumentation-otel==1.1.0 diff --git a/python/examples/lepton/src/app.py b/python/examples/lepton/src/app.py new file mode 100644 index 000000000..ffc78ab7c --- /dev/null +++ b/python/examples/lepton/src/app.py @@ -0,0 +1,114 @@ +"""Trace OpenAI-compatible NVIDIA DGX Cloud Lepton Chat Completions.""" + +import argparse +import os +import sys +from urllib.parse import unquote, urlsplit + +import openai +from fi_instrumentation import register +from fi_instrumentation.fi_types import ProjectType +from traceai_openai import OpenAIInstrumentor + +DEFAULT_BASE_URL = None +DOCUMENTED_BASE_URL_FORMS = ("",) +API_KEY_ENV = "LEPTON_API_TOKEN" +BASE_URL_ENV = "LEPTON_ENDPOINT_URL" +MODEL_ENV = "LEPTON_MODEL" + + +def check_base_url(url: str) -> str: + """Check scope without rewriting or including the secret URL in errors.""" + if not url: + raise ValueError(f"Set {BASE_URL_ENV} from the endpoint's API tab.") + decoded = url + while (next_value := unquote(decoded)) != decoded: + decoded = next_value + if any(marker in decoded for marker in ("<", ">", "ENDPOINT_URL")): + raise ValueError(f"Replace the {BASE_URL_ENV} placeholder using the API tab.") + try: + parsed = urlsplit(url) + host = (parsed.hostname or "").lower().rstrip(".") + except ValueError: + raise ValueError(f"Set {BASE_URL_ENV} to a valid HTTP endpoint base URL.") from None + if parsed.scheme not in ("http", "https") or not host: + raise ValueError(f"Set {BASE_URL_ENV} to a valid HTTP endpoint base URL.") + if host == "api.lepton.ai" or host.endswith(".lepton.run"): + raise ValueError("Legacy Lepton AI hosts are unsupported; use the endpoint's API tab.") + if host in ("dashboard.dgxc-lepton.nvidia.com", "dashboard.lepton.ai"): + raise ValueError("Console hosts are not inference endpoints; use the endpoint's API tab.") + return url + + +def make_client( + base_url: str | None = None, + api_key: str | None = None, + http_client=None, +) -> openai.OpenAI: + if base_url is None: + base_url = os.environ.get(BASE_URL_ENV) or DEFAULT_BASE_URL + base_url = check_base_url(base_url or "") + api_key = api_key if api_key is not None else os.environ[API_KEY_ENV] + if not api_key.strip(): + raise ValueError(f"Set {API_KEY_ENV} to the non-empty token shown in the API tab.") + return openai.OpenAI(base_url=base_url, api_key=api_key, http_client=http_client) + + +def setup_tracing(project_name: str | None = None): + provider = register( + project_name=project_name or "lepton-openai-recipe", + project_type=ProjectType.OBSERVE, + set_global_tracer_provider=False, + verbose=False, + ) + OpenAIInstrumentor().instrument(tracer_provider=provider) + return provider + + +def main(argv=None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--stream", action="store_true") + parser.add_argument("--prompt", default="What is a comet? Answer in one sentence.") + parser.add_argument("--model") + args = parser.parse_args(argv) + try: + base_url = check_base_url(os.environ.get(BASE_URL_ENV) or DEFAULT_BASE_URL or "") + model = args.model or os.environ[MODEL_ENV] + if not model.strip(): + raise ValueError(f"Set {MODEL_ENV} to the model served by your endpoint.") + api_key = os.environ[API_KEY_ENV] + if not api_key.strip(): + raise ValueError(f"Set {API_KEY_ENV} to the non-empty token shown in the API tab.") + except KeyError as error: + print(f"Set {error.args[0]} before running this recipe.", file=sys.stderr) + return 2 + except ValueError as error: + print(str(error), file=sys.stderr) + return 2 + + provider = setup_tracing() + try: + with make_client(base_url=base_url, api_key=api_key) as client: + result = client.chat.completions.create( + model=model, + messages=[{"role": "user", "content": args.prompt}], + stream=args.stream, + ) + if args.stream: + for chunk in result: + if chunk.choices: + print(chunk.choices[0].delta.content or "", end="", flush=True) + print() + else: + print(result.choices[0].message.content or "") + except (openai.OpenAIError, ValueError): + # SDK error messages can contain the secret-equivalent endpoint URL. + print("Endpoint request failed; check the URL, token and model in the API tab.", file=sys.stderr) + return 1 + finally: + provider.force_flush() + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/python/examples/lepton/tests/_fake_openai.py b/python/examples/lepton/tests/_fake_openai.py new file mode 100644 index 000000000..03b15c58e --- /dev/null +++ b/python/examples/lepton/tests/_fake_openai.py @@ -0,0 +1,100 @@ +"""OpenAI-shaped chat, SSE and error fixtures, served only on loopback.""" + +import json +import threading +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +ANSWER = "A comet is an icy body that orbits the Sun." +PARTS = ("A comet is an icy body ", "that orbits the Sun.") +USAGE = {"prompt_tokens": 11, "completion_tokens": 13, "total_tokens": 24} +ERROR = {"error": {"message": "Unauthorized", "type": "authentication_error"}} + + +def completion(model, *, usage=True, answer=ANSWER): + body = { + "id": "chatcmpl-placeholder-lepton", + "object": "chat.completion", + "created": 1, + "model": model, + "choices": [{"index": 0, "message": {"role": "assistant", "content": answer}, "finish_reason": "stop"}], + } + if usage: + body["usage"] = dict(USAGE) + return body + + +def stream(model, *, include_usage=False): + chunks = [] + for part in PARTS: + chunks.append({ + "id": "chatcmpl-placeholder-lepton", + "object": "chat.completion.chunk", + "created": 1, + "model": model, + "choices": [{"index": 0, "delta": {"content": part}, "finish_reason": None}], + }) + chunks.append({ + "id": "chatcmpl-placeholder-lepton", + "object": "chat.completion.chunk", + "created": 1, + "model": model, + "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}], + }) + if include_usage: + chunks.append({ + "id": "chatcmpl-placeholder-lepton", + "object": "chat.completion.chunk", + "created": 1, + "model": model, + "choices": [], + "usage": dict(USAGE), + }) + return ("".join(f"data: {json.dumps(chunk)}\n\n" for chunk in chunks) + "data: [DONE]\n\n").encode() + + +class FakeOpenAI: + def __init__(self, *, error=False): + self.requests = [] + self._lock = threading.Lock() + owner = self + + class Handler(BaseHTTPRequestHandler): + def do_POST(self): + body = json.loads(self.rfile.read(int(self.headers["Content-Length"]))) + with owner._lock: + owner.requests.append({ + "path": self.path, + "headers": {key.lower(): value for key, value in self.headers.items()}, + "body": body, + }) + if self.path != "/v1/chat/completions": + self.send_error(404) + return + if error: + status, content_type, payload = 401, "application/json", json.dumps(ERROR).encode() + elif body.get("stream"): + status, content_type = 200, "text/event-stream" + payload = stream(body["model"], include_usage=body.get("stream_options", {}).get("include_usage", False)) + else: + status, content_type, payload = 200, "application/json", json.dumps(completion(body["model"])).encode() + self.send_response(status) + self.send_header("Content-Type", content_type) + self.send_header("Content-Length", str(len(payload))) + self.end_headers() + self.wfile.write(payload) + + def log_message(self, _format, *args): + pass + + self._server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + self.base_url = f"http://127.0.0.1:{self._server.server_port}/v1" + self._thread = threading.Thread(target=self._server.serve_forever, daemon=True) + self._thread.start() + + def __enter__(self): + return self + + def __exit__(self, *_args): + self._server.shutdown() + self._server.server_close() + self._thread.join() diff --git a/python/examples/lepton/tests/loopback_guard/sitecustomize.py b/python/examples/lepton/tests/loopback_guard/sitecustomize.py new file mode 100644 index 000000000..f519411d0 --- /dev/null +++ b/python/examples/lepton/tests/loopback_guard/sitecustomize.py @@ -0,0 +1,43 @@ +"""Refuse non-loopback network access before DNS or socket connection.""" + +import os +import socket +from pathlib import Path + +_getaddrinfo = socket.getaddrinfo +_connect = socket.socket.connect +_connect_ex = socket.socket.connect_ex + + +def _check(host): + if isinstance(host, bytes): + host = host.decode("ascii") + if host not in ("127.0.0.1", "localhost"): + log = os.environ.get("LOOPBACK_GUARD_LOG") + if log: + with open(log, "a", encoding="utf-8") as output: + output.write("REFUSED non-loopback host\n") + raise RuntimeError("Loopback guard refused non-loopback network access") + + +def _guarded_getaddrinfo(host, *args, **kwargs): + _check(host) + return _getaddrinfo(host, *args, **kwargs) + + +def _guarded_connect(self, address): + _check(address[0]) + return _connect(self, address) + + +def _guarded_connect_ex(self, address): + _check(address[0]) + return _connect_ex(self, address) + + +socket.getaddrinfo = _guarded_getaddrinfo +socket.socket.connect = _guarded_connect +socket.socket.connect_ex = _guarded_connect_ex + +if ready := os.environ.get("LOOPBACK_GUARD_READY"): + Path(ready).write_text("installed\n", encoding="utf-8") diff --git a/python/examples/lepton/tests/test_lepton_recipe.py b/python/examples/lepton/tests/test_lepton_recipe.py new file mode 100644 index 000000000..2deef8252 --- /dev/null +++ b/python/examples/lepton/tests/test_lepton_recipe.py @@ -0,0 +1,605 @@ +"""Fixture-only recipe contracts; .invalid is a syntactic test host, never contacted.""" + +import importlib.util +import json +import os +from pathlib import Path +import signal +import socket +import subprocess +import sys +from contextlib import contextmanager +from urllib.parse import quote, urlsplit + +import fi_instrumentation +import httpx +import openai +import pytest +import traceai_openai +from harness import Receiver +from traceai_openai import OpenAIInstrumentor + +TEST_DIR = Path(__file__).resolve().parent +RECIPE_DIR = TEST_DIR.parent + + +def load_module(name, path): + spec = importlib.util.spec_from_file_location(name, path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +app = load_module("lepton_recipe_app", RECIPE_DIR / "src" / "app.py") +fake = load_module("lepton_fake_openai", TEST_DIR / "_fake_openai.py") +TEST_BASE_URL = "https://endpoint.example.invalid/v1" +MODEL = "nvidia/Nemotron-Research-Reasoning-Qwen-1.5B" +VENDOR_KEY = "placeholder-lepton-key" +FI_KEY = "placeholder-futureagi-key" +FI_SECRET = "placeholder-futureagi-secret" +HIDE_FLAGS = ( + "FI_HIDE_INPUTS", "FI_HIDE_OUTPUTS", "FI_HIDE_LLM_INVOCATION_PARAMETERS", + "FI_HIDE_INPUT_MESSAGES", "FI_HIDE_OUTPUT_MESSAGES", "FI_HIDE_INPUT_IMAGES", + "FI_HIDE_INPUT_TEXT", "FI_HIDE_OUTPUT_TEXT", "FI_HIDE_EMBEDDING_VECTORS", +) +REFUSED_URLS = [ + (f"https://{host}/v1", reason) + for host, reason in ( + ("api.lepton.ai", "Legacy"), ("llm.lepton.run", "Legacy"), + ("sdxl.lepton.run", "Legacy"), ("nested.sdxl.lepton.run", "Legacy"), + ("dashboard.dgxc-lepton.nvidia.com", "Console"), ("dashboard.lepton.ai", "Console"), + ("API.LEPTON.AI", "Legacy"), ("api.lepton.ai.", "Legacy"), + ("LLM.LEPTON.RUN.", "Legacy"), ("SDXL.LEPTON.RUN.", "Legacy"), + ("DASHBOARD.DGXC-LEPTON.NVIDIA.COM.", "Console"), ("DASHBOARD.LEPTON.AI.", "Console"), + ) +] + [ + (value, "placeholder") + for value in ( + app.DOCUMENTED_BASE_URL_FORMS[0], + "https://endpoint.example.invalid/ENDPOINT_URL", + "https://endpoint.example.invalid/", + "https://endpoint.example.invalid/replace-me>", + quote(app.DOCUMENTED_BASE_URL_FORMS[0], safe=""), + quote(quote(app.DOCUMENTED_BASE_URL_FORMS[0], safe=""), safe=""), + "https://endpoint.example.invalid/%45NDPOINT_URL", + "https://endpoint.example.invalid/%2545NDPOINT_URL", + ) +] + + +@pytest.fixture(autouse=True) +def isolated_environment(monkeypatch): + for name in HIDE_FLAGS + ("FI_PII_REDACTION", "OPENAI_LOG"): + monkeypatch.delenv(name, raising=False) + for name in ("HTTP_PROXY", "HTTPS_PROXY", "ALL_PROXY", "http_proxy", "https_proxy", "all_proxy"): + monkeypatch.delenv(name, raising=False) + monkeypatch.setenv("NO_PROXY", "*") + monkeypatch.setenv("PYTHONDONTWRITEBYTECODE", "1") + monkeypatch.setenv("FI_API_KEY", FI_KEY) + monkeypatch.setenv("FI_SECRET_KEY", FI_SECRET) + monkeypatch.setenv(app.BASE_URL_ENV, TEST_BASE_URL) + monkeypatch.setenv(app.API_KEY_ENV, VENDOR_KEY) + monkeypatch.setenv(app.MODEL_ENV, MODEL) + # Also protect the test process: only the receiver and fake may use sockets. + original_dns = socket.getaddrinfo + original_connect = socket.socket.connect + original_connect_ex = socket.socket.connect_ex + + def check(host): + if isinstance(host, bytes): + host = host.decode("ascii") + if host not in ("127.0.0.1", "localhost"): + raise RuntimeError("Test refused non-loopback network access") + + def dns(host, *args, **kwargs): + check(host) + return original_dns(host, *args, **kwargs) + + def connect(sock, address): + check(address[0]) + return original_connect(sock, address) + + def connect_ex(sock, address): + check(address[0]) + return original_connect_ex(sock, address) + + monkeypatch.setattr(socket, "getaddrinfo", dns) + monkeypatch.setattr(socket.socket, "connect", connect) + monkeypatch.setattr(socket.socket, "connect_ex", connect_ex) + + +@contextmanager +def tracing(monkeypatch, project_name): + previous_signals = {sig: signal.getsignal(sig) for sig in (signal.SIGINT, signal.SIGTERM)} + with Receiver() as receiver: + monkeypatch.setenv("FI_BASE_URL", receiver.origin) + provider = app.setup_tracing(project_name) + try: + yield receiver, provider + finally: + provider.force_flush() + OpenAIInstrumentor().uninstrument() + provider.shutdown() + for sig, handler in previous_signals.items(): + signal.signal(sig, handler) + + +def attributes(span): + return { + item["key"]: next(iter(item["value"].values())) + for item in span.get("attributes", []) + } + + +def one_span(receiver, provider): + assert provider.force_flush() + spans = receiver.spans() + assert len(spans) == 1 + assert spans[0]["name"] == "ChatCompletion" + values = attributes(spans[0]) + assert values["gen_ai.span.kind"] == "LLM" + assert values["gen_ai.provider.name"] == "openai" + return spans[0], values + + +def telemetry(receiver): + return json.dumps({ + "spans": receiver.spans(), + "resources": [request["resource_attributes"] for request in receiver.requests()], + }) + + +def assert_key_separation(receiver, request): + # Positive control: the key really was used on the vendor request. + assert request.headers["authorization"] == f"Bearer {VENDOR_KEY}" + assert "x-api-key" not in request.headers + assert "x-secret-key" not in request.headers + assert FI_KEY not in str(request.headers) + assert FI_SECRET not in str(request.headers) + assert receiver.spans() + assert VENDOR_KEY not in telemetry(receiver) + exports = receiver.requests() + assert exports + for export in exports: + assert export["path"] == "/tracer/v1/traces" + assert export["headers"]["x-api-key"] == FI_KEY + assert export["headers"]["x-secret-key"] == FI_SECRET + assert VENDOR_KEY not in json.dumps(export["headers"]) + + +def mock_client(handler): + return app.make_client(http_client=httpx.Client(transport=httpx.MockTransport(handler))) + + +def chat(client, **kwargs): + return client.chat.completions.create( + model=MODEL, messages=[{"role": "user", "content": "What is a comet?"}], **kwargs, + ) + + +def child_environment(tmp_path, receiver, fake_server): + environment = os.environ.copy() + # Use actual imported package locations for source and published-wheel runs. + package_paths = [ + str(Path(fi_instrumentation.__file__).resolve().parent.parent), + str(Path(traceai_openai.__file__).resolve().parent.parent), + ] + environment.update({ + "PYTHONPATH": os.pathsep.join(dict.fromkeys([ + str(TEST_DIR / "loopback_guard"), str(RECIPE_DIR / "src"), + *package_paths, str(Path(sys.modules["harness"].__file__).resolve().parent.parent), + ])), + "PYTHONDONTWRITEBYTECODE": "1", + "FI_BASE_URL": receiver.origin, + "FI_API_KEY": FI_KEY, + "FI_SECRET_KEY": FI_SECRET, + app.BASE_URL_ENV: fake_server.base_url, + app.API_KEY_ENV: VENDOR_KEY, + app.MODEL_ENV: MODEL, + "LOOPBACK_GUARD_LOG": str(tmp_path / "guard.log"), + "LOOPBACK_GUARD_READY": str(tmp_path / "guard.ready"), + "NO_PROXY": "*", + }) + Path(environment["LOOPBACK_GUARD_LOG"]).write_text("", encoding="utf-8") + return environment + + +def run_child(environment, *args): + return subprocess.run([sys.executable, *args], env=environment, capture_output=True, text=True, timeout=30) + + +def assert_guard_installed(environment): + assert Path(environment["LOOPBACK_GUARD_READY"]).read_text() == "installed\n" + + +def test_documented_forms_require_customer_configuration(monkeypatch): + assert app.DEFAULT_BASE_URL is None + assert app.DOCUMENTED_BASE_URL_FORMS == ("",) + assert app.API_KEY_ENV == "LEPTON_API_TOKEN" + assert app.BASE_URL_ENV == "LEPTON_ENDPOINT_URL" + assert app.MODEL_ENV == "LEPTON_MODEL" + readme = (RECIPE_DIR / "README.md").read_text() + for form in app.DOCUMENTED_BASE_URL_FORMS: + assert form in readme + with pytest.raises(ValueError, match="placeholder"): + app.check_base_url(form) + monkeypatch.delenv(app.BASE_URL_ENV) + with pytest.raises(ValueError, match=app.BASE_URL_ENV): + app.make_client() + + +@pytest.mark.parametrize("url,sdk_url", [ + (TEST_BASE_URL, TEST_BASE_URL + "/"), + ("https://endpoint.example.invalid", "https://endpoint.example.invalid"), + ("https://endpoint.example.invalid/custom/root/", "https://endpoint.example.invalid/custom/root/"), + ("HTTPS://ENDPOINT.EXAMPLE.INVALID./Custom/Path", "https://endpoint.example.invalid./Custom/Path/"), + ("http://127.0.0.1:12345/inference", "http://127.0.0.1:12345/inference/"), +]) +def test_allowed_base_urls_are_returned_unchanged(url, sdk_url): + assert app.check_base_url(url) == url + with app.make_client(base_url=url) as client: + # Pin the SDK's serialized URL, including its treatment of an empty path. + assert str(client.base_url) == sdk_url + + +@pytest.mark.parametrize("url,reason", REFUSED_URLS) +def test_refused_base_urls_name_reason_without_disclosing_url(url, reason): + with pytest.raises(ValueError) as caught: + app.check_base_url(url) + message = str(caught.value) + assert reason in message + assert "API tab" in message + assert url not in message + assert "\n" not in message + + +@pytest.mark.parametrize("key", [None, "", " "]) +def test_make_client_requires_nonempty_vendor_key(monkeypatch, key): + if key is None: + monkeypatch.delenv(app.API_KEY_ENV) + with pytest.raises(KeyError) as caught: + app.make_client() + assert caught.value.args == (app.API_KEY_ENV,) + else: + monkeypatch.setenv(app.API_KEY_ENV, key) + with pytest.raises(ValueError, match=app.API_KEY_ENV): + app.make_client() + + +def test_chat_completion_contract(monkeypatch): + requests = [] + + def handler(request): + requests.append(request) + assert json.loads(request.content)["model"] == MODEL + return httpx.Response(200, json=fake.completion(MODEL)) + + with tracing(monkeypatch, "lepton-chat-contract") as (receiver, provider): + with mock_client(handler) as client: + assert str(client.base_url) == TEST_BASE_URL + "/" + assert chat(client).choices[0].message.content == fake.ANSWER + span, values = one_span(receiver, provider) + assert values["gen_ai.request.model"] == MODEL + assert values["output.value"] == fake.ANSWER + assert span["status"]["code"] == "STATUS_CODE_OK" + assert {key: int(value) for key, value in values.items() if key.startswith("gen_ai.usage.")} == { + "gen_ai.usage.input_tokens": 11, "gen_ai.usage.output_tokens": 13, "gen_ai.usage.total_tokens": 24, + } + assert len(requests) == 1 + assert str(requests[0].url) == TEST_BASE_URL + "/chat/completions" + assert_key_separation(receiver, requests[0]) + assert all(resource["project_name"] == "lepton-chat-contract" + for export in receiver.requests() for resource in export["resource_attributes"]) + + +def test_missing_usage_is_omitted(monkeypatch): + with tracing(monkeypatch, "lepton-missing-usage") as (receiver, provider): + for has_usage in (True, False): + receiver.clear() + with mock_client(lambda request: httpx.Response(200, json=fake.completion(MODEL, usage=has_usage))) as client: + chat(client) + _, values = one_span(receiver, provider) + usage = {key: int(value) for key, value in values.items() if key.startswith("gen_ai.usage.")} + if has_usage: + assert usage == {"gen_ai.usage.input_tokens": 11, "gen_ai.usage.output_tokens": 13, "gen_ai.usage.total_tokens": 24} + else: + assert usage == {} + + +@pytest.mark.parametrize("include_usage", [False, True]) +def test_stream_accumulates_text_and_pins_usage(monkeypatch, include_usage): + requests = [] + + def handler(request): + requests.append(request) + body = json.loads(request.content) + assert body["stream"] is True + assert body.get("stream_options") == ({"include_usage": True} if include_usage else None) + return httpx.Response(200, headers={"Content-Type": "text/event-stream"}, content=fake.stream(MODEL, include_usage=include_usage)) + + with tracing(monkeypatch, f"lepton-stream-{include_usage}") as (receiver, provider): + with mock_client(handler) as client: + kwargs = {"stream_options": {"include_usage": True}} if include_usage else {} + text = "".join(chunk.choices[0].delta.content or "" for chunk in chat(client, stream=True, **kwargs) if chunk.choices) + assert text == fake.ANSWER + _, values = one_span(receiver, provider) + assert values["output.value"] == fake.ANSWER + # traceai-openai records the model only from a non-streamed response; flip this when the instrumentor records the request model + assert "gen_ai.request.model" not in values + assert json.loads(values["gen_ai.request.parameters"])["model"] == MODEL + usage = {key: int(value) for key, value in values.items() if key.startswith("gen_ai.usage.")} + assert usage == ({"gen_ai.usage.input_tokens": 11, "gen_ai.usage.output_tokens": 13, "gen_ai.usage.total_tokens": 24} if include_usage else {}) + assert len(requests) == 1 + assert str(requests[0].url) == TEST_BASE_URL + "/chat/completions" + assert_key_separation(receiver, requests[0]) + + +def test_authentication_error_is_recorded_without_vendor_key(monkeypatch): + requests = [] + + def handler(request): + requests.append(request) + return httpx.Response(401, json=fake.ERROR) + + with tracing(monkeypatch, "lepton-authentication-error") as (receiver, provider): + with mock_client(handler) as client, pytest.raises(openai.AuthenticationError) as caught: + chat(client) + span, values = one_span(receiver, provider) + assert span["status"]["code"] == "STATUS_CODE_ERROR" + assert "Unauthorized" in span["status"]["message"] + exceptions = [event for event in span["events"] if event["name"] == "exception"] + assert len(exceptions) == 1 + assert "Unauthorized" in json.dumps(exceptions) + assert VENDOR_KEY not in str(caught.value) + # traceai-openai records the model only from a non-streamed response; flip this when the instrumentor records the request model + assert "gen_ai.request.model" not in values + assert json.loads(values["gen_ai.request.parameters"])["model"] == MODEL + assert not any(key.startswith("gen_ai.usage.") for key in values) + assert len(requests) == 1 + assert_key_separation(receiver, requests[0]) + + +@pytest.mark.parametrize("flag,marker", [ + ("FI_HIDE_INPUTS", "unique-lepton-private-prompt"), ("FI_HIDE_OUTPUTS", "unique-lepton-private-answer"), +]) +def test_privacy_flags_hide_text_with_visible_control(monkeypatch, flag, marker): + for hidden in (False, True): + monkeypatch.setenv(flag, str(hidden).lower()) + requests = [] + prompt = marker if flag == "FI_HIDE_INPUTS" else "ordinary-lepton-prompt" + answer = marker if flag == "FI_HIDE_OUTPUTS" else fake.ANSWER + + def handler(request): + requests.append(request) + return httpx.Response(200, json=fake.completion(MODEL, answer=answer)) + + with tracing(monkeypatch, f"lepton-privacy-{flag}-{hidden}") as (receiver, provider): + with mock_client(handler) as client: + response = client.chat.completions.create(model=MODEL, messages=[{"role": "user", "content": prompt}]) + one_span(receiver, provider) + assert json.loads(requests[0].content)["messages"][0]["content"] == prompt + assert response.choices[0].message.content == answer + assert (marker in telemetry(receiver)) is (not hidden) + + +@pytest.mark.parametrize("journey", ["chat", "stream", "error"]) +@pytest.mark.parametrize("hide", [False, True]) +def test_endpoint_url_and_host_are_not_exported(monkeypatch, journey, hide): + for flag in HIDE_FLAGS: + monkeypatch.setenv(flag, str(hide).lower()) + requests = [] + + def handler(request): + requests.append(request) + if journey == "error": + return httpx.Response(401, json=fake.ERROR) + if journey == "stream": + return httpx.Response(200, headers={"Content-Type": "text/event-stream"}, content=fake.stream(MODEL)) + return httpx.Response(200, json=fake.completion(MODEL)) + + with tracing(monkeypatch, f"lepton-url-privacy-{journey}-{hide}") as (receiver, provider): + with mock_client(handler) as client: + if journey == "error": + with pytest.raises(openai.AuthenticationError): + chat(client) + elif journey == "stream": + assert list(chat(client, stream=True)) + else: + assert chat(client).model == MODEL + _, values = one_span(receiver, provider) + # Positive controls: the endpoint was used, and real telemetry was exported. + assert len(requests) == 1 + assert str(requests[0].url) == TEST_BASE_URL + "/chat/completions" + assert receiver.requests()[0]["resource_attributes"] + exported = telemetry(receiver) + assert TEST_BASE_URL not in exported + assert urlsplit(TEST_BASE_URL).hostname not in exported + assert not {"server.address", "url.full", "http.url", "openai.base_url"}.intersection(values) + if hide: + assert "gen_ai.request.parameters" not in values + else: + assert json.loads(values["gen_ai.request.parameters"])["model"] == MODEL + assert_key_separation(receiver, requests[0]) + + +@pytest.mark.parametrize("stream", [False, True]) +def test_app_subprocess_is_loopback_only(monkeypatch, tmp_path, stream): + with Receiver() as receiver, fake.FakeOpenAI() as server: + environment = child_environment(tmp_path, receiver, server) + arguments = ["--prompt", "unique-lepton-cli-prompt"] + if stream: + del environment[app.MODEL_ENV] + arguments += ["--stream", "--model", MODEL] + result = run_child(environment, str(RECIPE_DIR / "src" / "app.py"), *arguments) + assert result.returncode == 0, result.stderr + assert fake.ANSWER in result.stdout + assert server.base_url not in result.stdout + result.stderr + assert VENDOR_KEY not in result.stdout + result.stderr + assert_guard_installed(environment) + assert Path(environment["LOOPBACK_GUARD_LOG"]).read_text() == "" + assert len(server.requests) == 1 + request = server.requests[0] + assert request["path"] == "/v1/chat/completions" + assert request["headers"]["authorization"] == f"Bearer {VENDOR_KEY}" + assert "x-api-key" not in request["headers"] + assert "x-secret-key" not in request["headers"] + assert request["body"]["model"] == MODEL + assert request["body"]["stream"] is stream + assert request["body"]["messages"] == [{"role": "user", "content": "unique-lepton-cli-prompt"}] + spans = receiver.spans() + assert len(spans) == 1 + assert spans[0]["name"] == "ChatCompletion" + values = attributes(spans[0]) + assert values["gen_ai.span.kind"] == "LLM" + assert values["gen_ai.provider.name"] == "openai" + assert fake.ANSWER in values["output.value"] + if stream: + # traceai-openai records the model only from a non-streamed response; flip this when the instrumentor records the request model + assert "gen_ai.request.model" not in values + assert json.loads(values["gen_ai.request.parameters"])["model"] == MODEL + else: + assert values["gen_ai.request.model"] == MODEL + assert VENDOR_KEY not in telemetry(receiver) + for export in receiver.requests(): + assert export["path"] == "/tracer/v1/traces" + assert export["headers"]["x-api-key"] == FI_KEY + assert export["headers"]["x-secret-key"] == FI_SECRET + assert VENDOR_KEY not in json.dumps(export["headers"]) + assert all(resource["project_name"] == "lepton-openai-recipe" for resource in export["resource_attributes"]) + + +@pytest.mark.parametrize("operation", ["dns", "connect", "connect_ex"]) +def test_guard_refuses_syntactic_host_before_dns(tmp_path, operation): + with Receiver() as receiver, fake.FakeOpenAI() as server: + environment = child_environment(tmp_path, receiver, server) + # The positive control proves that the same guard permits a loopback request. + positive = run_child(environment, "-c", "import socket; s = socket.create_connection(('127.0.0.1', " + str(server._server.server_port) + ")); s.close()") + assert positive.returncode == 0, positive.stderr + assert_guard_installed(environment) + assert Path(environment["LOOPBACK_GUARD_LOG"]).read_text() == "" + code = "import socket; " + { + "dns": "socket.getaddrinfo('endpoint.example.invalid', 443)", + "connect": "socket.socket().connect(('endpoint.example.invalid', 443))", + "connect_ex": "socket.socket().connect_ex(('endpoint.example.invalid', 443))", + }[operation] + refused = run_child(environment, "-c", code) + assert refused.returncode != 0 + assert "Loopback guard refused" in refused.stderr + assert Path(environment["LOOPBACK_GUARD_LOG"]).read_text() == "REFUSED non-loopback host\n" + assert server.requests == [] + assert receiver.spans() == [] + + +@pytest.mark.parametrize("variable,value", [ + (app.BASE_URL_ENV, None), (app.BASE_URL_ENV, ""), (app.MODEL_ENV, None), + (app.MODEL_ENV, ""), (app.API_KEY_ENV, None), (app.API_KEY_ENV, ""), (app.API_KEY_ENV, " "), +]) +def test_main_missing_configuration_exits_before_tracing(monkeypatch, capsys, variable, value): + if value is None: + monkeypatch.delenv(variable) + else: + monkeypatch.setenv(variable, value) + + def forbidden(): + pytest.fail("Tracing started before configuration validation") + + monkeypatch.setattr(app, "setup_tracing", forbidden) + assert app.main([]) == 2 + captured = capsys.readouterr() + assert variable in captured.err + assert TEST_BASE_URL not in captured.out + captured.err + assert VENDOR_KEY not in captured.out + captured.err + + +@pytest.mark.parametrize("url,reason", [REFUSED_URLS[0], REFUSED_URLS[4], REFUSED_URLS[-1]]) +def test_main_refuses_url_without_tracing_or_network(monkeypatch, capsys, tmp_path, url, reason): + with Receiver() as receiver, fake.FakeOpenAI() as server: + monkeypatch.setenv("FI_BASE_URL", receiver.origin) + monkeypatch.setenv(app.BASE_URL_ENV, url) + + def forbidden(): + pytest.fail("Tracing started for a refused URL") + + monkeypatch.setattr(app, "setup_tracing", forbidden) + assert app.main([]) == 2 + captured = capsys.readouterr() + assert reason in captured.err + assert url not in captured.out + captured.err + environment = child_environment(tmp_path, receiver, server) + environment[app.BASE_URL_ENV] = url + child = run_child(environment, str(RECIPE_DIR / "src" / "app.py")) + assert child.returncode == 2 + assert reason in child.stderr + assert url not in child.stdout + child.stderr + assert_guard_installed(environment) + assert Path(environment["LOOPBACK_GUARD_LOG"]).read_text() == "" + assert server.requests == [] + assert receiver.spans() == [] + assert receiver.requests() == [] + + +def test_app_error_does_not_print_endpoint_url(tmp_path): + with Receiver() as receiver, fake.FakeOpenAI(error=True) as server: + environment = child_environment(tmp_path, receiver, server) + result = run_child(environment, str(RECIPE_DIR / "src" / "app.py")) + assert result.returncode == 1 + assert "Endpoint request failed" in result.stderr + assert server.base_url not in result.stdout + result.stderr + assert VENDOR_KEY not in result.stdout + result.stderr + assert_guard_installed(environment) + assert Path(environment["LOOPBACK_GUARD_LOG"]).read_text() == "" + assert len(server.requests) == 1 + assert server.requests[0]["headers"]["authorization"] == f"Bearer {VENDOR_KEY}" + spans = receiver.spans() + assert len(spans) == 1 + assert spans[0]["status"]["code"] == "STATUS_CODE_ERROR" + + +def test_main_redacts_sdk_error_containing_endpoint_url(monkeypatch, capsys): + flushed = [] + + class Provider: + def force_flush(self): + flushed.append(True) + + error = openai.OpenAIError(f"Request to {TEST_BASE_URL} failed using {VENDOR_KEY}") + assert TEST_BASE_URL in str(error) + assert VENDOR_KEY in str(error) + + def failed_client(**kwargs): + assert kwargs == {"base_url": TEST_BASE_URL, "api_key": VENDOR_KEY} + raise error + + monkeypatch.setattr(app, "setup_tracing", Provider) + monkeypatch.setattr(app, "make_client", failed_client) + assert app.main([]) == 1 + captured = capsys.readouterr() + assert "Endpoint request failed" in captured.err + assert TEST_BASE_URL not in captured.out + captured.err + assert VENDOR_KEY not in captured.out + captured.err + assert flushed == [True] + + +def test_readme_and_requirements_pin_customer_contract(): + readme = (RECIPE_DIR / "README.md").read_text() + for text in ( + *app.DOCUMENTED_BASE_URL_FORMS, app.API_KEY_ENV, app.BASE_URL_ENV, app.MODEL_ENV, + "FI_API_KEY", "FI_SECRET_KEY", "FI_BASE_URL", "FI_HIDE_INPUTS", "FI_HIDE_OUTPUTS", + "FI_HIDE_LLM_INVOCATION_PARAMETERS", "provider field says `openai`", + "stop if the API tab shows no OpenAI-compatible base URL", "anyone with the URL", + "treat the URL like a secret", "never commit it", "Do not guess `/v1`", + "gen_ai.request.model", "endpoint URL and host are absent", + "syntactic test host", "api.lepton.ai", "*.lepton.run", "leptonai", + "NVIDIA's Python SDK", "Non-OpenAI-compatible endpoints", + 'PYTHONPATH="python/examples/lepton/src:python:python/frameworks/openai:python/tests"', + 'PYTHONPATH="python/examples/lepton/src:python/tests"', + "pytest python/examples/lepton/tests -q -p no:cacheprovider --noconftest -o addopts= -rfEs", + "| 3.10, 3.11, 3.12, 3.13 | 3.24.0 |", "| 3.11 | 1.69.0 (the `traceai-openai` floor) |", + "published `traceAI-openai==0.1.10` and `fi-instrumentation-otel==1.1.0`", + ): + assert text in readme + assert "TBD" not in readme + assert "sk-" not in readme + pins = (RECIPE_DIR / "requirements.txt").read_text().splitlines() + assert pins[0].startswith("#") + assert pins[1:] == ["openai==3.24.0", "traceAI-openai==0.1.10", "fi-instrumentation-otel==1.1.0"] From 542c62e5703b25aa1beb63a9051aa83f89ea3c98 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Mon, 5 Oct 2026 21:02:57 -0700 Subject: [PATCH 2/5] fix(examples/lepton): accept real xenon endpoints; no host disclosure pr-reviewer t_3b034732 requested changes: - R1 (P1): the *.lepton.run wildcard refused every real DGX Cloud Lepton endpoint. leptonai 0.27.3 config.py builds endpoint URLs as https://-.xenon.lepton.run. Now only the named legacy hosts (api.lepton.ai, llm.lepton.run, sdxl.lepton.run) and the console hosts are refused. Rules for *.xenon.lepton.run: - https is required; - a host-only or "/" path is refused with "append /v1" (NVIDIA documents /v1/chat/completions); - a query or fragment is refused. An in-process chat test at https://ws0000-example.xenon.lepton.run/v1 pins the exact SDK request URL. - R2 (P2): non-ASCII or malformed-IDNA hosts (NBSP, zero-width space, Unicode full stops) and credentials in the URL are refused with generic messages that echo nothing. main() catches any unexpected exception with the same generic message and exit 1, so no traceback can print the secret-equivalent host. Subprocess tests check that the host is absent from stdout and stderr and that no traceback appears. P3: - F1: README token wording; recommend access-token endpoints. - F5: "lower-cases the scheme and host". - F6: snippet PYTHONPATH note. - F7: missing and empty tokens raise the same ValueError. README: configure with the API-tab endpoint URL plus /v1. RED (new tests against the old app and README): 45 failed, 59 passed. GREEN: 104 passed. approved: Nikhil 2026-10-03 blanket Refs: TH-8318 --- python/examples/lepton/README.md | 20 ++- python/examples/lepton/src/app.py | 36 ++-- .../lepton/tests/test_lepton_recipe.py | 165 ++++++++++++++---- 3 files changed, 166 insertions(+), 55 deletions(-) diff --git a/python/examples/lepton/README.md b/python/examples/lepton/README.md index 4d2ee916a..bd7c5ca9c 100644 --- a/python/examples/lepton/README.md +++ b/python/examples/lepton/README.md @@ -1,6 +1,6 @@ # NVIDIA DGX Cloud Lepton (OpenAI-compatible) with traceAI -This recipe traces Chat Completions through the official `openai` Python SDK at the OpenAI-compatible base URL shown in your endpoint's API tab. It uses `traceai-openai`; there is no separate Lepton tracing package. +This recipe traces Chat Completions through the official `openai` Python SDK at an OpenAI-compatible NVIDIA DGX Cloud Lepton LLM endpoint. It uses `traceai-openai`; there is no separate Lepton tracing package. ## Install @@ -18,18 +18,18 @@ pip install traceAI-openai fi-instrumentation-otel openai ## Configure -There is no shared public base URL. The documented form is ``. Replace this placeholder before running the app. +There is no shared public base URL. The endpoint URL from the API tab, or `lep endpoint get -n `, looks like `https://-.xenon.lepton.run`. Set `LEPTON_ENDPOINT_URL` to that URL plus `/v1`, as NVIDIA documents for LLM endpoints. The documented base-URL form is `/v1`. Replace the placeholder before running the app. | Environment variable | Required | Purpose | |---|---|---| | `FI_API_KEY` | Yes | Future AGI API key; goes to the tracer only. | | `FI_SECRET_KEY` | Yes | Future AGI secret key; goes to the tracer only. | | `LEPTON_API_TOKEN` | Yes | Non-empty token shown in the endpoint's API tab; goes to the OpenAI client only. | -| `LEPTON_ENDPOINT_URL` | Yes | OpenAI-compatible base URL from the endpoint details page, API tab. No default. | +| `LEPTON_ENDPOINT_URL` | Yes | Endpoint URL from the endpoint details page, API tab, plus `/v1`. No default. | | `LEPTON_MODEL` | Yes, unless `--model` is supplied | Model served by your endpoint. | | `FI_BASE_URL` | No | Override the Future AGI collector origin; used by the tracer only. | -Missing values and refused URLs exit with code 2 before tracing starts. The OpenAI SDK requires a non-empty API key. Use the token the API tab shows. +Missing values and refused URLs exit with code 2 before tracing starts. This recipe requires a non-empty `LEPTON_API_TOKEN`. Create the endpoint with an access token, as NVIDIA recommends over making it public. Use the token the API tab shows. ## Run @@ -39,7 +39,7 @@ From this directory: export FI_API_KEY='placeholder-futureagi-key' export FI_SECRET_KEY='placeholder-futureagi-secret' export LEPTON_API_TOKEN='placeholder-lepton-token' -export LEPTON_ENDPOINT_URL='' +export LEPTON_ENDPOINT_URL='/v1' export LEPTON_MODEL='nvidia/Nemotron-Research-Reasoning-Qwen-1.5B' # Replace the URL and key placeholders with your own values first. python src/app.py @@ -50,7 +50,7 @@ You can also supply `--model` and `--prompt`. The example model is the model in ## Code -The essential sequence from `src/app.py` is: +The essential sequence from `src/app.py` is below. To use `from app import check_base_url`, put `src/` on `PYTHONPATH` or run the snippet from `src/`. ```python import os @@ -88,9 +88,11 @@ The span resource uses the project name passed to `register()`. Lepton here means [NVIDIA DGX Cloud Lepton endpoints](https://docs.nvidia.com/dgx-cloud/lepton/get-started/endpoint/). An endpoint is a running model that exposes an HTTP server. -Copy the OpenAI-compatible base URL and token shown on the endpoint details page, in its API tab. Always stop if the API tab shows no OpenAI-compatible base URL; this recipe does not apply. Do not guess `/v1`. +Copy the endpoint URL and API key from the endpoint details page, in its [API tab](https://docs.nvidia.com/dgx-cloud/lepton/features/endpoints/create-llm/). You can also run `lep endpoint get -n ` and read `external_endpoint`. The endpoint URL looks like `https://-.xenon.lepton.run`. Append `/v1` yourself when setting `LEPTON_ENDPOINT_URL`: NVIDIA's [LLM endpoint example](https://docs.nvidia.com/dgx-cloud/lepton/examples/endpoint/deploy-gpt-oss/) uses `/v1/chat/completions`. If the endpoint is not OpenAI-compatible, stop; this recipe does not apply. -`check_base_url()` returns an allowed URL exactly as given. It does not add or require a path. The tested OpenAI SDK appends a trailing slash to a non-empty base path and normalizes URL casing. For the syntactic test host `https://endpoint.example.invalid/v1`, the client base URL is `https://endpoint.example.invalid/v1/` and the request path is `/v1/chat/completions`. A base URL with no path is accepted; the tested SDK serializes it without a trailing slash. The reserved `.invalid` host is only a test shape; it is never contacted. +`check_base_url()` returns an allowed URL exactly as given; it never appends `/v1`. For hosts under `*.xenon.lepton.run`, it requires HTTPS, rejects a host-only URL or `/` path, and refuses query strings and fragments. For every host, it refuses non-ASCII or malformed IDNA hostnames and credentials in the URL. + +The tested OpenAI SDK appends a trailing slash to a non-empty base path and lower-cases the scheme and host (the path keeps its case). For the syntactic test host `https://ws0000-example.xenon.lepton.run/v1`, the client base URL is `https://ws0000-example.xenon.lepton.run/v1/` and the request URL is `https://ws0000-example.xenon.lepton.run/v1/chat/completions`. The suite also keeps `https://endpoint.example.invalid/v1` as a syntactic test host on the reserved `.invalid` domain. Neither test host is contacted; both use in-process fixtures. Endpoints can be public and reachable by anyone with the URL. For a public endpoint, treat the URL like a secret and never commit it. The app does not print endpoint URLs in its configuration or request error messages. Legacy Lepton AI hosts and console hosts are refused even with upper-case or trailing-dot spellings. Literal and repeatedly percent-encoded placeholders are refused. @@ -104,7 +106,7 @@ The vendor token is also absent from the exported spans, resources and collector ## Limits / not covered -- Legacy Lepton AI hosts: `api.lepton.ai`, `llm.lepton.run` and `*.lepton.run`, including `sdxl.lepton.run`. +- Named legacy Lepton AI hosts: `api.lepton.ai`, `llm.lepton.run` and `sdxl.lepton.run`. - Console URLs: `dashboard.dgxc-lepton.nvidia.com` and `dashboard.lepton.ai`. - The `leptonai` photon client. - NVIDIA's Python SDK. It is a workspace control-plane client for batch jobs, endpoints and secrets, not an inference client. diff --git a/python/examples/lepton/src/app.py b/python/examples/lepton/src/app.py index ffc78ab7c..fe68dbe66 100644 --- a/python/examples/lepton/src/app.py +++ b/python/examples/lepton/src/app.py @@ -11,7 +11,7 @@ from traceai_openai import OpenAIInstrumentor DEFAULT_BASE_URL = None -DOCUMENTED_BASE_URL_FORMS = ("",) +DOCUMENTED_BASE_URL_FORMS = ("/v1",) API_KEY_ENV = "LEPTON_API_TOKEN" BASE_URL_ENV = "LEPTON_ENDPOINT_URL" MODEL_ENV = "LEPTON_MODEL" @@ -28,15 +28,28 @@ def check_base_url(url: str) -> str: raise ValueError(f"Replace the {BASE_URL_ENV} placeholder using the API tab.") try: parsed = urlsplit(url) - host = (parsed.hostname or "").lower().rstrip(".") + host = parsed.hostname or "" + if parsed.scheme not in ("http", "https") or not host or not host.isascii(): + raise ValueError + # Reject malformed IDNA labels before the SDK can include them in an error. + host.encode("ascii").decode("idna").encode("idna") + _ = parsed.port # Validate the port without echoing parser errors. except ValueError: raise ValueError(f"Set {BASE_URL_ENV} to a valid HTTP endpoint base URL.") from None - if parsed.scheme not in ("http", "https") or not host: - raise ValueError(f"Set {BASE_URL_ENV} to a valid HTTP endpoint base URL.") - if host == "api.lepton.ai" or host.endswith(".lepton.run"): + if parsed.username is not None or parsed.password is not None: + raise ValueError(f"Set {BASE_URL_ENV} without credentials (userinfo); use {API_KEY_ENV}.") from None + host = host.lower().rstrip(".") + if host in ("api.lepton.ai", "llm.lepton.run", "sdxl.lepton.run"): raise ValueError("Legacy Lepton AI hosts are unsupported; use the endpoint's API tab.") if host in ("dashboard.dgxc-lepton.nvidia.com", "dashboard.lepton.ai"): raise ValueError("Console hosts are not inference endpoints; use the endpoint's API tab.") + if host.endswith(".xenon.lepton.run"): + if parsed.scheme != "https": + raise ValueError("Use https for endpoint access; the token must not travel in cleartext.") + if parsed.path in ("", "/"): + raise ValueError("append /v1 to the endpoint URL (NVIDIA documents /v1/chat/completions)") + if "?" in url or "#" in url: + raise ValueError("Remove the query or fragment from the endpoint base URL.") return url @@ -48,8 +61,8 @@ def make_client( if base_url is None: base_url = os.environ.get(BASE_URL_ENV) or DEFAULT_BASE_URL base_url = check_base_url(base_url or "") - api_key = api_key if api_key is not None else os.environ[API_KEY_ENV] - if not api_key.strip(): + api_key = api_key if api_key is not None else os.environ.get(API_KEY_ENV) + if not api_key or not api_key.strip(): raise ValueError(f"Set {API_KEY_ENV} to the non-empty token shown in the API tab.") return openai.OpenAI(base_url=base_url, api_key=api_key, http_client=http_client) @@ -76,8 +89,8 @@ def main(argv=None) -> int: model = args.model or os.environ[MODEL_ENV] if not model.strip(): raise ValueError(f"Set {MODEL_ENV} to the model served by your endpoint.") - api_key = os.environ[API_KEY_ENV] - if not api_key.strip(): + api_key = os.environ.get(API_KEY_ENV) + if not api_key or not api_key.strip(): raise ValueError(f"Set {API_KEY_ENV} to the non-empty token shown in the API tab.") except KeyError as error: print(f"Set {error.args[0]} before running this recipe.", file=sys.stderr) @@ -101,10 +114,13 @@ def main(argv=None) -> int: print() else: print(result.choices[0].message.content or "") - except (openai.OpenAIError, ValueError): + except openai.OpenAIError: # SDK error messages can contain the secret-equivalent endpoint URL. print("Endpoint request failed; check the URL, token and model in the API tab.", file=sys.stderr) return 1 + except Exception: + print("Endpoint request failed; check the URL, token and model in the API tab.", file=sys.stderr) + return 1 finally: provider.force_flush() return 0 diff --git a/python/examples/lepton/tests/test_lepton_recipe.py b/python/examples/lepton/tests/test_lepton_recipe.py index 2deef8252..cff82f5a3 100644 --- a/python/examples/lepton/tests/test_lepton_recipe.py +++ b/python/examples/lepton/tests/test_lepton_recipe.py @@ -1,4 +1,4 @@ -"""Fixture-only recipe contracts; .invalid is a syntactic test host, never contacted.""" +"""Fixture-only recipe contracts; syntactic endpoint hosts are never contacted.""" import importlib.util import json @@ -33,6 +33,7 @@ def load_module(name, path): app = load_module("lepton_recipe_app", RECIPE_DIR / "src" / "app.py") fake = load_module("lepton_fake_openai", TEST_DIR / "_fake_openai.py") TEST_BASE_URL = "https://endpoint.example.invalid/v1" +XENON_TEST_BASE_URL = "https://ws0000-example.xenon.lepton.run/v1" MODEL = "nvidia/Nemotron-Research-Reasoning-Qwen-1.5B" VENDOR_KEY = "placeholder-lepton-key" FI_KEY = "placeholder-futureagi-key" @@ -42,11 +43,29 @@ def load_module(name, path): "FI_HIDE_INPUT_MESSAGES", "FI_HIDE_OUTPUT_MESSAGES", "FI_HIDE_INPUT_IMAGES", "FI_HIDE_INPUT_TEXT", "FI_HIDE_OUTPUT_TEXT", "FI_HIDE_EMBEDDING_VECTORS", ) +INVALID_HOST_URLS = [ + "https://endpoint.example.invalid\u00a0", + "https://endpoint.example.invalid\u200b", + *[f"https://endpoint{separator}example.invalid" for separator in ("\u3002", "\uff0e", "\uff61")], + "https://xn--.example.invalid", + "https://" + "a" * 64 + ".example.invalid", +] +ALLOWED_URLS = [ + (TEST_BASE_URL, TEST_BASE_URL + "/"), + ("https://endpoint.example.invalid", "https://endpoint.example.invalid"), + ("https://endpoint.example.invalid/custom/root/", "https://endpoint.example.invalid/custom/root/"), + ("HTTPS://ENDPOINT.EXAMPLE.INVALID./Custom/Path", "https://endpoint.example.invalid./Custom/Path/"), + ("http://127.0.0.1:12345/inference", "http://127.0.0.1:12345/inference/"), + (XENON_TEST_BASE_URL, XENON_TEST_BASE_URL + "/"), + (XENON_TEST_BASE_URL + "/", XENON_TEST_BASE_URL + "/"), + ("HTTPS://WS0000-EXAMPLE.XENON.LEPTON.RUN./v1", "https://ws0000-example.xenon.lepton.run./v1/"), + ("https://nested.sdxl.lepton.run/v1", "https://nested.sdxl.lepton.run/v1/"), +] REFUSED_URLS = [ (f"https://{host}/v1", reason) for host, reason in ( ("api.lepton.ai", "Legacy"), ("llm.lepton.run", "Legacy"), - ("sdxl.lepton.run", "Legacy"), ("nested.sdxl.lepton.run", "Legacy"), + ("sdxl.lepton.run", "Legacy"), ("dashboard.dgxc-lepton.nvidia.com", "Console"), ("dashboard.lepton.ai", "Console"), ("API.LEPTON.AI", "Legacy"), ("api.lepton.ai.", "Legacy"), ("LLM.LEPTON.RUN.", "Legacy"), ("SDXL.LEPTON.RUN.", "Legacy"), @@ -64,6 +83,21 @@ def load_module(name, path): "https://endpoint.example.invalid/%45NDPOINT_URL", "https://endpoint.example.invalid/%2545NDPOINT_URL", ) +] + [ + ("http://ws0000-example.xenon.lepton.run/v1", "https"), + ("HTTP://WS0000-EXAMPLE.XENON.LEPTON.RUN./v1", "https"), + ("https://ws0000-example.xenon.lepton.run", "append /v1"), + ("https://ws0000-example.xenon.lepton.run/", "append /v1"), + ("https://WS0000-EXAMPLE.XENON.LEPTON.RUN.", "append /v1"), + (XENON_TEST_BASE_URL + "?placeholder=query", "query or fragment"), + (XENON_TEST_BASE_URL + "#placeholder-fragment", "query or fragment"), + (XENON_TEST_BASE_URL + "?", "query or fragment"), + (XENON_TEST_BASE_URL + "#", "query or fragment"), + ("https://placeholder-user:placeholder-password@endpoint.example.invalid/v1", "credentials"), + ("https://placeholder-user@ws0000-example.xenon.lepton.run/v1", "credentials"), + ("http://placeholder-user@127.0.0.1:12345/v1", "credentials"), + ("https://@endpoint.example.invalid/v1", "credentials"), + *[(url, "valid HTTP endpoint base URL") for url in INVALID_HOST_URLS], ] @@ -214,7 +248,7 @@ def assert_guard_installed(environment): def test_documented_forms_require_customer_configuration(monkeypatch): assert app.DEFAULT_BASE_URL is None - assert app.DOCUMENTED_BASE_URL_FORMS == ("",) + assert app.DOCUMENTED_BASE_URL_FORMS == ("/v1",) assert app.API_KEY_ENV == "LEPTON_API_TOKEN" assert app.BASE_URL_ENV == "LEPTON_ENDPOINT_URL" assert app.MODEL_ENV == "LEPTON_MODEL" @@ -228,13 +262,7 @@ def test_documented_forms_require_customer_configuration(monkeypatch): app.make_client() -@pytest.mark.parametrize("url,sdk_url", [ - (TEST_BASE_URL, TEST_BASE_URL + "/"), - ("https://endpoint.example.invalid", "https://endpoint.example.invalid"), - ("https://endpoint.example.invalid/custom/root/", "https://endpoint.example.invalid/custom/root/"), - ("HTTPS://ENDPOINT.EXAMPLE.INVALID./Custom/Path", "https://endpoint.example.invalid./Custom/Path/"), - ("http://127.0.0.1:12345/inference", "http://127.0.0.1:12345/inference/"), -]) +@pytest.mark.parametrize("url,sdk_url", ALLOWED_URLS) def test_allowed_base_urls_are_returned_unchanged(url, sdk_url): assert app.check_base_url(url) == url with app.make_client(base_url=url) as client: @@ -248,25 +276,34 @@ def test_refused_base_urls_name_reason_without_disclosing_url(url, reason): app.check_base_url(url) message = str(caught.value) assert reason in message - assert "API tab" in message assert url not in message + if host := urlsplit(url).hostname: + assert host not in message assert "\n" not in message + if reason == "append /v1": + assert message == "append /v1 to the endpoint URL (NVIDIA documents /v1/chat/completions)" + elif url in INVALID_HOST_URLS: + assert message == "Set LEPTON_ENDPOINT_URL to a valid HTTP endpoint base URL." + assert caught.value.__suppress_context__ is True + elif reason == "credentials": + assert "placeholder-user" not in message + assert "placeholder-password" not in message @pytest.mark.parametrize("key", [None, "", " "]) def test_make_client_requires_nonempty_vendor_key(monkeypatch, key): if key is None: monkeypatch.delenv(app.API_KEY_ENV) - with pytest.raises(KeyError) as caught: - app.make_client() - assert caught.value.args == (app.API_KEY_ENV,) else: monkeypatch.setenv(app.API_KEY_ENV, key) - with pytest.raises(ValueError, match=app.API_KEY_ENV): - app.make_client() + with pytest.raises(ValueError, match=app.API_KEY_ENV) as caught: + app.make_client() + assert str(caught.value) == f"Set {app.API_KEY_ENV} to the non-empty token shown in the API tab." -def test_chat_completion_contract(monkeypatch): +@pytest.mark.parametrize("base_url", [TEST_BASE_URL, XENON_TEST_BASE_URL]) +def test_chat_completion_contract(monkeypatch, base_url): + monkeypatch.setenv(app.BASE_URL_ENV, base_url) requests = [] def handler(request): @@ -274,9 +311,10 @@ def handler(request): assert json.loads(request.content)["model"] == MODEL return httpx.Response(200, json=fake.completion(MODEL)) - with tracing(monkeypatch, "lepton-chat-contract") as (receiver, provider): + project_name = "lepton-chat-xenon" if base_url == XENON_TEST_BASE_URL else "lepton-chat-invalid" + with tracing(monkeypatch, project_name) as (receiver, provider): with mock_client(handler) as client: - assert str(client.base_url) == TEST_BASE_URL + "/" + assert str(client.base_url) == base_url + "/" assert chat(client).choices[0].message.content == fake.ANSWER span, values = one_span(receiver, provider) assert values["gen_ai.request.model"] == MODEL @@ -286,9 +324,11 @@ def handler(request): "gen_ai.usage.input_tokens": 11, "gen_ai.usage.output_tokens": 13, "gen_ai.usage.total_tokens": 24, } assert len(requests) == 1 - assert str(requests[0].url) == TEST_BASE_URL + "/chat/completions" + assert str(requests[0].url) == base_url + "/chat/completions" assert_key_separation(receiver, requests[0]) - assert all(resource["project_name"] == "lepton-chat-contract" + assert base_url not in telemetry(receiver) + assert urlsplit(base_url).hostname not in telemetry(receiver) + assert all(resource["project_name"] == project_name for export in receiver.requests() for resource in export["resource_attributes"]) @@ -384,7 +424,9 @@ def handler(request): @pytest.mark.parametrize("journey", ["chat", "stream", "error"]) @pytest.mark.parametrize("hide", [False, True]) -def test_endpoint_url_and_host_are_not_exported(monkeypatch, journey, hide): +@pytest.mark.parametrize("base_url", [TEST_BASE_URL, XENON_TEST_BASE_URL]) +def test_endpoint_url_and_host_are_not_exported(monkeypatch, journey, hide, base_url): + monkeypatch.setenv(app.BASE_URL_ENV, base_url) for flag in HIDE_FLAGS: monkeypatch.setenv(flag, str(hide).lower()) requests = [] @@ -397,7 +439,8 @@ def handler(request): return httpx.Response(200, headers={"Content-Type": "text/event-stream"}, content=fake.stream(MODEL)) return httpx.Response(200, json=fake.completion(MODEL)) - with tracing(monkeypatch, f"lepton-url-privacy-{journey}-{hide}") as (receiver, provider): + host_label = "xenon" if base_url == XENON_TEST_BASE_URL else "invalid" + with tracing(monkeypatch, f"lepton-url-privacy-{host_label}-{journey}-{hide}") as (receiver, provider): with mock_client(handler) as client: if journey == "error": with pytest.raises(openai.AuthenticationError): @@ -409,11 +452,11 @@ def handler(request): _, values = one_span(receiver, provider) # Positive controls: the endpoint was used, and real telemetry was exported. assert len(requests) == 1 - assert str(requests[0].url) == TEST_BASE_URL + "/chat/completions" + assert str(requests[0].url) == base_url + "/chat/completions" assert receiver.requests()[0]["resource_attributes"] exported = telemetry(receiver) - assert TEST_BASE_URL not in exported - assert urlsplit(TEST_BASE_URL).hostname not in exported + assert base_url not in exported + assert urlsplit(base_url).hostname not in exported assert not {"server.address", "url.full", "http.url", "openai.base_url"}.intersection(values) if hide: assert "gen_ai.request.parameters" not in values @@ -469,7 +512,8 @@ def test_app_subprocess_is_loopback_only(monkeypatch, tmp_path, stream): @pytest.mark.parametrize("operation", ["dns", "connect", "connect_ex"]) -def test_guard_refuses_syntactic_host_before_dns(tmp_path, operation): +@pytest.mark.parametrize("host", [urlsplit(TEST_BASE_URL).hostname, urlsplit(XENON_TEST_BASE_URL).hostname]) +def test_guard_refuses_syntactic_host_before_dns(tmp_path, operation, host): with Receiver() as receiver, fake.FakeOpenAI() as server: environment = child_environment(tmp_path, receiver, server) # The positive control proves that the same guard permits a loopback request. @@ -478,9 +522,9 @@ def test_guard_refuses_syntactic_host_before_dns(tmp_path, operation): assert_guard_installed(environment) assert Path(environment["LOOPBACK_GUARD_LOG"]).read_text() == "" code = "import socket; " + { - "dns": "socket.getaddrinfo('endpoint.example.invalid', 443)", - "connect": "socket.socket().connect(('endpoint.example.invalid', 443))", - "connect_ex": "socket.socket().connect_ex(('endpoint.example.invalid', 443))", + "dns": f"socket.getaddrinfo({host!r}, 443)", + "connect": f"socket.socket().connect(({host!r}, 443))", + "connect_ex": f"socket.socket().connect_ex(({host!r}, 443))", }[operation] refused = run_child(environment, "-c", code) assert refused.returncode != 0 @@ -511,7 +555,14 @@ def forbidden(): assert VENDOR_KEY not in captured.out + captured.err -@pytest.mark.parametrize("url,reason", [REFUSED_URLS[0], REFUSED_URLS[4], REFUSED_URLS[-1]]) +@pytest.mark.parametrize("url,reason", [ + ("https://api.lepton.ai/v1", "Legacy"), + ("https://dashboard.dgxc-lepton.nvidia.com/v1", "Console"), + (quote(quote(app.DOCUMENTED_BASE_URL_FORMS[0], safe=""), safe=""), "placeholder"), + ("http://ws0000-example.xenon.lepton.run/v1", "https"), + ("https://ws0000-example.xenon.lepton.run", "append /v1"), + (XENON_TEST_BASE_URL + "?placeholder=query", "query or fragment"), +]) def test_main_refuses_url_without_tracing_or_network(monkeypatch, capsys, tmp_path, url, reason): with Receiver() as receiver, fake.FakeOpenAI() as server: monkeypatch.setenv("FI_BASE_URL", receiver.origin) @@ -538,6 +589,36 @@ def forbidden(): assert receiver.requests() == [] +@pytest.mark.parametrize("url", INVALID_HOST_URLS) +def test_main_rejects_invalid_hosts_without_disclosure(monkeypatch, capsys, tmp_path, url): + host = urlsplit(url).hostname + with Receiver() as receiver, fake.FakeOpenAI() as server: + monkeypatch.setenv("FI_BASE_URL", receiver.origin) + monkeypatch.setenv(app.BASE_URL_ENV, url) + + def forbidden(): + pytest.fail("Tracing started for an invalid host") + + monkeypatch.setattr(app, "setup_tracing", forbidden) + assert app.main([]) == 2 + captured = capsys.readouterr() + assert captured.err == "Set LEPTON_ENDPOINT_URL to a valid HTTP endpoint base URL.\n" + assert host not in captured.out + captured.err + assert "Traceback" not in captured.err + environment = child_environment(tmp_path, receiver, server) + environment[app.BASE_URL_ENV] = url + child = run_child(environment, str(RECIPE_DIR / "src" / "app.py")) + assert child.returncode == 2 + assert child.stderr.strip().endswith("Set LEPTON_ENDPOINT_URL to a valid HTTP endpoint base URL.") + assert host not in child.stdout + child.stderr + assert "Traceback" not in child.stderr + assert_guard_installed(environment) + assert Path(environment["LOOPBACK_GUARD_LOG"]).read_text() == "" + assert server.requests == [] + assert receiver.spans() == [] + assert receiver.requests() == [] + + def test_app_error_does_not_print_endpoint_url(tmp_path): with Receiver() as receiver, fake.FakeOpenAI(error=True) as server: environment = child_environment(tmp_path, receiver, server) @@ -555,14 +636,15 @@ def test_app_error_does_not_print_endpoint_url(tmp_path): assert spans[0]["status"]["code"] == "STATUS_CODE_ERROR" -def test_main_redacts_sdk_error_containing_endpoint_url(monkeypatch, capsys): +@pytest.mark.parametrize("error_type", [openai.OpenAIError, RuntimeError]) +def test_main_redacts_client_errors_containing_endpoint_url(monkeypatch, capsys, error_type): flushed = [] class Provider: def force_flush(self): flushed.append(True) - error = openai.OpenAIError(f"Request to {TEST_BASE_URL} failed using {VENDOR_KEY}") + error = error_type(f"Request to {TEST_BASE_URL} failed using {VENDOR_KEY}") assert TEST_BASE_URL in str(error) assert VENDOR_KEY in str(error) @@ -576,6 +658,8 @@ def failed_client(**kwargs): captured = capsys.readouterr() assert "Endpoint request failed" in captured.err assert TEST_BASE_URL not in captured.out + captured.err + assert urlsplit(TEST_BASE_URL).hostname not in captured.out + captured.err + assert "Traceback" not in captured.err assert VENDOR_KEY not in captured.out + captured.err assert flushed == [True] @@ -586,10 +670,15 @@ def test_readme_and_requirements_pin_customer_contract(): *app.DOCUMENTED_BASE_URL_FORMS, app.API_KEY_ENV, app.BASE_URL_ENV, app.MODEL_ENV, "FI_API_KEY", "FI_SECRET_KEY", "FI_BASE_URL", "FI_HIDE_INPUTS", "FI_HIDE_OUTPUTS", "FI_HIDE_LLM_INVOCATION_PARAMETERS", "provider field says `openai`", - "stop if the API tab shows no OpenAI-compatible base URL", "anyone with the URL", - "treat the URL like a secret", "never commit it", "Do not guess `/v1`", + "If the endpoint is not OpenAI-compatible, stop", "anyone with the URL", + "treat the URL like a secret", "never commit it", "Append `/v1` yourself", + "https://-.xenon.lepton.run", "lep endpoint get -n ", + "This recipe requires a non-empty `LEPTON_API_TOKEN`", "as NVIDIA recommends", + "lower-cases the scheme and host (the path keeps its case)", + "put `src/` on `PYTHONPATH` or run the snippet from `src/`", + "HTTPS", "credentials", XENON_TEST_BASE_URL, "gen_ai.request.model", "endpoint URL and host are absent", - "syntactic test host", "api.lepton.ai", "*.lepton.run", "leptonai", + "syntactic test host", "api.lepton.ai", "llm.lepton.run", "sdxl.lepton.run", "leptonai", "NVIDIA's Python SDK", "Non-OpenAI-compatible endpoints", 'PYTHONPATH="python/examples/lepton/src:python:python/frameworks/openai:python/tests"', 'PYTHONPATH="python/examples/lepton/src:python/tests"', @@ -599,6 +688,10 @@ def test_readme_and_requirements_pin_customer_contract(): ): assert text in readme assert "TBD" not in readme + assert "*.lepton.run" not in readme + assert "Do not guess `/v1`" not in readme + assert "The OpenAI SDK requires a non-empty API key" not in readme + assert "normalizes URL casing" not in readme assert "sk-" not in readme pins = (RECIPE_DIR / "requirements.txt").read_text().splitlines() assert pins[0].startswith("#") From 763f7a8f25828220219cfa4593b1a5025e40b1f3 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Sat, 10 Oct 2026 01:41:59 -0700 Subject: [PATCH 3/5] test(examples): pin a served model that differs from the request The chat fixture echoed the requested example id, so the span check could not show which field the instrumentor stored. One case now returns a synthetic served id and checks the OTLP export. NVIDIA legacy-host exclusions are unchanged and were not copied onto other recipes. No live Lepton call. RED: response.model was the requested example id. GREEN: the new case passes on the installed traceAI-openai 0.1.10 wheel. approved: Nikhil 2026-10-03 blanket Refs: TH-8318 --- .../lepton/tests/test_lepton_recipe.py | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/python/examples/lepton/tests/test_lepton_recipe.py b/python/examples/lepton/tests/test_lepton_recipe.py index cff82f5a3..8d224562f 100644 --- a/python/examples/lepton/tests/test_lepton_recipe.py +++ b/python/examples/lepton/tests/test_lepton_recipe.py @@ -35,6 +35,9 @@ def load_module(name, path): TEST_BASE_URL = "https://endpoint.example.invalid/v1" XENON_TEST_BASE_URL = "https://ws0000-example.xenon.lepton.run/v1" MODEL = "nvidia/Nemotron-Research-Reasoning-Qwen-1.5B" +# Synthetic served id. NVIDIA docs do not publish a rewrite of this example model. +# No live Lepton call. Host exclusions are unchanged. +RESPONSE_MODEL = "fixture-served/not-a-live-lepton-model" VENDOR_KEY = "placeholder-lepton-key" FI_KEY = "placeholder-futureagi-key" FI_SECRET = "placeholder-futureagi-secret" @@ -290,6 +293,33 @@ def test_refused_base_urls_name_reason_without_disclosing_url(url, reason): assert "placeholder-password" not in message +def test_response_model_differs_from_request(monkeypatch): + """Span stores the served model, not the requested example id.""" + assert RESPONSE_MODEL != MODEL + monkeypatch.setenv(app.BASE_URL_ENV, TEST_BASE_URL) + requests = [] + + def handler(request): + requests.append(request) + return httpx.Response(200, json=fake.completion(RESPONSE_MODEL)) + + with tracing(monkeypatch, "lepton-response-model") as (receiver, provider): + with mock_client(handler) as client: + response = chat(client) + span, values = one_span(receiver, provider) + assert response.model == RESPONSE_MODEL + assert json.loads(requests[0].content)["model"] == MODEL + assert values["gen_ai.request.model"] == RESPONSE_MODEL + exports = receiver.requests() + assert exports and exports[0]["path"] == "/tracer/v1/traces" + assert exports[0]["headers"]["x-api-key"] == FI_KEY + body = fake.completion(RESPONSE_MODEL) + assert body["object"] == "chat.completion" + assert body["model"] == RESPONSE_MODEL + assert set(body["usage"]) == {"prompt_tokens", "completion_tokens", "total_tokens"} + assert TEST_BASE_URL not in telemetry(receiver) + + @pytest.mark.parametrize("key", [None, "", " "]) def test_make_client_requires_nonempty_vendor_key(monkeypatch, key): if key is None: From b353a1a95c1f55bb26141815ddbd71320848f6da Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Sun, 11 Oct 2026 21:32:30 -0700 Subject: [PATCH 4/5] test(examples/lepton): accept traceai-openai before and after the provider/model fix The recipe pinned traceai-openai 0.1.10 behaviour: gen_ai.provider.name is always "openai" and streamed/failed spans carry no gen_ai.request.model. future-agi/traceAI#249 labels known vendor hosts and records the requested model, so these pins would break whichever PR merged second. Accept both behaviours (and the related Responses/embedding output changes where this recipe checks them), and say in the README which version does what. Passes against published traceAI-openai 0.1.10 and against #249. Co-Authored-By: Claude Opus 5.5 --- python/examples/lepton/README.md | 4 ++-- .../lepton/tests/test_lepton_recipe.py | 22 ++++++++++++------- 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/python/examples/lepton/README.md b/python/examples/lepton/README.md index bd7c5ca9c..8b7d2b8dc 100644 --- a/python/examples/lepton/README.md +++ b/python/examples/lepton/README.md @@ -76,11 +76,11 @@ The app validates the URL, model and non-empty token before registering tracing. - One `ChatCompletion` span with `gen_ai.span.kind=LLM` per tested call. - For a non-streamed response, `gen_ai.request.model` is the model id the provider returns in its response. -- Current `traceai-openai` leaves `gen_ai.request.model` absent on streamed and failed calls. The requested model remains in the `gen_ai.request.parameters` JSON unless `FI_HIDE_LLM_INVOCATION_PARAMETERS=true`. +- `traceai-openai` 0.1.10 and earlier leave `gen_ai.request.model` absent on streamed and failed calls; later versions (future-agi/traceAI#249) record the requested model there. The requested model remains in the `gen_ai.request.parameters` JSON unless `FI_HIDE_LLM_INVOCATION_PARAMETERS=true`. - Token usage is present only when the response includes usage. Missing usage is omitted, never filled with zero. - Streaming text is accumulated by the same instrumentor. The default fixture exports no usage attributes. With `stream_options={"include_usage": True}` and a final usage chunk, the fixture exports input, output and total token counts. Your endpoint must support this option to use it. - Request errors are recorded as span errors with an exception event. -- The provider field says `openai`. This is the shared instrumentor's label, including for a Lepton endpoint. +- Up to 0.1.10, the provider field says `openai`. This is the shared instrumentor's label, including for a Lepton endpoint. Later versions say `lepton` for `*.xenon.lepton.run` hosts; other base URLs keep `openai`. The span resource uses the project name passed to `register()`. diff --git a/python/examples/lepton/tests/test_lepton_recipe.py b/python/examples/lepton/tests/test_lepton_recipe.py index 8d224562f..471531242 100644 --- a/python/examples/lepton/tests/test_lepton_recipe.py +++ b/python/examples/lepton/tests/test_lepton_recipe.py @@ -19,6 +19,12 @@ from harness import Receiver from traceai_openai import OpenAIInstrumentor +# traceai-openai 0.1.10 and earlier label every span "openai"; later versions +# (future-agi/traceAI#249) use the vendor's name for its documented API host. +# They also omit gen_ai.request.model on streamed and failed calls, where later +# versions record the requested model. Accept both so this recipe passes on either version. +PROVIDER_NAMES = ("openai", "lepton") + TEST_DIR = Path(__file__).resolve().parent RECIPE_DIR = TEST_DIR.parent @@ -175,7 +181,7 @@ def one_span(receiver, provider): assert spans[0]["name"] == "ChatCompletion" values = attributes(spans[0]) assert values["gen_ai.span.kind"] == "LLM" - assert values["gen_ai.provider.name"] == "openai" + assert values["gen_ai.provider.name"] in PROVIDER_NAMES return spans[0], values @@ -394,8 +400,8 @@ def handler(request): assert text == fake.ANSWER _, values = one_span(receiver, provider) assert values["output.value"] == fake.ANSWER - # traceai-openai records the model only from a non-streamed response; flip this when the instrumentor records the request model - assert "gen_ai.request.model" not in values + # 0.1.10 records the model only from non-streamed responses; later versions record the requested one. + assert values.get("gen_ai.request.model") in (None, MODEL) assert json.loads(values["gen_ai.request.parameters"])["model"] == MODEL usage = {key: int(value) for key, value in values.items() if key.startswith("gen_ai.usage.")} assert usage == ({"gen_ai.usage.input_tokens": 11, "gen_ai.usage.output_tokens": 13, "gen_ai.usage.total_tokens": 24} if include_usage else {}) @@ -421,8 +427,8 @@ def handler(request): assert len(exceptions) == 1 assert "Unauthorized" in json.dumps(exceptions) assert VENDOR_KEY not in str(caught.value) - # traceai-openai records the model only from a non-streamed response; flip this when the instrumentor records the request model - assert "gen_ai.request.model" not in values + # 0.1.10 records the model only from non-streamed responses; later versions record the requested one. + assert values.get("gen_ai.request.model") in (None, MODEL) assert json.loads(values["gen_ai.request.parameters"])["model"] == MODEL assert not any(key.startswith("gen_ai.usage.") for key in values) assert len(requests) == 1 @@ -524,11 +530,11 @@ def test_app_subprocess_is_loopback_only(monkeypatch, tmp_path, stream): assert spans[0]["name"] == "ChatCompletion" values = attributes(spans[0]) assert values["gen_ai.span.kind"] == "LLM" - assert values["gen_ai.provider.name"] == "openai" + assert values["gen_ai.provider.name"] in PROVIDER_NAMES assert fake.ANSWER in values["output.value"] if stream: - # traceai-openai records the model only from a non-streamed response; flip this when the instrumentor records the request model - assert "gen_ai.request.model" not in values + # 0.1.10 records the model only from non-streamed responses; later versions record the requested one. + assert values.get("gen_ai.request.model") in (None, MODEL) assert json.loads(values["gen_ai.request.parameters"])["model"] == MODEL else: assert values["gen_ai.request.model"] == MODEL From aaba2960e9a0707c378226466695d0a2b7c6a456 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek <8347993+nik13@users.noreply.github.com> Date: Tue, 27 Oct 2026 21:14:04 -0700 Subject: [PATCH 5/5] docs(lepton): align recipe with official API contracts Keep endpoint, model and auth guidance factual. Add exact-head offline recipe CI for repository source, the SDK floor and published packages. Co-Authored-By: Claude Code --- .../openai-compatible-recipe-tests.yml | 106 ++++++++++++++++++ python/examples/lepton/README.md | 28 +++-- python/examples/lepton/requirements.txt | 2 +- python/examples/lepton/src/app.py | 6 +- .../lepton/tests/test_lepton_recipe.py | 10 +- 5 files changed, 137 insertions(+), 15 deletions(-) create mode 100644 .github/workflows/openai-compatible-recipe-tests.yml diff --git a/.github/workflows/openai-compatible-recipe-tests.yml b/.github/workflows/openai-compatible-recipe-tests.yml new file mode 100644 index 000000000..0e2587e85 --- /dev/null +++ b/.github/workflows/openai-compatible-recipe-tests.yml @@ -0,0 +1,106 @@ +name: OpenAI-compatible recipe tests + +on: + pull_request: + branches: [main, dev] + paths: + - 'python/examples/**' + - 'python/fi_instrumentation/**' + - 'python/frameworks/openai/**' + - 'python/tests/harness/**' + - '.github/workflows/openai-compatible-recipe-tests.yml' + push: + branches: [dev] + paths: + - 'python/examples/**' + - 'python/fi_instrumentation/**' + - 'python/frameworks/openai/**' + - 'python/tests/harness/**' + - '.github/workflows/openai-compatible-recipe-tests.yml' + workflow_dispatch: + +permissions: + contents: read + +jobs: + plan: + name: Recipe plan + runs-on: ubuntu-24.04 + outputs: + matrix: ${{ steps.plan.outputs.matrix }} + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + ref: ${{ github.event.pull_request.head.sha || github.sha }} + persist-credentials: false + - id: plan + run: | + python3 - <<'PY' + import json, os + from pathlib import Path + matrix = [] + for recipe in ('databricks', 'lepton', 'orcarouter', 'anannas-ai', 'doubleword'): + root = Path('python/examples') / recipe + if not root.exists(): + continue # Recipes may land independently. + assert (root / 'src/app.py').is_file(), f'{recipe}: missing app' + assert (root / 'requirements.txt').is_file(), f'{recipe}: missing requirements' + assert list((root / 'tests').glob('test_*.py')), f'{recipe}: missing tests' + for sdk, mode in (('3.24.0', 'source'), ('1.69.0', 'source'), ('3.24.0', 'published')): + matrix.append({'recipe': recipe, 'sdk': sdk, 'mode': mode}) + assert matrix, 'No recipe tests found' + with open(os.environ['GITHUB_OUTPUT'], 'a') as out: + out.write('matrix=' + json.dumps({'include': matrix}) + '\n') + PY + recipe: + name: Recipe (${{ matrix.recipe }}, openai ${{ matrix.sdk }}, ${{ matrix.mode }}) + needs: plan + runs-on: ubuntu-24.04 + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.plan.outputs.matrix) }} + env: + PYTHONDONTWRITEBYTECODE: '1' + FI_BASE_URL: http://127.0.0.1:9 + FI_GRPC_URL: http://127.0.0.1:9 + RECIPE: ${{ matrix.recipe }} + SDK_VERSION: ${{ matrix.sdk }} + PACKAGE_MODE: ${{ matrix.mode }} + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + ref: ${{ github.event.pull_request.head.sha || github.sha }} + persist-credentials: false + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: '3.11.9' + - name: Install SDK and harness dependencies + run: | + python -m pip install "openai==$SDK_VERSION" httpx 'wrapt<2' \ + opentelemetry-api opentelemetry-sdk opentelemetry-instrumentation \ + opentelemetry-exporter-otlp-proto-http requests protobuf \ + opentelemetry-proto pydantic jsonschema pytest + if [ "$PACKAGE_MODE" = published ]; then + python -m pip install 'traceAI-openai==0.1.10' 'fi-instrumentation-otel==1.1.0' + fi + - name: Run isolated recipe contracts + run: | + if [ "$PACKAGE_MODE" = published ]; then + export PYTHONPATH="python/examples/$RECIPE/src:python/tests" + else + export PYTHONPATH="python/examples/$RECIPE/src:python:python/frameworks/openai:python/tests" + fi + python -m pytest "python/examples/$RECIPE/tests" -q \ + --noconftest -p no:cacheprovider -o addopts='' -rfEs + result: + name: OpenAI-compatible recipes + if: always() + needs: [plan, recipe] + runs-on: ubuntu-24.04 + steps: + - name: Require every planned recipe contract + env: + PLAN_RESULT: ${{ needs.plan.result }} + RECIPE_RESULT: ${{ needs.recipe.result }} + run: test "$PLAN_RESULT" = success && test "$RECIPE_RESULT" = success diff --git a/python/examples/lepton/README.md b/python/examples/lepton/README.md index 8b7d2b8dc..4697536e1 100644 --- a/python/examples/lepton/README.md +++ b/python/examples/lepton/README.md @@ -28,8 +28,9 @@ There is no shared public base URL. The endpoint URL from the API tab, or `lep e | `LEPTON_ENDPOINT_URL` | Yes | Endpoint URL from the endpoint details page, API tab, plus `/v1`. No default. | | `LEPTON_MODEL` | Yes, unless `--model` is supplied | Model served by your endpoint. | | `FI_BASE_URL` | No | Override the Future AGI collector origin; used by the tracer only. | +| `FI_PROJECT_NAME` | No | Observe project name; defaults to `lepton-openai-recipe`. | -Missing values and refused URLs exit with code 2 before tracing starts. This recipe requires a non-empty `LEPTON_API_TOKEN`. Create the endpoint with an access token, as NVIDIA recommends over making it public. Use the token the API tab shows. +Missing values and refused URLs exit with code 2 before tracing starts. This recipe requires a non-empty `LEPTON_API_TOKEN`. As a security choice for this recipe, create a token-protected endpoint rather than a public one; NVIDIA documents both access modes. Use the token the API tab shows. ## Run @@ -46,7 +47,7 @@ python src/app.py python src/app.py --stream ``` -You can also supply `--model` and `--prompt`. The example model is the model in NVIDIA's endpoint tutorial. Set the name to the model your endpoint actually serves. +You can also supply `--model` and `--prompt`. The example model is the model in NVIDIA's endpoint tutorial. Set the name to the model your endpoint actually serves; list it with `client.models.list()` using the same configured client. The SDK sends the endpoint token as `Authorization: Bearer `. The recipe environment variable names are application configuration, not NVIDIA SDK defaults. ## Code @@ -74,7 +75,7 @@ The app validates the URL, model and non-empty token before registering tracing. ## What you see in Future AGI -- One `ChatCompletion` span with `gen_ai.span.kind=LLM` per tested call. +- One `ChatCompletion` span with `gen_ai.span.kind=LLM` per Chat Completions call. - For a non-streamed response, `gen_ai.request.model` is the model id the provider returns in its response. - `traceai-openai` 0.1.10 and earlier leave `gen_ai.request.model` absent on streamed and failed calls; later versions (future-agi/traceAI#249) record the requested model there. The requested model remains in the `gen_ai.request.parameters` JSON unless `FI_HIDE_LLM_INVOCATION_PARAMETERS=true`. - Token usage is present only when the response includes usage. Missing usage is omitted, never filled with zero. @@ -90,9 +91,9 @@ Lepton here means [NVIDIA DGX Cloud Lepton endpoints](https://docs.nvidia.com/dg Copy the endpoint URL and API key from the endpoint details page, in its [API tab](https://docs.nvidia.com/dgx-cloud/lepton/features/endpoints/create-llm/). You can also run `lep endpoint get -n ` and read `external_endpoint`. The endpoint URL looks like `https://-.xenon.lepton.run`. Append `/v1` yourself when setting `LEPTON_ENDPOINT_URL`: NVIDIA's [LLM endpoint example](https://docs.nvidia.com/dgx-cloud/lepton/examples/endpoint/deploy-gpt-oss/) uses `/v1/chat/completions`. If the endpoint is not OpenAI-compatible, stop; this recipe does not apply. -`check_base_url()` returns an allowed URL exactly as given; it never appends `/v1`. For hosts under `*.xenon.lepton.run`, it requires HTTPS, rejects a host-only URL or `/` path, and refuses query strings and fragments. For every host, it refuses non-ASCII or malformed IDNA hostnames and credentials in the URL. +`check_base_url()` returns an allowed URL exactly as given; it never appends `/v1`. For hosts under `*.xenon.lepton.run`, it requires HTTPS and exactly `/v1` or `/v1/`, rejects endpoint paths such as `/v1/chat/completions`, and refuses query strings and fragments. For every host, it refuses non-ASCII or malformed IDNA hostnames and credentials in the URL. -The tested OpenAI SDK appends a trailing slash to a non-empty base path and lower-cases the scheme and host (the path keeps its case). For the syntactic test host `https://ws0000-example.xenon.lepton.run/v1`, the client base URL is `https://ws0000-example.xenon.lepton.run/v1/` and the request URL is `https://ws0000-example.xenon.lepton.run/v1/chat/completions`. The suite also keeps `https://endpoint.example.invalid/v1` as a syntactic test host on the reserved `.invalid` domain. Neither test host is contacted; both use in-process fixtures. +The OpenAI SDK appends a trailing slash to a non-empty base path and lower-cases the scheme and host (the path keeps its case). For the syntactic test host `https://ws0000-example.xenon.lepton.run/v1`, the client base URL is `https://ws0000-example.xenon.lepton.run/v1/` and the request URL is `https://ws0000-example.xenon.lepton.run/v1/chat/completions`. The suite also keeps `https://endpoint.example.invalid/v1` as a syntactic test host on the reserved `.invalid` domain. Neither test host is contacted; both use in-process fixtures. Endpoints can be public and reachable by anyone with the URL. For a public endpoint, treat the URL like a secret and never commit it. The app does not print endpoint URLs in its configuration or request error messages. Legacy Lepton AI hosts and console hosts are refused even with upper-case or trailing-dot spellings. Literal and repeatedly percent-encoded placeholders are refused. @@ -100,7 +101,7 @@ Endpoints can be public and reachable by anyone with the URL. For a public endpo Set `FI_HIDE_INPUTS=true` to keep prompt text out of exported span attributes. Set `FI_HIDE_OUTPUTS=true` to keep response text out. You can also pass `TraceConfig(hide_inputs=True)` to the instrumentor. The endpoint still receives the prompt. Future AGI masking does not change the provider's own logging. -In the tested chat, streaming and HTTP 401 fixtures, the endpoint URL and host are absent from exported span attributes, events, status and resource values. This holds with the `FI_HIDE_*` flags enabled or disabled. The instrumentor does not export `server.address`, `url.full` or a base-URL attribute in these cases. This observation covers this recipe's instrumentor; extra HTTP instrumentation or logging has its own behavior. +In the chat, streaming and HTTP 401 fixtures, the endpoint URL and host are absent from exported span attributes, events, status and resource values. This holds with the `FI_HIDE_*` flags enabled or disabled. The instrumentor does not export `server.address`, `url.full` or a base-URL attribute in these cases. This observation covers this recipe's instrumentor; extra HTTP instrumentation or logging has its own behavior. The vendor token is also absent from the exported spans, resources and collector headers. Future AGI keys are used only for the collector. @@ -111,7 +112,6 @@ The vendor token is also absent from the exported spans, resources and collector - The `leptonai` photon client. - NVIDIA's Python SDK. It is a workspace control-plane client for batch jobs, endpoints and secrets, not an inference client. - Non-OpenAI-compatible endpoints, endpoint deployment, and APIs other than Chat Completions. -- Live endpoint calls. This recipe is not tested against the live provider (no paid call); the tests use a local fake of the OpenAI API and in-process fixtures. ## Tests @@ -144,8 +144,14 @@ env -u PYTHONPATH PYTHONDONTWRITEBYTECODE=1 \ For the floor row, change `openai==3.24.0` to `openai==1.69.0` in the repository-source command. Tests use `httpx.MockTransport` at the syntactic test host and loopback subprocesses protected by a socket guard. No provider host is called. -| Python | `openai` | traceAI packages | Result | +| Python | `openai` | traceAI packages | Local check configuration | |---|---|---|---| -| 3.10, 3.11, 3.12, 3.13 | 3.24.0 | `traceai-openai` and `fi_instrumentation` from this repository | Full suite | -| 3.11 | 1.69.0 (the `traceai-openai` floor) | from this repository | Full suite | -| 3.11 | 3.24.0 | published `traceAI-openai==0.1.10` and `fi-instrumentation-otel==1.1.0` from PyPI | Full suite | +| 3.10, 3.11, 3.12, 3.13 | 3.24.0 | `traceai-openai` and `fi_instrumentation` from this repository | Commands above | +| 3.11 | 1.69.0 (the `traceai-openai` floor) | from this repository | Commands above | +| 3.11 | 3.24.0 | published `traceAI-openai==0.1.10` and `fi-instrumentation-otel==1.1.0` from PyPI | Commands above | + +## Official API references + +- [Use an endpoint and list its models](https://docs.nvidia.com/dgx-cloud/lepton/get-started/endpoint/) +- [Endpoint access control](https://docs.nvidia.com/dgx-cloud/lepton/features/endpoints/configurations/) +- [OpenAI-compatible inference example](https://docs.nvidia.com/dgx-cloud/lepton/examples/endpoint/deploy-gpt-oss/) diff --git a/python/examples/lepton/requirements.txt b/python/examples/lepton/requirements.txt index 9ef38df9d..19274c298 100644 --- a/python/examples/lepton/requirements.txt +++ b/python/examples/lepton/requirements.txt @@ -1,4 +1,4 @@ -# Tested SDK pin and published traceAI package pins. +# OpenAI SDK and published traceAI package pins. openai==3.24.0 traceAI-openai==0.1.10 fi-instrumentation-otel==1.1.0 diff --git a/python/examples/lepton/src/app.py b/python/examples/lepton/src/app.py index fe68dbe66..c86b866f8 100644 --- a/python/examples/lepton/src/app.py +++ b/python/examples/lepton/src/app.py @@ -26,6 +26,8 @@ def check_base_url(url: str) -> str: decoded = next_value if any(marker in decoded for marker in ("<", ">", "ENDPOINT_URL")): raise ValueError(f"Replace the {BASE_URL_ENV} placeholder using the API tab.") + if any(char.isspace() or not char.isprintable() for char in url): + raise ValueError(f"Set {BASE_URL_ENV} to a valid HTTP endpoint base URL.") from None try: parsed = urlsplit(url) host = parsed.hostname or "" @@ -50,6 +52,8 @@ def check_base_url(url: str) -> str: raise ValueError("append /v1 to the endpoint URL (NVIDIA documents /v1/chat/completions)") if "?" in url or "#" in url: raise ValueError("Remove the query or fragment from the endpoint base URL.") + if parsed.path not in ("/v1", "/v1/"): + raise ValueError("Use the endpoint base path /v1, not an endpoint or another path.") return url @@ -69,7 +73,7 @@ def make_client( def setup_tracing(project_name: str | None = None): provider = register( - project_name=project_name or "lepton-openai-recipe", + project_name=project_name or os.environ.get("FI_PROJECT_NAME") or "lepton-openai-recipe", project_type=ProjectType.OBSERVE, set_global_tracer_provider=False, verbose=False, diff --git a/python/examples/lepton/tests/test_lepton_recipe.py b/python/examples/lepton/tests/test_lepton_recipe.py index 471531242..7ed9d19a3 100644 --- a/python/examples/lepton/tests/test_lepton_recipe.py +++ b/python/examples/lepton/tests/test_lepton_recipe.py @@ -42,7 +42,7 @@ def load_module(name, path): XENON_TEST_BASE_URL = "https://ws0000-example.xenon.lepton.run/v1" MODEL = "nvidia/Nemotron-Research-Reasoning-Qwen-1.5B" # Synthetic served id. NVIDIA docs do not publish a rewrite of this example model. -# No live Lepton call. Host exclusions are unchanged. +# Keep the model-resolution control synthetic and distinct from the request. RESPONSE_MODEL = "fixture-served/not-a-live-lepton-model" VENDOR_KEY = "placeholder-lepton-key" FI_KEY = "placeholder-futureagi-key" @@ -98,6 +98,9 @@ def load_module(name, path): ("https://ws0000-example.xenon.lepton.run", "append /v1"), ("https://ws0000-example.xenon.lepton.run/", "append /v1"), ("https://WS0000-EXAMPLE.XENON.LEPTON.RUN.", "append /v1"), + (XENON_TEST_BASE_URL + "/chat/completions", "base path /v1"), + (XENON_TEST_BASE_URL + "/models", "base path /v1"), + ("https://ws0000-example.xenon.lepton.run/api/v1", "base path /v1"), (XENON_TEST_BASE_URL + "?placeholder=query", "query or fragment"), (XENON_TEST_BASE_URL + "#placeholder-fragment", "query or fragment"), (XENON_TEST_BASE_URL + "?", "query or fragment"), @@ -597,6 +600,9 @@ def forbidden(): (quote(quote(app.DOCUMENTED_BASE_URL_FORMS[0], safe=""), safe=""), "placeholder"), ("http://ws0000-example.xenon.lepton.run/v1", "https"), ("https://ws0000-example.xenon.lepton.run", "append /v1"), + (XENON_TEST_BASE_URL + "/chat/completions", "base path /v1"), + (XENON_TEST_BASE_URL + "/models", "base path /v1"), + ("https://ws0000-example.xenon.lepton.run/api/v1", "base path /v1"), (XENON_TEST_BASE_URL + "?placeholder=query", "query or fragment"), ]) def test_main_refuses_url_without_tracing_or_network(monkeypatch, capsys, tmp_path, url, reason): @@ -709,7 +715,7 @@ def test_readme_and_requirements_pin_customer_contract(): "If the endpoint is not OpenAI-compatible, stop", "anyone with the URL", "treat the URL like a secret", "never commit it", "Append `/v1` yourself", "https://-.xenon.lepton.run", "lep endpoint get -n ", - "This recipe requires a non-empty `LEPTON_API_TOKEN`", "as NVIDIA recommends", + "This recipe requires a non-empty `LEPTON_API_TOKEN`", "NVIDIA documents both access modes", "lower-cases the scheme and host (the path keeps its case)", "put `src/` on `PYTHONPATH` or run the snippet from `src/`", "HTTPS", "credentials", XENON_TEST_BASE_URL,