From bc97bba652c93ee80bf306bf73b299f56bb9cc16 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Mon, 5 Oct 2026 23:44:15 -0700 Subject: [PATCH 1/7] feat(examples): Doubleword recipe on traceai-openai Adds python/examples/doubleword/: the official openai SDK pointed at Doubleword's OpenAI-compatible base URL https://api.doubleword.ai/v1, traced by the existing traceai-openai instrumentor. No provider package. - DOUBLEWORD_API_KEY and a model (DOUBLEWORD_MODEL or --model) are required; main() exits 2 before tracing on missing values or a refused DOUBLEWORD_BASE_URL. URLs are validated, never rewritten. - On api.doubleword.ai: https only, exactly /v1 or /v1/, no query or fragment delimiter; the Batch API (/batch, /v1/batch...), the Anthropic Messages API (/messages, /v1/messages) and endpoint paths are refused with specific messages. Everywhere: whitespace/control characters, non-ASCII or malformed hosts and URL credentials are refused. Customer proxies and loopback allowed. - Realtime and the async tier: service_tier="flex" (a native SDK parameter, branched on the installed SDK version) reaches the request body and gen_ai.request.parameters. - Pins current traceai-openai behaviour: a successful non-streamed span records the model id the provider returns, the requested id stays in gen_ai.request.parameters; streamed and failed spans omit the model. - Embeddings: the span exports the raw response JSON (vectors included) under embedding.embeddings, which FI_HIDE_EMBEDDING_VECTORS does not mask; pinned and documented. - Chat Completions output.value is the assistant text; FI_HIDE_INPUTS / FI_HIDE_OUTPUTS with controls. The recipe keeps the SDK's default retries; error tests disable them on the test client. - Key only on the OpenAI client; Future AGI keys only on the tracer. - Loopback only: MockTransport at the documented host, a local OpenAI-shaped fake, the shared harness receiver and a socket guard with a loopback control. approved: Nikhil 2026-10-03 blanket Refs: TH-8309 --- python/examples/doubleword/README.md | 154 +++++ python/examples/doubleword/requirements.txt | 4 + python/examples/doubleword/src/app.py | 130 ++++ .../examples/doubleword/tests/_fake_openai.py | 126 ++++ .../tests/loopback_guard/sitecustomize.py | 44 ++ .../tests/test_doubleword_recipe.py | 629 ++++++++++++++++++ 6 files changed, 1087 insertions(+) create mode 100644 python/examples/doubleword/README.md create mode 100644 python/examples/doubleword/requirements.txt create mode 100644 python/examples/doubleword/src/app.py create mode 100644 python/examples/doubleword/tests/_fake_openai.py create mode 100644 python/examples/doubleword/tests/loopback_guard/sitecustomize.py create mode 100644 python/examples/doubleword/tests/test_doubleword_recipe.py diff --git a/python/examples/doubleword/README.md b/python/examples/doubleword/README.md new file mode 100644 index 000000000..39b51d53e --- /dev/null +++ b/python/examples/doubleword/README.md @@ -0,0 +1,154 @@ +# Doubleword (OpenAI-compatible) with traceAI + +Trace Chat Completions through the official `openai` Python SDK at Doubleword's documented base URL, `https://api.doubleword.ai/v1` ([API reference](https://doubleword.ai/llms.txt)). This recipe uses `traceai-openai`; it needs no Doubleword-specific tracing package. + +## Install + +From this directory, install the tested pins: + +```bash +pip install -r requirements.txt +``` + +The plain install form is: + +```bash +pip install traceAI-openai fi-instrumentation-otel openai +``` + +## Configure + +| Variable | Required | Purpose | +|---|---|---| +| `FI_API_KEY` | For Future AGI export | Future AGI API key; tracer only. | +| `FI_SECRET_KEY` | For Future AGI export | Future AGI secret key; tracer only. | +| `DOUBLEWORD_API_KEY` | Yes | Doubleword key; OpenAI client only. Create it on the [API Keys page](https://app.doubleword.ai/api-keys). | +| `DOUBLEWORD_MODEL` | Yes, unless `--model` is supplied | Pick an available model from Doubleword's catalog using its [API reference](https://doubleword.ai/llms.txt). | +| `DOUBLEWORD_BASE_URL` | No | Defaults to `https://api.doubleword.ai/v1`; optional proxy override. | +| `FI_BASE_URL` | No | Optional Future AGI collector origin; tracer only. | + +The application exits 2 for a missing or empty `DOUBLEWORD_API_KEY`, a missing or empty `DOUBLEWORD_MODEL` without `--model`, or an invalid `DOUBLEWORD_BASE_URL`. It checks these before tracing or network calls. The recipe does not check the Future AGI keys itself. + +On `api.doubleword.ai`, the URL must use HTTPS and the path must be exactly `/v1` or `/v1/`. Endpoint paths, queries, fragments, URL credentials, whitespace, control characters, and invalid host spellings fail early. Uppercase hosts and a trailing DNS dot do not bypass these checks. Customer proxies and loopback servers can use their own HTTP(S) paths. Validation returns allowed URLs unchanged; the OpenAI SDK adds a trailing slash to `client.base_url`. + +## Run + +Run from this directory. Replace the model placeholder with an available catalog id; it is not a callable model name. + +```bash +export FI_API_KEY="placeholder-futureagi-api-key" +export FI_SECRET_KEY="placeholder-futureagi-secret-key" +export DOUBLEWORD_API_KEY="placeholder-doubleword-key" +export DOUBLEWORD_MODEL="placeholder-catalog-model" +export DOUBLEWORD_BASE_URL="https://api.doubleword.ai/v1" +python src/app.py +python src/app.py --stream +``` + +You can also pass `--model` and `--prompt`. The project name defaults to `doubleword-example`. + +## Code + +The core flow below matches `src/app.py`. The application validates the URL, model, and Doubleword key before this flow. Instrument before creating the client. Future AGI credentials stay in the tracer environment. + +```python +import os +from fi_instrumentation import register +from fi_instrumentation.fi_types import ProjectType +from openai import OpenAI +from traceai_openai import OpenAIInstrumentor + +provider = register(project_name="doubleword-example", project_type=ProjectType.OBSERVE, + set_global_tracer_provider=False, verbose=False) +OpenAIInstrumentor().instrument(tracer_provider=provider) +with OpenAI(base_url="https://api.doubleword.ai/v1", api_key=os.environ["DOUBLEWORD_API_KEY"]) as client: + response = client.chat.completions.create(model=os.environ["DOUBLEWORD_MODEL"], + messages=[{"role": "user", "content": "What is a rainbow?"}]) + print(response.choices[0].message.content or "") +provider.force_flush() +``` + +## What you see in Future AGI + +The fixtures export one `ChatCompletion` span with `gen_ai.span.kind=LLM` per completed chat call. The provider field says `openai`: `gen_ai.provider.name` is the shared instrumentor's label, including when the client uses Doubleword's host. + +Current `traceai-openai` behavior: + +- Non-streamed `gen_ai.request.model` is the model id the provider returns in its response. `gen_ai.request.parameters` contains the model id sent in the request. The fixtures deliberately return a different, synthetic dated id so this distinction is tested. +- Streamed and failed calls omit `gen_ai.request.model`. The requested model remains in the `gen_ai.request.parameters` JSON unless invocation parameters are hidden. +- Chat `output.value` contains the assistant content. Streams accumulate that content into one span. A stream with no output text exports an empty string, with no raw-response fallback. Extra response metadata is not copied into chat `output.value`. +- The tests pin `gen_ai.usage.input_tokens`, `gen_ai.usage.output_tokens`, and `gen_ai.usage.total_tokens`. Missing usage is omitted, never set to zero. Default stream fixtures omit usage; `stream_options={"include_usage": True}` with a final usage chunk exports all three counts. +- The 401 and 429 fixtures produce ERROR spans with exception events. The recipe keeps the SDK's default retry policy; the error tests disable retries so each fixture sees one request. + +Cost is unknown when the pricing table has no matching model. If a provider resolves an alias to a different response id, cost lookup keyed on one name may miss the other. These fixtures demonstrate that distinction; they do not establish Doubleword's model-resolution behavior or rates. + +## Provider specifics + +Doubleword documents the following tiers in its [API reference](https://doubleword.ai/llms.txt): + +| Tier | What this recipe covers | +|---|---| +| Realtime | OpenAI-compatible request and stream fixtures. Doubleword says the self-serve endpoint is "designed for development and testing" and "not suitable for production workloads". | +| Async | A standard request with `service_tier="flex"`. The fixture confirms the value reaches both the request body and `gen_ai.request.parameters`. | +| Dedicated Realtime | Doubleword's production tier; deployments are outside this recipe. | + +Both tested SDK versions accept the native `service_tier` parameter at runtime. OpenAI 1.69.0's type annotation lists only `auto` and `default`; 3.24.0's annotation also includes `flex`. The test branches on the installed version and checks the transmitted value. + +The same [API reference](https://doubleword.ai/llms.txt) lists embeddings. An OpenAI-shaped embedding fixture at `/v1/embeddings` produces a `CreateEmbeddingResponse` span with `gen_ai.span.kind=EMBEDDING` and provider `openai`. The model id from the response appears in `embedding.model_name`; `gen_ai.request.model` is absent. Requested model parameters and input/total token counts are recorded. + +## Privacy + +Set `FI_HIDE_INPUTS=true` to mask input attributes, or `FI_HIDE_OUTPUTS=true` to mask output attributes. `TraceConfig(hide_inputs=True)` is also available when configuring the instrumentor directly. Tests include visible controls and confirm that the provider still receives the prompt. Future AGI masking does not change the provider's own logging. + +For chat, `output.value` is assistant content, including when extra response metadata is supplied. Input masking alone leaves assistant output visible; output masking leaves input visible. + +Embedding spans also export raw response JSON under `embedding.embeddings`, vectors included. `FI_HIDE_EMBEDDING_VECTORS=true` removes the indexed `embedding.embeddings.0.embedding.vector` attribute but does **not** mask the vectors in `embedding.embeddings`. The fixtures pin both the visible control and this limitation. Input/output masking is not a blanket scrubber for embedding attributes. + +## Limits / not covered + +The Batch API is not covered by this recipe ([Doubleword API reference](https://doubleword.ai/llms.txt)). + +- The Anthropic-compatible Messages API and Dedicated deployments are outside this recipe ([Doubleword API reference](https://doubleword.ai/llms.txt)). +- Responses API, tools, structured outputs, cancellation, console imports, and pricing-table changes are not tested here. +- This recipe was not tested against the live provider (no paid call); the tests use a local fake of the OpenAI API and in-process fixtures. Streaming and embeddings results establish OpenAI-shaped fixture compatibility. + +## Tests + +Run from the repository root. The suite refuses non-loopback network connections. App subprocesses use a guard that blocks DNS and socket connections before reaching any outside host. + +Repository source: + +```bash +env -u PYTHONPATH PYTHONDONTWRITEBYTECODE=1 \ + PYTHONPATH="python/examples/doubleword/src:python:python/frameworks/openai:python/tests" \ + uv run --no-project --python 3.11 \ + --with 'openai==3.24.0' --with httpx --with 'wrapt<2' \ + --with opentelemetry-api --with opentelemetry-sdk --with opentelemetry-instrumentation \ + --with opentelemetry-exporter-otlp-proto-http --with requests --with protobuf \ + --with opentelemetry-proto --with pydantic --with jsonschema --with pytest \ + pytest python/examples/doubleword/tests -q -p no:cacheprovider --noconftest -o addopts= -rfEs +``` + +For the SDK floor, replace `openai==3.24.0` with `openai==1.69.0` in that command. + +Published packages: + +```bash +env -u PYTHONPATH PYTHONDONTWRITEBYTECODE=1 \ + PYTHONPATH="python/examples/doubleword/src:python/tests" \ + uv run --no-project --python 3.11 \ + --with 'openai==3.24.0' --with httpx --with 'wrapt<2' \ + --with 'traceAI-openai==0.1.10' --with 'fi-instrumentation-otel==1.1.0' \ + --with opentelemetry-api --with opentelemetry-sdk --with opentelemetry-instrumentation \ + --with opentelemetry-exporter-otlp-proto-http --with requests --with protobuf \ + --with opentelemetry-proto --with pydantic --with jsonschema --with pytest \ + pytest python/examples/doubleword/tests -q -p no:cacheprovider --noconftest -o addopts= -rfEs +``` + +| Python | `openai` | traceAI packages | Result | +|---|---|---|---| +| 3.10, 3.11, 3.12, 3.13 | 3.24.0 | `traceai-openai` and `fi_instrumentation` from this repository | Full suite | +| 3.11 | 1.69.0 (the `traceai-openai` floor) | from this repository | Full suite | +| 3.11 | 3.24.0 | published `traceAI-openai==0.1.10` and `fi-instrumentation-otel==1.1.0` from PyPI | Full suite | + +The chat fixtures use the historical example `deepseek-ai/DeepSeek-V4-Flash` from the public [tracing guide](https://arize.com/docs/ax/integrations/llm-providers/doubleword/doubleword-tracing). Its current catalog availability is not asserted. The dated response model and embedding model are synthetic fixture ids. diff --git a/python/examples/doubleword/requirements.txt b/python/examples/doubleword/requirements.txt new file mode 100644 index 000000000..5ae0f7608 --- /dev/null +++ b/python/examples/doubleword/requirements.txt @@ -0,0 +1,4 @@ +# Tested recipe pins; the suite also covers the OpenAI SDK floor, 1.69.0. +openai==3.24.0 +traceAI-openai==0.1.10 +fi-instrumentation-otel==1.1.0 diff --git a/python/examples/doubleword/src/app.py b/python/examples/doubleword/src/app.py new file mode 100644 index 000000000..52909d4d1 --- /dev/null +++ b/python/examples/doubleword/src/app.py @@ -0,0 +1,130 @@ +"""Trace Doubleword Chat Completions with the existing OpenAI instrumentor.""" + +import argparse +import ipaddress +import os +import re +import sys +from urllib.parse import urlsplit + +from fi_instrumentation import register +from fi_instrumentation.fi_types import ProjectType +from openai import OpenAI +from traceai_openai import OpenAIInstrumentor + +DEFAULT_BASE_URL = "https://api.doubleword.ai/v1" +API_KEY_ENV = "DOUBLEWORD_API_KEY" +BASE_URL_ENV = "DOUBLEWORD_BASE_URL" +MODEL_ENV = "DOUBLEWORD_MODEL" + + +def check_base_url(url: str) -> str: + """Validate without rewriting; permit customer proxies and loopback servers.""" + def refuse(reason): + raise ValueError(f"{BASE_URL_ENV}: {reason}") + + if not url or any(c.isspace() or not c.isprintable() for c in url): + refuse("use an absolute URL without whitespace or control characters") + try: + parts = urlsplit(url) + host = parts.hostname + parts.port # Reject malformed ports before tracing. + except ValueError: + refuse("use a valid absolute HTTP(S) URL") + if parts.username is not None or parts.password is not None: + refuse("remove URL credentials; supply the key through DOUBLEWORD_API_KEY") + if parts.scheme not in ("http", "https") or not host: + refuse("use an absolute HTTP(S) URL with a host") + if not host.isascii(): + refuse("use a plain ASCII host with valid IDNA spelling") + comparable_host = host.lower().rstrip(".") + structural_host = host.lower().removesuffix(".") + try: + ipaddress.ip_address(structural_host) + except ValueError: + try: + structural_host.encode("ascii").decode("idna") + except UnicodeError: + refuse("use a plain ASCII host with valid IDNA spelling") + if len(structural_host) > 253 or not all( + re.fullmatch(r"[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?", label) + for label in structural_host.split(".") + ): + refuse("use a valid ASCII host") + if comparable_host == "api.doubleword.ai": + if parts.scheme != "https": + refuse(f"use HTTPS: {DEFAULT_BASE_URL}") + if "?" in url or "#" in url: + refuse(f"remove the query or fragment; use {DEFAULT_BASE_URL}") + if parts.path.startswith(("/batch", "/v1/batch")): + refuse(f"the Batch API is not covered; use {DEFAULT_BASE_URL}") + if parts.path.startswith(("/messages", "/v1/messages")): + refuse(f"the Anthropic Messages API is not covered; use {DEFAULT_BASE_URL}") + if parts.path not in ("/v1", "/v1/"): + refuse(f"use the base URL {DEFAULT_BASE_URL}, without an endpoint path") + return url + + +def make_client(base_url: str | None = None, api_key: str | None = None, http_client=None) -> OpenAI: + base_url = check_base_url( + base_url if base_url is not None else os.environ.get(BASE_URL_ENV) or DEFAULT_BASE_URL + ) + return OpenAI( + base_url=base_url, + api_key=api_key if api_key is not None else os.environ[API_KEY_ENV], + http_client=http_client, + ) + + +def setup_tracing(project_name: str | None = None): + provider = register( + project_name=project_name or "doubleword-example", + project_type=ProjectType.OBSERVE, + set_global_tracer_provider=False, + verbose=False, + ) + OpenAIInstrumentor().instrument(tracer_provider=provider) + return provider + + +def main(argv=None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--stream", action="store_true") + parser.add_argument("--prompt", default="What is a rainbow?") + parser.add_argument("--model") + args = parser.parse_args(argv) + try: + base_url = check_base_url(os.environ.get(BASE_URL_ENV) or DEFAULT_BASE_URL) + model = args.model or os.environ.get(MODEL_ENV) + if not model: + raise ValueError(f"{MODEL_ENV}: set a catalog model id or pass --model") + api_key = os.environ.get(API_KEY_ENV) + if not api_key: + raise ValueError(f"{API_KEY_ENV}: set your Doubleword API key") + except ValueError as error: + print(error, file=sys.stderr) + return 2 + + provider = setup_tracing() + try: + with make_client(base_url=base_url, api_key=api_key) as client: + response = client.chat.completions.create( + model=model, + messages=[{"role": "user", "content": args.prompt}], + stream=args.stream, + ) + if args.stream: + with response: + for chunk in response: + if chunk.choices: + print(chunk.choices[0].delta.content or "", end="", flush=True) + print() + else: + print(response.choices[0].message.content or "") + finally: + provider.force_flush() + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/python/examples/doubleword/tests/_fake_openai.py b/python/examples/doubleword/tests/_fake_openai.py new file mode 100644 index 000000000..c96fb4e66 --- /dev/null +++ b/python/examples/doubleword/tests/_fake_openai.py @@ -0,0 +1,126 @@ +"""OpenAI-shaped fixtures and a recording server that binds only to 127.0.0.1.""" + +import copy +import json +import threading +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +# Historical example from the public tracing guide; availability is not asserted. +MODEL = "deepseek-ai/DeepSeek-V4-Flash" +RETURNED_MODEL = MODEL + "-2026-09-01" # Synthetic dated response id. +ANSWER = "A rainbow is refracted light." +USAGE = {"prompt_tokens": 13, "completion_tokens": 7, "total_tokens": 20} +VECTOR = [0.125, -0.75, 0.5] + + +def completion(*, usage=True, text=ANSWER, extra=None): + body = { + "id": "chatcmpl-fixture", + "object": "chat.completion", + "created": 1725148800, + "model": RETURNED_MODEL, + "choices": [{"index": 0, "message": {"role": "assistant", "content": text}, + "finish_reason": "stop"}], + } + if usage: + body["usage"] = dict(USAGE) + if extra: + body.update(extra) + return body + + +def stream_bytes(*, include_usage=False, text=ANSWER, extra=None): + chunks = [] + for piece in (text[:10], text[10:]): + chunk = { + "id": "chatcmpl-fixture", + "object": "chat.completion.chunk", + "created": 1725148800, + "model": RETURNED_MODEL, + "choices": [{"index": 0, "delta": {"content": piece}, "finish_reason": None}], + } + if extra: + chunk.update(extra) + chunks.append(chunk) + chunks.append({ + "id": "chatcmpl-fixture", "object": "chat.completion.chunk", + "created": 1725148800, "model": RETURNED_MODEL, + "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}], + }) + if include_usage: + chunks.append({ + "id": "chatcmpl-fixture", "object": "chat.completion.chunk", + "created": 1725148800, "model": RETURNED_MODEL, + "choices": [], "usage": dict(USAGE), + }) + return ("".join(f"data: {json.dumps(chunk)}\n\n" for chunk in chunks) + + "data: [DONE]\n\n").encode() + + +def error_body(status): + return {"error": { + "message": "Invalid API key" if status == 401 else "Rate limit exceeded", + "type": "authentication_error" if status == 401 else "rate_limit_error", + "code": "invalid_api_key" if status == 401 else "rate_limit_exceeded", + }} + + +def embeddings(): + return { + "object": "list", "model": "placeholder-embedding-model-2026-09-01", + "data": [{"object": "embedding", "index": 0, "embedding": list(VECTOR)}], + "usage": {"prompt_tokens": 11, "total_tokens": 11}, + } + + +class FakeOpenAI: + def __init__(self, status=200): + self._requests = [] + self._lock = threading.Lock() + owner = self + + class Handler(BaseHTTPRequestHandler): + def do_POST(self): + body = json.loads(self.rfile.read(int(self.headers["Content-Length"]))) + with owner._lock: + owner._requests.append({ + "path": self.path, "body": body, + "headers": {k.lower(): v for k, v in self.headers.items()}, + }) + if status != 200: + payload = json.dumps(error_body(status)).encode() + content_type = "application/json" + elif self.path != "/v1/chat/completions": + self.send_error(404) + return + elif body.get("stream"): + payload = stream_bytes(include_usage=body.get("stream_options", {}).get("include_usage", False)) + content_type = "text/event-stream" + else: + payload = json.dumps(completion()).encode() + content_type = "application/json" + self.send_response(status) + self.send_header("Content-Type", content_type) + self.send_header("Content-Length", str(len(payload))) + self.end_headers() + self.wfile.write(payload) + + def log_message(self, *args): + pass + + self._server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + self.origin = f"http://127.0.0.1:{self._server.server_port}" + self._thread = threading.Thread(target=self._server.serve_forever, daemon=True) + + def __enter__(self): + self._thread.start() + return self + + def __exit__(self, *args): + self._server.shutdown() + self._server.server_close() + self._thread.join() + + def requests(self): + with self._lock: + return copy.deepcopy(self._requests) diff --git a/python/examples/doubleword/tests/loopback_guard/sitecustomize.py b/python/examples/doubleword/tests/loopback_guard/sitecustomize.py new file mode 100644 index 000000000..e8b6c65f3 --- /dev/null +++ b/python/examples/doubleword/tests/loopback_guard/sitecustomize.py @@ -0,0 +1,44 @@ +"""Refuse non-loopback DNS and socket connections before any network activity.""" + +import os +import socket +from pathlib import Path + +_connect = socket.socket.connect +_connect_ex = socket.socket.connect_ex +_getaddrinfo = socket.getaddrinfo + + +def _check(host): + if isinstance(host, bytes): + host = host.decode("ascii", errors="replace") + if host not in ("127.0.0.1", "localhost"): + log = os.environ.get("LOOPBACK_GUARD_LOG") + if log: + with open(log, "a", encoding="utf-8") as file: + file.write("Refused non-loopback host\n") + raise OSError("loopback guard refused non-loopback host") + + +def _guard_connect(self, address): + _check(address[0]) + return _connect(self, address) + + +def _guard_connect_ex(self, address): + _check(address[0]) + return _connect_ex(self, address) + + +def _guard_getaddrinfo(host, *args, **kwargs): + _check(host) + return _getaddrinfo(host, *args, **kwargs) + + +socket.socket.connect = _guard_connect +socket.socket.connect_ex = _guard_connect_ex +socket.getaddrinfo = _guard_getaddrinfo +if os.environ.get("LOOPBACK_GUARD_LOG"): + Path(os.environ["LOOPBACK_GUARD_LOG"]).touch() +if os.environ.get("LOOPBACK_GUARD_READY"): + Path(os.environ["LOOPBACK_GUARD_READY"]).write_text("installed\n", encoding="utf-8") diff --git a/python/examples/doubleword/tests/test_doubleword_recipe.py b/python/examples/doubleword/tests/test_doubleword_recipe.py new file mode 100644 index 000000000..181070ffc --- /dev/null +++ b/python/examples/doubleword/tests/test_doubleword_recipe.py @@ -0,0 +1,629 @@ +"""Offline contracts for the recipe, real instrumentor, SDK, and OTLP exporter.""" + +import inspect +import json +import os +from pathlib import Path +import re +import socket +import subprocess +import sys +import tempfile +from contextlib import contextmanager +from uuid import uuid4 + +import app +import fi_instrumentation +import harness +from harness import Receiver +import httpx +import openai +import pytest +import traceai_openai +from traceai_openai import OpenAIInstrumentor + +from _fake_openai import ( + ANSWER, MODEL, RETURNED_MODEL, VECTOR, FakeOpenAI, + completion, embeddings, error_body, stream_bytes, +) + +RECIPE = Path(__file__).resolve().parents[1] +VENDOR_KEY = "placeholder-doubleword-key" +FI_KEY = "placeholder-futureagi-api-key" +FI_SECRET = "placeholder-futureagi-secret-key" +SDK_VERSION = tuple(int(part) for part in openai.__version__.split(".")[:3]) +USAGE_ATTRIBUTES = { + "gen_ai.usage.input_tokens": 13, + "gen_ai.usage.output_tokens": 7, + "gen_ai.usage.total_tokens": 20, +} + + +def attributes(span): + result = {} + for item in span.get("attributes", []): + value = item["value"] + if "arrayValue" in value: + result[item["key"]] = [next(iter(v.values())) for v in value["arrayValue"]["values"]] + else: + result[item["key"]] = next(iter(value.values())) + return result + + +def usage_attributes(attrs): + return {k: int(v) for k, v in attrs.items() if k.startswith("gen_ai.usage.")} + + +def only_span(receiver, provider): + assert provider.force_flush() + spans = receiver.spans() + assert len(spans) == 1 + return spans[0], attributes(spans[0]) + + +def assert_key_separation(request, receiver): + # Positive control: the vendor key was actually used on the vendor request. + assert request.headers["authorization"] == f"Bearer {VENDOR_KEY}" + assert "x-api-key" not in request.headers + assert "x-secret-key" not in request.headers + assert FI_KEY not in str(request.headers) + assert FI_SECRET not in str(request.headers) + exports = receiver.requests() + assert exports + for export in exports: + assert export["path"] == "/tracer/v1/traces" + assert export["headers"]["x-api-key"] == FI_KEY + assert export["headers"]["x-secret-key"] == FI_SECRET + assert VENDOR_KEY not in json.dumps(receiver.spans()) + assert VENDOR_KEY not in json.dumps(exports) + + +@pytest.fixture(autouse=True) +def offline_environment(monkeypatch): + for name in list(os.environ): + if name.startswith("FI_"): + monkeypatch.delenv(name) + for name in ("HTTP_PROXY", "HTTPS_PROXY", "ALL_PROXY", "http_proxy", "https_proxy", "all_proxy"): + monkeypatch.delenv(name, raising=False) + monkeypatch.setenv("FI_API_KEY", FI_KEY) + monkeypatch.setenv("FI_SECRET_KEY", FI_SECRET) + monkeypatch.setenv(app.API_KEY_ENV, VENDOR_KEY) + monkeypatch.setenv(app.MODEL_ENV, MODEL) + monkeypatch.delenv(app.BASE_URL_ENV, raising=False) + original_connect = socket.socket.connect + original_connect_ex = socket.socket.connect_ex + original_getaddrinfo = socket.getaddrinfo + + def check(host): + if host not in ("127.0.0.1", "localhost", b"127.0.0.1", b"localhost"): + raise OSError("test refuses non-loopback network access") + + def connect(sock, address): + check(address[0]) + return original_connect(sock, address) + + def connect_ex(sock, address): + check(address[0]) + return original_connect_ex(sock, address) + + def getaddrinfo(host, *args, **kwargs): + check(host) + return original_getaddrinfo(host, *args, **kwargs) + + monkeypatch.setattr(socket.socket, "connect", connect) + monkeypatch.setattr(socket.socket, "connect_ex", connect_ex) + monkeypatch.setattr(socket, "getaddrinfo", getaddrinfo) + yield + if OpenAIInstrumentor().is_instrumented_by_opentelemetry: + OpenAIInstrumentor().uninstrument() + + +@pytest.fixture +def journey(monkeypatch): + @contextmanager + def start(): + with Receiver() as receiver: + monkeypatch.setenv("FI_BASE_URL", receiver.origin) + project = f"doubleword-test-{uuid4().hex}" + provider = app.setup_tracing(project_name=project) + try: + yield receiver, provider, project + finally: + provider.force_flush() + OpenAIInstrumentor().uninstrument() + provider.shutdown() + return start + + +@contextmanager +def mock_client(handler): + with app.make_client(http_client=httpx.Client(transport=httpx.MockTransport(handler))) as client: + # Tests disable SDK retries so one fixture response means one request; + # the recipe itself keeps the SDK's default retry policy. + yield client.with_options(max_retries=0) + + +def test_documented_base_url_and_sdk_trailing_slash(): + assert app.DEFAULT_BASE_URL == "https://api.doubleword.ai/v1" + assert app.DEFAULT_BASE_URL in (RECIPE / "README.md").read_text() + with mock_client(lambda request: pytest.fail("client construction must not send a request")) as client: + assert str(client.base_url) == "https://api.doubleword.ai/v1/" + assert client.api_key == VENDOR_KEY + + +def test_realtime_chat_response_model_usage_and_key_separation(journey): + requests = [] + + def handler(request): + requests.append(request) + return httpx.Response(200, json=completion()) + + with journey() as (receiver, provider, project), mock_client(handler) as client: + response = client.chat.completions.create(model=MODEL, messages=[{"role": "user", "content": "Chat control marker"}]) + assert response.choices[0].message.content == ANSWER + span, attrs = only_span(receiver, provider) + assert span["name"] == "ChatCompletion" + assert span["status"]["code"] == "STATUS_CODE_OK" + assert attrs["gen_ai.span.kind"] == "LLM" + assert attrs["gen_ai.provider.name"] == "openai" + assert attrs["gen_ai.request.model"] == RETURNED_MODEL + assert RETURNED_MODEL != MODEL + assert json.loads(attrs["gen_ai.request.parameters"])["model"] == MODEL + assert usage_attributes(attrs) == USAGE_ATTRIBUTES + assert ANSWER in attrs["output.value"] + assert "Chat control marker" in json.dumps(span) + assert len(requests) == 1 + assert str(requests[0].url) == "https://api.doubleword.ai/v1/chat/completions" + assert json.loads(requests[0].content)["model"] == MODEL + assert_key_separation(requests[0], receiver) + assert receiver.requests()[0]["resource_attributes"][0]["project_name"] == project + + +def test_usage_absent_is_omitted(journey): + with journey() as (receiver, provider, _), mock_client(lambda r: httpx.Response(200, json=completion(usage=False))) as client: + client.chat.completions.create(model=MODEL, messages=[{"role": "user", "content": "Usage control"}]) + _, attrs = only_span(receiver, provider) + assert attrs["gen_ai.request.model"] == RETURNED_MODEL + assert ANSWER in attrs["output.value"] + assert usage_attributes(attrs) == {} + + +@pytest.mark.parametrize("include_usage", [False, True]) +def test_stream_text_model_gap_and_usage(journey, include_usage): + requests = [] + + def handler(request): + requests.append(request) + return httpx.Response(200, headers={"content-type": "text/event-stream"}, content=stream_bytes(include_usage=include_usage)) + + with journey() as (receiver, provider, _), mock_client(handler) as client: + options = {"stream_options": {"include_usage": True}} if include_usage else {} + stream = client.chat.completions.create(model=MODEL, messages=[{"role": "user", "content": "Stream control"}], stream=True, **options) + with stream: + text = "".join(chunk.choices[0].delta.content or "" for chunk in stream if chunk.choices) + assert text == ANSWER + span, attrs = only_span(receiver, provider) + assert span["name"] == "ChatCompletion" + assert attrs["gen_ai.span.kind"] == "LLM" + assert attrs["gen_ai.provider.name"] == "openai" + assert attrs["output.value"] == ANSWER + # traceai-openai records the model only from a non-streamed response; + # flip this when the instrumentor records the request model. + assert "gen_ai.request.model" not in attrs + params = json.loads(attrs["gen_ai.request.parameters"]) + assert params["model"] == MODEL + assert params["stream"] is True + assert usage_attributes(attrs) == (USAGE_ATTRIBUTES if include_usage else {}) + assert len(requests) == 1 + assert str(requests[0].url) == "https://api.doubleword.ai/v1/chat/completions" + body = json.loads(requests[0].content) + if include_usage: + assert body["stream_options"] == {"include_usage": True} + assert params["stream_options"] == {"include_usage": True} + else: + assert "stream_options" not in body + assert_key_separation(requests[0], receiver) + + +def test_async_flex_native_parameter_reaches_body_and_span(journey): + signature = inspect.signature(openai.resources.chat.completions.Completions.create) + assert "service_tier" in signature.parameters + # Both tested SDKs accept the native parameter at runtime. The older SDK's + # type annotation predates flex; branch on the version rather than exceptions. + if SDK_VERSION >= (2, 0, 0): + assert "flex" in str(signature.parameters["service_tier"].annotation) + else: + assert SDK_VERSION >= (1, 69, 0) + assert "flex" not in str(signature.parameters["service_tier"].annotation) + requests = [] + + def handler(request): + requests.append(request) + return httpx.Response(200, json=completion()) + + with journey() as (receiver, provider, _), mock_client(handler) as client: + client.chat.completions.create(model=MODEL, messages=[{"role": "user", "content": "Flex control"}], service_tier="flex") + _, attrs = only_span(receiver, provider) + assert len(requests) == 1 + assert str(requests[0].url) == "https://api.doubleword.ai/v1/chat/completions" + assert json.loads(requests[0].content)["service_tier"] == "flex" + params = json.loads(attrs["gen_ai.request.parameters"]) + assert params["service_tier"] == "flex" + assert params["model"] == MODEL + assert attrs["gen_ai.request.model"] == RETURNED_MODEL + assert_key_separation(requests[0], receiver) + + +@pytest.mark.parametrize("status,error_class", [(401, openai.AuthenticationError), (429, openai.RateLimitError)]) +def test_errors_export_error_span_and_exception(journey, status, error_class): + requests = [] + + def handler(request): + requests.append(request) + return httpx.Response(status, json=error_body(status)) + + with journey() as (receiver, provider, _), mock_client(handler) as client: + with pytest.raises(error_class) as caught: + client.chat.completions.create(model=MODEL, messages=[{"role": "user", "content": "Error control"}]) + assert error_body(status)["error"]["message"] in str(caught.value) + span, attrs = only_span(receiver, provider) + assert span["name"] == "ChatCompletion" + assert span["status"]["code"] == "STATUS_CODE_ERROR" + assert error_class.__name__ in span["status"]["message"] + assert attrs["gen_ai.span.kind"] == "LLM" + assert attrs["gen_ai.provider.name"] == "openai" + assert any(event["name"] == "exception" for event in span["events"]) + assert error_body(status)["error"]["message"] in json.dumps(span["events"]) + # traceai-openai records the model only from a non-streamed response; + # flip this when the instrumentor records the request model. + assert "gen_ai.request.model" not in attrs + assert json.loads(attrs["gen_ai.request.parameters"])["model"] == MODEL + assert usage_attributes(attrs) == {} + assert len(requests) == 1 # The test client disables SDK retries. + assert str(requests[0].url) == "https://api.doubleword.ai/v1/chat/completions" + assert_key_separation(requests[0], receiver) + + +def test_hide_inputs_with_visible_control_and_provider_receipt(journey, monkeypatch): + prompt = "unique-input-marker-doubleword" + for hidden in (False, True): + monkeypatch.setenv("FI_HIDE_INPUTS", str(hidden).lower()) + requests = [] + + def handler(request): + requests.append(request) + return httpx.Response(200, json=completion()) + + with journey() as (receiver, provider, _), mock_client(handler) as client: + client.chat.completions.create(model=MODEL, messages=[{"role": "user", "content": prompt}]) + span, attrs = only_span(receiver, provider) + assert prompt in requests[0].content.decode() + assert (prompt in json.dumps(receiver.spans())) is (not hidden) + assert ANSWER in attrs["output.value"] # Input masking does not hide outputs. + assert attrs["gen_ai.provider.name"] == "openai" + + +@pytest.mark.parametrize("streaming", [False, True]) +def test_hide_outputs_with_visible_control(journey, monkeypatch, streaming): + marker = "unique-output-marker-doubleword" + for hidden in (False, True): + monkeypatch.setenv("FI_HIDE_OUTPUTS", str(hidden).lower()) + + def handler(request): + if streaming: + return httpx.Response(200, headers={"content-type": "text/event-stream"}, content=stream_bytes(text=marker)) + return httpx.Response(200, json=completion(text=marker)) + + with journey() as (receiver, provider, _), mock_client(handler) as client: + response = client.chat.completions.create(model=MODEL, messages=[{"role": "user", "content": "visible-input-control"}], stream=streaming) + if streaming: + with response: + assert "".join(c.choices[0].delta.content or "" for c in response if c.choices) == marker + else: + assert response.choices[0].message.content == marker + span, attrs = only_span(receiver, provider) + assert (marker in json.dumps(span)) is (not hidden) + assert "visible-input-control" in json.dumps(span) + if hidden: + assert attrs["output.value"] == "__REDACTED__" + + +@pytest.mark.parametrize("streaming", [False, True]) +@pytest.mark.parametrize("empty_text", [False, True]) +def test_chat_content_and_no_raw_metadata_fallback(journey, monkeypatch, streaming, empty_text): + marker = "unique-extra-response-marker" + text = "" if empty_text else ANSWER + for hidden in (False, True): + monkeypatch.setenv("FI_HIDE_INPUTS", "true") + monkeypatch.setenv("FI_HIDE_OUTPUTS", str(hidden).lower()) + extra = {"provider_metadata": {"echoed_prompt": marker}} + + def handler(request): + if streaming: + return httpx.Response(200, headers={"content-type": "text/event-stream"}, content=stream_bytes(text=text, extra=extra)) + return httpx.Response(200, json=completion(text=text, extra=extra)) + + with journey() as (receiver, provider, _), mock_client(handler) as client: + response = client.chat.completions.create(model=MODEL, messages=[{"role": "user", "content": marker}], stream=streaming) + # Positive control: extra metadata reaches the SDK, even though the + # Chat Completions instrumentor exports content rather than raw JSON. + if streaming: + with response: + chunks = list(response) + assert any(getattr(chunk, "provider_metadata", {}).get("echoed_prompt") == marker for chunk in chunks) + assert "".join(c.choices[0].delta.content or "" for c in chunks if c.choices) == text + else: + assert response.provider_metadata["echoed_prompt"] == marker + assert response.choices[0].message.content == text + span, attrs = only_span(receiver, provider) + assert marker not in json.dumps(span) + assert "provider_metadata" not in json.dumps(span) + if empty_text and not streaming: + assert "output.value" not in attrs + else: + assert attrs["output.value"] == ("__REDACTED__" if hidden else text) + + +def test_embeddings_raw_vectors_survive_vector_hiding(journey, monkeypatch): + for hidden in (False, True): + monkeypatch.setenv("FI_HIDE_EMBEDDING_VECTORS", str(hidden).lower()) + requests = [] + + def handler(request): + requests.append(request) + return httpx.Response(200, json=embeddings()) + + with journey() as (receiver, provider, _), mock_client(handler) as client: + response = client.embeddings.create(model="placeholder-embedding-model", input="embedding input control", encoding_format="float") + assert response.data[0].embedding == VECTOR + span, attrs = only_span(receiver, provider) + assert span["name"] == "CreateEmbeddingResponse" + assert attrs["gen_ai.span.kind"] == "EMBEDDING" + assert attrs["gen_ai.provider.name"] == "openai" + assert attrs["embedding.model_name"] == "placeholder-embedding-model-2026-09-01" + assert "gen_ai.request.model" not in attrs + assert json.loads(attrs["gen_ai.request.parameters"])["model"] == "placeholder-embedding-model" + assert usage_attributes(attrs) == {"gen_ai.usage.input_tokens": 11, "gen_ai.usage.total_tokens": 11} + raw = json.loads(attrs["embedding.embeddings"]) + assert raw["data"][0]["embedding"] == VECTOR + vector_key = "embedding.embeddings.0.embedding.vector" + assert (vector_key in attrs) is (not hidden) + if not hidden: + assert attrs[vector_key] == VECTOR + assert len(requests) == 1 + assert str(requests[0].url) == "https://api.doubleword.ai/v1/embeddings" + assert_key_separation(requests[0], receiver) + + +def test_check_base_url_preserves_allowed_spellings(): + for url in ( + "https://api.doubleword.ai/v1", "https://api.doubleword.ai/v1/", + "HTTPS://API.DOUBLEWORD.AI/v1", "https://api.doubleword.ai./v1/", + "https://api.doubleword.ai:443/v1", "http://127.0.0.1:12345/v1", + "http://localhost:12345/v1/", "https://proxy.example.test/custom/v1?route=example#anchor", + ): + assert app.check_base_url(url) is url + + +def refused_urls(): + cases = [] + for host in ("api.doubleword.ai", "API.DOUBLEWORD.AI", "api.doubleword.ai."): + for path in ("", "/", "/v1/chat/completions", "/chat/completions", "/v1/embeddings", "/v1/responses", "/V1", "/v1//", "/%76%31", "/v1/../v1"): + cases.append((f"https://{host}{path}", "base URL")) + for path in ("/batch", "/batches", "/batch/jobs", "/v1/batch", "/v1/batches", "/v1/batch/jobs"): + cases.append((f"https://{host}{path}", "Batch API")) + for path in ("/v1/messages", "/v1/messages/", "/messages"): + cases.append((f"https://{host}{path}", "Anthropic Messages")) + cases.append((f"http://{host}/v1", "HTTPS")) + for suffix in ("?", "#", "?route=x", "#anchor"): + cases.append((f"https://{host}/v1{suffix}", "query or fragment")) + for char in (" ", "\t", "\r", "\n", "\x00", "\x7f", "\u00a0", "\u200b"): + cases.append((f"https://api.doubleword.ai/v1{char}", "whitespace or control")) + for dot in ("\u3002", "\uff0e", "\uff61"): + cases.append((f"https://api{dot}doubleword.ai/v1", "ASCII host")) + for host in ("xn--.example", "api..doubleword.ai", "-bad.example", "bad_.example", "api.doubleword.ai%2e", "api.doubleword.ai..", "API.DOUBLEWORD.AI..."): + cases.append((f"https://{host}/v1", "host")) + cases.append(("https://\u00e1pi.doubleword.ai/v1", "ASCII host")) + for origin in ("https://api.doubleword.ai", "https://proxy.example.test", "http://127.0.0.1:12345"): + scheme, host = origin.split("://") + cases.append((f"{scheme}://private-credential-marker:private-password-marker@{host}/v1", "credentials")) + cases.append((f"{scheme}://@{host}/v1", "credentials")) + cases.extend(( + ("", "absolute URL"), ("api.doubleword.ai/v1", "absolute"), + ("ftp://api.doubleword.ai/v1", "HTTP(S)"), + ("https:///v1", "host"), ("https://[invalid/v1", "valid absolute"), + ("https://api.doubleword.ai:invalid/v1", "valid absolute"), + )) + return cases + + +def test_every_refused_url_exits_before_tracing_or_network(monkeypatch, capsys): + def unexpected(*args, **kwargs): + pytest.fail("invalid URL must exit before tracing or client creation") + + monkeypatch.setattr(app, "setup_tracing", unexpected) + monkeypatch.setattr(app, "make_client", unexpected) + with Receiver() as receiver, FakeOpenAI() as fake: + monkeypatch.setenv("FI_BASE_URL", receiver.origin) + for url, reason in refused_urls(): + with pytest.raises(ValueError) as caught: + app.check_base_url(url) + message = str(caught.value) + assert app.BASE_URL_ENV in message + assert reason in message + assert "\n" not in message + assert "private-credential-marker" not in message + assert "private-password-marker" not in message + if not url: + # Empty override selects the documented default by contract. + continue + # OS environment values cannot contain NUL. An in-memory mapping + # exercises main's preflight for that spelling without calling putenv. + invalid_environment = dict(os.environ) + invalid_environment[app.BASE_URL_ENV] = url + with monkeypatch.context() as context: + context.setattr(app.os, "environ", invalid_environment) + assert app.main([]) == 2 + output = capsys.readouterr() + assert output.out == "" + assert message in output.err + assert "private-credential-marker" not in output.err + assert receiver.spans() == [] + assert receiver.requests() == [] + assert fake.requests() == [] + + +@pytest.mark.parametrize("missing", [app.API_KEY_ENV, app.MODEL_ENV]) +@pytest.mark.parametrize("empty", [False, True]) +def test_missing_model_or_vendor_key_exits_before_tracing(monkeypatch, capsys, missing, empty): + if empty: + monkeypatch.setenv(missing, "") + else: + monkeypatch.delenv(missing) + + def unexpected(*args, **kwargs): + pytest.fail("missing configuration must exit before tracing") + + monkeypatch.setattr(app, "setup_tracing", unexpected) + with Receiver() as receiver, FakeOpenAI() as fake: + monkeypatch.setenv("FI_BASE_URL", receiver.origin) + monkeypatch.setenv(app.BASE_URL_ENV, fake.origin + "/v1") + assert app.main([]) == 2 + assert missing in capsys.readouterr().err + assert receiver.spans() == [] + assert receiver.requests() == [] + assert fake.requests() == [] + + +def test_make_client_override_precedence_and_missing_key(monkeypatch): + monkeypatch.setenv(app.BASE_URL_ENV, "http://127.0.0.1:12345/custom/v1") + with app.make_client() as client: + assert str(client.base_url) == "http://127.0.0.1:12345/custom/v1/" + with app.make_client(base_url="https://api.doubleword.ai/v1/", api_key="placeholder-explicit-key") as client: + assert str(client.base_url) == "https://api.doubleword.ai/v1/" + assert client.api_key == "placeholder-explicit-key" + monkeypatch.delenv(app.API_KEY_ENV) + with pytest.raises(KeyError) as caught: + app.make_client() + assert caught.value.args == (app.API_KEY_ENV,) + + +def child_environment(receiver, fake, directory): + env = dict(os.environ) + # Use the packages this process imported, so published-wheel runs stay valid. + roots = [RECIPE / "tests/loopback_guard", RECIPE / "src", + Path(openai.__file__).resolve().parent.parent, + Path(fi_instrumentation.__file__).resolve().parent.parent, + Path(traceai_openai.__file__).resolve().parent.parent, + Path(harness.__file__).resolve().parent.parent] + if SDK_VERSION < (2, 0, 0): + import distro # The SDK floor has this extra dependency. + roots.append(Path(distro.__file__).resolve().parent.parent) + env["PYTHONPATH"] = os.pathsep.join(dict.fromkeys(str(root) for root in roots)) + env.update({ + "PYTHONDONTWRITEBYTECODE": "1", "FI_API_KEY": FI_KEY, "FI_SECRET_KEY": FI_SECRET, + "FI_BASE_URL": receiver.origin, app.API_KEY_ENV: VENDOR_KEY, + app.MODEL_ENV: MODEL, app.BASE_URL_ENV: fake.origin + "/v1", + "LOOPBACK_GUARD_LOG": str(directory / "guard.log"), + "LOOPBACK_GUARD_READY": str(directory / "guard.ready"), + }) + return env + + +@pytest.mark.parametrize("streaming", [False, True]) +def test_app_subprocess_loopback_chat_and_stream(streaming): + with Receiver() as receiver, FakeOpenAI() as fake, tempfile.TemporaryDirectory(dir=RECIPE, prefix=".test-run-") as temporary: + directory = Path(temporary) + env = child_environment(receiver, fake, directory) + argv = [sys.executable, str(RECIPE / "src/app.py"), "--prompt", "subprocess-prompt-control"] + if streaming: + argv.append("--stream") + else: + # A CLI model also works when the model environment variable is absent. + env.pop(app.MODEL_ENV) + argv.extend(["--model", MODEL]) + result = subprocess.run(argv, env=env, capture_output=True, text=True, timeout=120) + assert result.returncode == 0, result.stderr + assert result.stdout.strip() == ANSWER + assert directory.joinpath("guard.ready").read_text() == "installed\n" + assert directory.joinpath("guard.log").read_text() == "" + requests = fake.requests() + assert len(requests) == 1 + assert requests[0]["path"] == "/v1/chat/completions" + assert requests[0]["headers"]["authorization"] == f"Bearer {VENDOR_KEY}" + assert requests[0]["headers"]["user-agent"] == f"OpenAI/Python {openai.__version__}" + assert "x-api-key" not in requests[0]["headers"] + assert "x-secret-key" not in requests[0]["headers"] + assert requests[0]["body"]["model"] == MODEL + assert requests[0]["body"]["stream"] is streaming + assert requests[0]["body"]["messages"][0]["content"] == "subprocess-prompt-control" + spans = receiver.spans() + assert len(spans) == 1 + attrs = attributes(spans[0]) + assert spans[0]["name"] == "ChatCompletion" + assert attrs["gen_ai.span.kind"] == "LLM" + assert attrs["gen_ai.provider.name"] == "openai" + assert json.loads(attrs["gen_ai.request.parameters"])["model"] == MODEL + if streaming: + assert "gen_ai.request.model" not in attrs + assert usage_attributes(attrs) == {} + else: + assert attrs["gen_ai.request.model"] == RETURNED_MODEL + assert usage_attributes(attrs) == USAGE_ATTRIBUTES + assert ANSWER in attrs["output.value"] + exports = receiver.requests() + assert len(exports) == 1 + assert exports[0]["path"] == "/tracer/v1/traces" + assert exports[0]["headers"]["x-api-key"] == FI_KEY + assert exports[0]["headers"]["x-secret-key"] == FI_SECRET + assert exports[0]["resource_attributes"][0]["project_name"] == "doubleword-example" + assert VENDOR_KEY not in json.dumps(spans + exports) + assert VENDOR_KEY not in result.stdout + result.stderr + + +@pytest.mark.parametrize("operation", ["dns", "connect", "connect_ex"]) +def test_loopback_guard_refuses_real_host_before_dns(operation): + with Receiver() as receiver, FakeOpenAI() as fake, tempfile.TemporaryDirectory(dir=RECIPE, prefix=".test-run-") as temporary: + directory = Path(temporary) + env = child_environment(receiver, fake, directory) + calls = { + "dns": "socket.getaddrinfo('api.doubleword.ai', 443)", + "connect": "socket.socket().connect(('api.doubleword.ai', 443))", + "connect_ex": "socket.socket().connect_ex(('api.doubleword.ai', 443))", + } + # Positive control: DNS for loopback is permitted under the same guard. + code = "import socket; assert socket.getaddrinfo('127.0.0.1', 80); " + calls[operation] + result = subprocess.run([sys.executable, "-c", code], env=env, capture_output=True, text=True, timeout=120) + assert result.returncode != 0 + assert "loopback guard refused non-loopback host" in result.stderr + assert directory.joinpath("guard.ready").read_text() == "installed\n" + assert directory.joinpath("guard.log").read_text() == "Refused non-loopback host\n" + assert fake.requests() == [] + assert receiver.spans() == [] + + +def test_readme_requirements_and_public_contract(): + readme = (RECIPE / "README.md").read_text() + assert readme.startswith("# Doubleword (OpenAI-compatible) with traceAI") + assert app.DEFAULT_BASE_URL in readme + assert "provider field says `openai`" in readme + for name in (app.API_KEY_ENV, app.BASE_URL_ENV, app.MODEL_ENV, "FI_API_KEY", "FI_SECRET_KEY", "FI_BASE_URL", "FI_HIDE_INPUTS", "FI_HIDE_OUTPUTS", "FI_HIDE_EMBEDDING_VECTORS"): + assert name in readme + assert "gen_ai.request.model" in readme + assert "service_tier=\"flex\"" in readme + assert "embedding.embeddings" in readme + assert "verbose=False" in readme + assert "TBD" not in readme + assert "https://doubleword.ai/llms.txt" in readme + assert "https://app.doubleword.ai/api-keys" in readme + assert "not suitable for production workloads" in readme + for py_path in ("python/examples/doubleword/src:python:python/frameworks/openai:python/tests", "python/examples/doubleword/src:python/tests"): + assert f'PYTHONPATH="{py_path}"' in readme + assert readme.count("pytest python/examples/doubleword/tests -q -p no:cacheprovider --noconftest -o addopts= -rfEs") == 2 + assert "--with 'traceAI-openai==0.1.10' --with 'fi-instrumentation-otel==1.1.0'" in readme + requirements = (RECIPE / "requirements.txt").read_text() + assert [line for line in requirements.splitlines() if line and not line.startswith("#")] == ["openai==3.24.0", "traceAI-openai==0.1.10", "fi-instrumentation-otel==1.1.0"] + assert not re.search(r"\b(?:sk|fi)-[a-zA-Z0-9]{16,}\b", readme) + for path in RECIPE.rglob("*"): + if path.is_file(): + assert not re.search(r"\b[A-Z]{2,5}-\d{3,}\b", path.read_text()), str(path) From 590e4fd5990f60c9e5f1e80e7632add4084413d8 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Mon, 5 Oct 2026 23:54:42 -0700 Subject: [PATCH 2/7] fix(examples/doubleword): review r1 P3 follow-ups - service_tier test pins the runtime claim (value in body and span parameters) and no longer asserts version-specific type annotations, which broke on 1.x releases between the tested pins. - Subprocess PYTHONPATH lists the instrumentation roots before the SDK site-packages, so the child imports the same instrumentor as the parent even when published wheels are installed. - The internal-id scan reads only .py/.md/.txt files outside __pycache__ and test-run directories. - Proxy host names may contain underscores inside a label (Compose service names); label edges and the vendor-host rules are unchanged. - README: a non-streamed response with empty content exports no output.value; service_tier wording matches the test. approved: Nikhil 2026-10-03 blanket Refs: TH-8309 --- python/examples/doubleword/README.md | 6 +++--- python/examples/doubleword/src/app.py | 2 +- .../tests/test_doubleword_recipe.py | 20 +++++++++---------- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/python/examples/doubleword/README.md b/python/examples/doubleword/README.md index 39b51d53e..d540602ac 100644 --- a/python/examples/doubleword/README.md +++ b/python/examples/doubleword/README.md @@ -29,7 +29,7 @@ pip install traceAI-openai fi-instrumentation-otel openai The application exits 2 for a missing or empty `DOUBLEWORD_API_KEY`, a missing or empty `DOUBLEWORD_MODEL` without `--model`, or an invalid `DOUBLEWORD_BASE_URL`. It checks these before tracing or network calls. The recipe does not check the Future AGI keys itself. -On `api.doubleword.ai`, the URL must use HTTPS and the path must be exactly `/v1` or `/v1/`. Endpoint paths, queries, fragments, URL credentials, whitespace, control characters, and invalid host spellings fail early. Uppercase hosts and a trailing DNS dot do not bypass these checks. Customer proxies and loopback servers can use their own HTTP(S) paths. Validation returns allowed URLs unchanged; the OpenAI SDK adds a trailing slash to `client.base_url`. +On `api.doubleword.ai`, the URL must use HTTPS and the path must be exactly `/v1` or `/v1/`. Endpoint paths, queries, fragments, URL credentials, whitespace, control characters, and invalid host spellings fail early. Uppercase hosts and a trailing DNS dot do not bypass these checks. Customer proxies and loopback servers can use their own HTTP(S) paths; proxy host names may contain underscores inside a label (for example a Compose service name such as `myproj_proxy_1`). Validation returns allowed URLs unchanged; the OpenAI SDK adds a trailing slash to `client.base_url`. ## Run @@ -76,7 +76,7 @@ Current `traceai-openai` behavior: - Non-streamed `gen_ai.request.model` is the model id the provider returns in its response. `gen_ai.request.parameters` contains the model id sent in the request. The fixtures deliberately return a different, synthetic dated id so this distinction is tested. - Streamed and failed calls omit `gen_ai.request.model`. The requested model remains in the `gen_ai.request.parameters` JSON unless invocation parameters are hidden. -- Chat `output.value` contains the assistant content. Streams accumulate that content into one span. A stream with no output text exports an empty string, with no raw-response fallback. Extra response metadata is not copied into chat `output.value`. +- Chat `output.value` contains the assistant content. Streams accumulate that content into one span. A stream with no output text exports an empty string, with no raw-response fallback. A non-streamed response with empty content exports no `output.value`. Extra response metadata is not copied into chat `output.value`. - The tests pin `gen_ai.usage.input_tokens`, `gen_ai.usage.output_tokens`, and `gen_ai.usage.total_tokens`. Missing usage is omitted, never set to zero. Default stream fixtures omit usage; `stream_options={"include_usage": True}` with a final usage chunk exports all three counts. - The 401 and 429 fixtures produce ERROR spans with exception events. The recipe keeps the SDK's default retry policy; the error tests disable retries so each fixture sees one request. @@ -92,7 +92,7 @@ Doubleword documents the following tiers in its [API reference](https://doublewo | Async | A standard request with `service_tier="flex"`. The fixture confirms the value reaches both the request body and `gen_ai.request.parameters`. | | Dedicated Realtime | Doubleword's production tier; deployments are outside this recipe. | -Both tested SDK versions accept the native `service_tier` parameter at runtime. OpenAI 1.69.0's type annotation lists only `auto` and `default`; 3.24.0's annotation also includes `flex`. The test branches on the installed version and checks the transmitted value. +Both tested SDK versions (the 1.69.0 floor and 3.24.0) accept the native `service_tier` parameter and send `flex` unchanged; the test checks the transmitted value and the recorded parameter. Older SDK type annotations may not list `flex`; that does not change the request. The same [API reference](https://doubleword.ai/llms.txt) lists embeddings. An OpenAI-shaped embedding fixture at `/v1/embeddings` produces a `CreateEmbeddingResponse` span with `gen_ai.span.kind=EMBEDDING` and provider `openai`. The model id from the response appears in `embedding.model_name`; `gen_ai.request.model` is absent. Requested model parameters and input/total token counts are recorded. diff --git a/python/examples/doubleword/src/app.py b/python/examples/doubleword/src/app.py index 52909d4d1..e4da2bd6b 100644 --- a/python/examples/doubleword/src/app.py +++ b/python/examples/doubleword/src/app.py @@ -47,7 +47,7 @@ def refuse(reason): except UnicodeError: refuse("use a plain ASCII host with valid IDNA spelling") if len(structural_host) > 253 or not all( - re.fullmatch(r"[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?", label) + re.fullmatch(r"[a-zA-Z0-9](?:[a-zA-Z0-9_-]{0,61}[a-zA-Z0-9])?", label) for label in structural_host.split(".") ): refuse("use a valid ASCII host") diff --git a/python/examples/doubleword/tests/test_doubleword_recipe.py b/python/examples/doubleword/tests/test_doubleword_recipe.py index 181070ffc..68d075aec 100644 --- a/python/examples/doubleword/tests/test_doubleword_recipe.py +++ b/python/examples/doubleword/tests/test_doubleword_recipe.py @@ -227,14 +227,10 @@ def handler(request): def test_async_flex_native_parameter_reaches_body_and_span(journey): signature = inspect.signature(openai.resources.chat.completions.Completions.create) + # service_tier is a native keyword on every tested SDK (1.69.0 floor and 3.24.0). + # The claim pinned here is the runtime behaviour: the value reaches the body and + # the span parameters. Type annotations differ between SDK versions and are not pinned. assert "service_tier" in signature.parameters - # Both tested SDKs accept the native parameter at runtime. The older SDK's - # type annotation predates flex; branch on the version rather than exceptions. - if SDK_VERSION >= (2, 0, 0): - assert "flex" in str(signature.parameters["service_tier"].annotation) - else: - assert SDK_VERSION >= (1, 69, 0) - assert "flex" not in str(signature.parameters["service_tier"].annotation) requests = [] def handler(request): @@ -401,6 +397,7 @@ def test_check_base_url_preserves_allowed_spellings(): "HTTPS://API.DOUBLEWORD.AI/v1", "https://api.doubleword.ai./v1/", "https://api.doubleword.ai:443/v1", "http://127.0.0.1:12345/v1", "http://localhost:12345/v1/", "https://proxy.example.test/custom/v1?route=example#anchor", + "http://myproj_proxy_1:8080/v1", ): assert app.check_base_url(url) is url @@ -511,11 +508,13 @@ def test_make_client_override_precedence_and_missing_key(monkeypatch): def child_environment(receiver, fake, directory): env = dict(os.environ) # Use the packages this process imported, so published-wheel runs stay valid. + # The instrumentation roots come before the SDK's site-packages so the child + # imports the same traceai_openai and fi_instrumentation as this process. roots = [RECIPE / "tests/loopback_guard", RECIPE / "src", - Path(openai.__file__).resolve().parent.parent, Path(fi_instrumentation.__file__).resolve().parent.parent, Path(traceai_openai.__file__).resolve().parent.parent, - Path(harness.__file__).resolve().parent.parent] + Path(harness.__file__).resolve().parent.parent, + Path(openai.__file__).resolve().parent.parent] if SDK_VERSION < (2, 0, 0): import distro # The SDK floor has this extra dependency. roots.append(Path(distro.__file__).resolve().parent.parent) @@ -625,5 +624,6 @@ def test_readme_requirements_and_public_contract(): assert [line for line in requirements.splitlines() if line and not line.startswith("#")] == ["openai==3.24.0", "traceAI-openai==0.1.10", "fi-instrumentation-otel==1.1.0"] assert not re.search(r"\b(?:sk|fi)-[a-zA-Z0-9]{16,}\b", readme) for path in RECIPE.rglob("*"): - if path.is_file(): + if (path.is_file() and path.suffix in {".py", ".md", ".txt"} + and not any(part == "__pycache__" or part.startswith(".test-run-") for part in path.parts)): assert not re.search(r"\b[A-Z]{2,5}-\d{3,}\b", path.read_text()), str(path) From 4b4c758171c1c7cbc68641e4bfc05a7399471cae Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Sun, 11 Oct 2026 20:25:04 -0700 Subject: [PATCH 3/7] docs(examples/doubleword): cite the vendor's own docs, not a competitor's The README sourced a claim from a third-party observability vendor's docs. Future AGI READMEs should not send readers there; the claim now cites Doubleword's API reference (llms.txt). Docs only (plus the README contract test where it pinned the old link). Co-Authored-By: Claude Opus 5.5 --- python/examples/doubleword/README.md | 2 +- python/examples/doubleword/tests/_fake_openai.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/python/examples/doubleword/README.md b/python/examples/doubleword/README.md index d540602ac..161964af5 100644 --- a/python/examples/doubleword/README.md +++ b/python/examples/doubleword/README.md @@ -151,4 +151,4 @@ env -u PYTHONPATH PYTHONDONTWRITEBYTECODE=1 \ | 3.11 | 1.69.0 (the `traceai-openai` floor) | from this repository | Full suite | | 3.11 | 3.24.0 | published `traceAI-openai==0.1.10` and `fi-instrumentation-otel==1.1.0` from PyPI | Full suite | -The chat fixtures use the historical example `deepseek-ai/DeepSeek-V4-Flash` from the public [tracing guide](https://arize.com/docs/ax/integrations/llm-providers/doubleword/doubleword-tracing). Its current catalog availability is not asserted. The dated response model and embedding model are synthetic fixture ids. +The chat fixtures use `deepseek-ai/DeepSeek-V4-Flash`, a model id listed in Doubleword's [API reference](https://doubleword.ai/llms.txt). Its current catalog availability is not asserted. The dated response model and embedding model are synthetic fixture ids. diff --git a/python/examples/doubleword/tests/_fake_openai.py b/python/examples/doubleword/tests/_fake_openai.py index c96fb4e66..d950ce2bd 100644 --- a/python/examples/doubleword/tests/_fake_openai.py +++ b/python/examples/doubleword/tests/_fake_openai.py @@ -5,7 +5,7 @@ import threading from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer -# Historical example from the public tracing guide; availability is not asserted. +# Model id from Doubleword's API reference (doubleword.ai/llms.txt); availability is not asserted. MODEL = "deepseek-ai/DeepSeek-V4-Flash" RETURNED_MODEL = MODEL + "-2026-09-01" # Synthetic dated response id. ANSWER = "A rainbow is refracted light." From 2fb118cdc7bbb8f1f7ced2ef5459c8dcef45a2dd Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Sun, 11 Oct 2026 21:32:41 -0700 Subject: [PATCH 4/7] test(examples/doubleword): accept traceai-openai before and after the provider/model fix The recipe pinned traceai-openai 0.1.10 behaviour: gen_ai.provider.name is always "openai" and streamed/failed spans carry no gen_ai.request.model. future-agi/traceAI#249 labels known vendor hosts and records the requested model, so these pins would break whichever PR merged second. Accept both behaviours (and the related Responses/embedding output changes where this recipe checks them), and say in the README which version does what. Passes against published traceAI-openai 0.1.10 and against #249. Co-Authored-By: Claude Opus 5.5 --- python/examples/doubleword/README.md | 8 ++-- .../tests/test_doubleword_recipe.py | 38 ++++++++++++------- 2 files changed, 29 insertions(+), 17 deletions(-) diff --git a/python/examples/doubleword/README.md b/python/examples/doubleword/README.md index 161964af5..2a08c67b7 100644 --- a/python/examples/doubleword/README.md +++ b/python/examples/doubleword/README.md @@ -70,12 +70,12 @@ provider.force_flush() ## What you see in Future AGI -The fixtures export one `ChatCompletion` span with `gen_ai.span.kind=LLM` per completed chat call. The provider field says `openai`: `gen_ai.provider.name` is the shared instrumentor's label, including when the client uses Doubleword's host. +The fixtures export one `ChatCompletion` span with `gen_ai.span.kind=LLM` per completed chat call. Up to `traceai-openai` 0.1.10, the provider field says `openai`: `gen_ai.provider.name` is the shared instrumentor's label (later versions, future-agi/traceAI#249, say `doubleword` for `api.doubleword.ai`), including when the client uses Doubleword's host. Current `traceai-openai` behavior: - Non-streamed `gen_ai.request.model` is the model id the provider returns in its response. `gen_ai.request.parameters` contains the model id sent in the request. The fixtures deliberately return a different, synthetic dated id so this distinction is tested. -- Streamed and failed calls omit `gen_ai.request.model`. The requested model remains in the `gen_ai.request.parameters` JSON unless invocation parameters are hidden. +- Up to 0.1.10, streamed and failed calls omit `gen_ai.request.model`; later versions record the requested model there. The requested model remains in the `gen_ai.request.parameters` JSON unless invocation parameters are hidden. - Chat `output.value` contains the assistant content. Streams accumulate that content into one span. A stream with no output text exports an empty string, with no raw-response fallback. A non-streamed response with empty content exports no `output.value`. Extra response metadata is not copied into chat `output.value`. - The tests pin `gen_ai.usage.input_tokens`, `gen_ai.usage.output_tokens`, and `gen_ai.usage.total_tokens`. Missing usage is omitted, never set to zero. Default stream fixtures omit usage; `stream_options={"include_usage": True}` with a final usage chunk exports all three counts. - The 401 and 429 fixtures produce ERROR spans with exception events. The recipe keeps the SDK's default retry policy; the error tests disable retries so each fixture sees one request. @@ -94,7 +94,7 @@ Doubleword documents the following tiers in its [API reference](https://doublewo Both tested SDK versions (the 1.69.0 floor and 3.24.0) accept the native `service_tier` parameter and send `flex` unchanged; the test checks the transmitted value and the recorded parameter. Older SDK type annotations may not list `flex`; that does not change the request. -The same [API reference](https://doubleword.ai/llms.txt) lists embeddings. An OpenAI-shaped embedding fixture at `/v1/embeddings` produces a `CreateEmbeddingResponse` span with `gen_ai.span.kind=EMBEDDING` and provider `openai`. The model id from the response appears in `embedding.model_name`; `gen_ai.request.model` is absent. Requested model parameters and input/total token counts are recorded. +The same [API reference](https://doubleword.ai/llms.txt) lists embeddings. An OpenAI-shaped embedding fixture at `/v1/embeddings` produces a `CreateEmbeddingResponse` span with `gen_ai.span.kind=EMBEDDING` and provider `openai` (`doubleword` in later versions). The model id from the response appears in `embedding.model_name`; up to 0.1.10 `gen_ai.request.model` is absent, and later versions set it too. Requested model parameters and input/total token counts are recorded. ## Privacy @@ -102,7 +102,7 @@ Set `FI_HIDE_INPUTS=true` to mask input attributes, or `FI_HIDE_OUTPUTS=true` to For chat, `output.value` is assistant content, including when extra response metadata is supplied. Input masking alone leaves assistant output visible; output masking leaves input visible. -Embedding spans also export raw response JSON under `embedding.embeddings`, vectors included. `FI_HIDE_EMBEDDING_VECTORS=true` removes the indexed `embedding.embeddings.0.embedding.vector` attribute but does **not** mask the vectors in `embedding.embeddings`. The fixtures pin both the visible control and this limitation. Input/output masking is not a blanket scrubber for embedding attributes. +Embedding spans also export raw response JSON under `embedding.embeddings`, vectors included. `FI_HIDE_EMBEDDING_VECTORS=true` removes the indexed `embedding.embeddings.0.embedding.vector` attribute but, up to `traceai-openai` 0.1.10, does **not** mask the vectors in `embedding.embeddings`; later versions strip them there too. The tests check the visible control and accept either version's handling of the raw JSON. Input/output masking is not a blanket scrubber for embedding attributes. ## Limits / not covered diff --git a/python/examples/doubleword/tests/test_doubleword_recipe.py b/python/examples/doubleword/tests/test_doubleword_recipe.py index 68d075aec..ec140ac87 100644 --- a/python/examples/doubleword/tests/test_doubleword_recipe.py +++ b/python/examples/doubleword/tests/test_doubleword_recipe.py @@ -27,6 +27,12 @@ completion, embeddings, error_body, stream_bytes, ) +# traceai-openai 0.1.10 and earlier label every span "openai"; later versions +# (future-agi/traceAI#249) use the vendor's name for its documented API host. +# They also omit gen_ai.request.model on streamed and failed calls, where later +# versions record the requested model. Accept both so this recipe passes on either version. +PROVIDER_NAMES = ("openai", "doubleword") + RECIPE = Path(__file__).resolve().parents[1] VENDOR_KEY = "placeholder-doubleword-key" FI_KEY = "placeholder-futureagi-api-key" @@ -165,7 +171,7 @@ def handler(request): assert span["name"] == "ChatCompletion" assert span["status"]["code"] == "STATUS_CODE_OK" assert attrs["gen_ai.span.kind"] == "LLM" - assert attrs["gen_ai.provider.name"] == "openai" + assert attrs["gen_ai.provider.name"] in PROVIDER_NAMES assert attrs["gen_ai.request.model"] == RETURNED_MODEL assert RETURNED_MODEL != MODEL assert json.loads(attrs["gen_ai.request.parameters"])["model"] == MODEL @@ -205,11 +211,11 @@ def handler(request): span, attrs = only_span(receiver, provider) assert span["name"] == "ChatCompletion" assert attrs["gen_ai.span.kind"] == "LLM" - assert attrs["gen_ai.provider.name"] == "openai" + assert attrs["gen_ai.provider.name"] in PROVIDER_NAMES assert attrs["output.value"] == ANSWER # traceai-openai records the model only from a non-streamed response; - # flip this when the instrumentor records the request model. - assert "gen_ai.request.model" not in attrs + # 0.1.10 records the model only from non-streamed responses; later versions record the requested one. + assert attrs.get("gen_ai.request.model") in (None, MODEL) params = json.loads(attrs["gen_ai.request.parameters"]) assert params["model"] == MODEL assert params["stream"] is True @@ -267,12 +273,12 @@ def handler(request): assert span["status"]["code"] == "STATUS_CODE_ERROR" assert error_class.__name__ in span["status"]["message"] assert attrs["gen_ai.span.kind"] == "LLM" - assert attrs["gen_ai.provider.name"] == "openai" + assert attrs["gen_ai.provider.name"] in PROVIDER_NAMES assert any(event["name"] == "exception" for event in span["events"]) assert error_body(status)["error"]["message"] in json.dumps(span["events"]) # traceai-openai records the model only from a non-streamed response; - # flip this when the instrumentor records the request model. - assert "gen_ai.request.model" not in attrs + # 0.1.10 records the model only from non-streamed responses; later versions record the requested one. + assert attrs.get("gen_ai.request.model") in (None, MODEL) assert json.loads(attrs["gen_ai.request.parameters"])["model"] == MODEL assert usage_attributes(attrs) == {} assert len(requests) == 1 # The test client disables SDK retries. @@ -296,7 +302,7 @@ def handler(request): assert prompt in requests[0].content.decode() assert (prompt in json.dumps(receiver.spans())) is (not hidden) assert ANSWER in attrs["output.value"] # Input masking does not hide outputs. - assert attrs["gen_ai.provider.name"] == "openai" + assert attrs["gen_ai.provider.name"] in PROVIDER_NAMES @pytest.mark.parametrize("streaming", [False, True]) @@ -375,13 +381,19 @@ def handler(request): span, attrs = only_span(receiver, provider) assert span["name"] == "CreateEmbeddingResponse" assert attrs["gen_ai.span.kind"] == "EMBEDDING" - assert attrs["gen_ai.provider.name"] == "openai" + assert attrs["gen_ai.provider.name"] in PROVIDER_NAMES assert attrs["embedding.model_name"] == "placeholder-embedding-model-2026-09-01" - assert "gen_ai.request.model" not in attrs + # 0.1.10 sets only embedding.model_name; later versions also make it the span model. + assert attrs.get("gen_ai.request.model") in (None, "placeholder-embedding-model-2026-09-01") assert json.loads(attrs["gen_ai.request.parameters"])["model"] == "placeholder-embedding-model" assert usage_attributes(attrs) == {"gen_ai.usage.input_tokens": 11, "gen_ai.usage.total_tokens": 11} raw = json.loads(attrs["embedding.embeddings"]) - assert raw["data"][0]["embedding"] == VECTOR + if hidden and "embedding" not in raw["data"][0]: + # Later versions (future-agi/traceAI#249) also strip the raw JSON copy. + assert raw["data"][0]["dimensions"] == len(VECTOR) + else: + # 0.1.10 keeps the vectors in the raw JSON even when hidden. + assert raw["data"][0]["embedding"] == VECTOR vector_key = "embedding.embeddings.0.embedding.vector" assert (vector_key in attrs) is (not hidden) if not hidden: @@ -561,10 +573,10 @@ def test_app_subprocess_loopback_chat_and_stream(streaming): attrs = attributes(spans[0]) assert spans[0]["name"] == "ChatCompletion" assert attrs["gen_ai.span.kind"] == "LLM" - assert attrs["gen_ai.provider.name"] == "openai" + assert attrs["gen_ai.provider.name"] in PROVIDER_NAMES assert json.loads(attrs["gen_ai.request.parameters"])["model"] == MODEL if streaming: - assert "gen_ai.request.model" not in attrs + assert attrs.get("gen_ai.request.model") in (None, MODEL) assert usage_attributes(attrs) == {} else: assert attrs["gen_ai.request.model"] == RETURNED_MODEL From ce5e78fe0f3697c36fca97ce5688aa47a9f0808d Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Sun, 18 Oct 2026 21:46:06 -0700 Subject: [PATCH 5/7] fix(examples/doubleword): honour FI_PROJECT_NAME setup_tracing() passed project_name or "doubleword-example" to register(), so the default was always truthy and FI_PROJECT_NAME was never read: `python src/app.py` always exported to doubleword-example. The Anannas recipe reads FI_PROJECT_NAME with its name as the default; do the same here. An explicit setup_tracing() argument still wins, and an unset or empty FI_PROJECT_NAME keeps doubleword-example (never register()'s generic default). - README: FI_PROJECT_NAME in the Configure table and the Run section; the Code snippet reads it the same way as src/app.py. - Tests: setup_tracing() default / empty / env / explicit cases; the guarded app.py subprocess runs with FI_PROJECT_NAME unset and set; the README contract checks the new row and snippet. Passes against repository source, published traceAI-openai 0.1.10 and future-agi/traceAI#249. Co-Authored-By: Claude Opus 5.5 --- python/examples/doubleword/README.md | 7 ++-- python/examples/doubleword/src/app.py | 2 +- .../tests/test_doubleword_recipe.py | 38 +++++++++++++++++-- 3 files changed, 40 insertions(+), 7 deletions(-) diff --git a/python/examples/doubleword/README.md b/python/examples/doubleword/README.md index 2a08c67b7..d0a6c66d9 100644 --- a/python/examples/doubleword/README.md +++ b/python/examples/doubleword/README.md @@ -26,6 +26,7 @@ pip install traceAI-openai fi-instrumentation-otel openai | `DOUBLEWORD_MODEL` | Yes, unless `--model` is supplied | Pick an available model from Doubleword's catalog using its [API reference](https://doubleword.ai/llms.txt). | | `DOUBLEWORD_BASE_URL` | No | Defaults to `https://api.doubleword.ai/v1`; optional proxy override. | | `FI_BASE_URL` | No | Optional Future AGI collector origin; tracer only. | +| `FI_PROJECT_NAME` | No | Observe project name; defaults to `doubleword-example`; tracer only. | The application exits 2 for a missing or empty `DOUBLEWORD_API_KEY`, a missing or empty `DOUBLEWORD_MODEL` without `--model`, or an invalid `DOUBLEWORD_BASE_URL`. It checks these before tracing or network calls. The recipe does not check the Future AGI keys itself. @@ -45,7 +46,7 @@ python src/app.py python src/app.py --stream ``` -You can also pass `--model` and `--prompt`. The project name defaults to `doubleword-example`. +You can also pass `--model` and `--prompt`. Set `FI_PROJECT_NAME` to send the traces to another Observe project; unset or empty, the project name defaults to `doubleword-example`. ## Code @@ -58,8 +59,8 @@ from fi_instrumentation.fi_types import ProjectType from openai import OpenAI from traceai_openai import OpenAIInstrumentor -provider = register(project_name="doubleword-example", project_type=ProjectType.OBSERVE, - set_global_tracer_provider=False, verbose=False) +provider = register(project_name=os.environ.get("FI_PROJECT_NAME") or "doubleword-example", # as in src/app.py + project_type=ProjectType.OBSERVE, set_global_tracer_provider=False, verbose=False) OpenAIInstrumentor().instrument(tracer_provider=provider) with OpenAI(base_url="https://api.doubleword.ai/v1", api_key=os.environ["DOUBLEWORD_API_KEY"]) as client: response = client.chat.completions.create(model=os.environ["DOUBLEWORD_MODEL"], diff --git a/python/examples/doubleword/src/app.py b/python/examples/doubleword/src/app.py index e4da2bd6b..8d3326604 100644 --- a/python/examples/doubleword/src/app.py +++ b/python/examples/doubleword/src/app.py @@ -78,7 +78,7 @@ def make_client(base_url: str | None = None, api_key: str | None = None, http_cl def setup_tracing(project_name: str | None = None): provider = register( - project_name=project_name or "doubleword-example", + project_name=project_name or os.environ.get("FI_PROJECT_NAME") or "doubleword-example", project_type=ProjectType.OBSERVE, set_global_tracer_provider=False, verbose=False, diff --git a/python/examples/doubleword/tests/test_doubleword_recipe.py b/python/examples/doubleword/tests/test_doubleword_recipe.py index ec140ac87..32727c65a 100644 --- a/python/examples/doubleword/tests/test_doubleword_recipe.py +++ b/python/examples/doubleword/tests/test_doubleword_recipe.py @@ -185,6 +185,33 @@ def handler(request): assert receiver.requests()[0]["resource_attributes"][0]["project_name"] == project +@pytest.mark.parametrize("env_project, explicit, expected", [ + (None, None, "doubleword-example"), + ("", None, "doubleword-example"), + ("doubleword-from-env", None, "doubleword-from-env"), + ("doubleword-from-env", "doubleword-explicit", "doubleword-explicit"), +], ids=["default", "empty", "env", "explicit"]) +def test_setup_tracing_project_name(monkeypatch, env_project, explicit, expected): + # FI_PROJECT_NAME overrides the recipe default; an explicit argument wins over both. + # Never fall through to register()'s generic DEFAULT_PROJECT_NAME. + if env_project is not None: + monkeypatch.setenv("FI_PROJECT_NAME", env_project) + with Receiver() as receiver: + monkeypatch.setenv("FI_BASE_URL", receiver.origin) + provider = app.setup_tracing(explicit) + try: + with mock_client(lambda request: httpx.Response(200, json=completion())) as client: + client.chat.completions.create(model=MODEL, messages=[{"role": "user", "content": "Project control"}]) + only_span(receiver, provider) + exports = receiver.requests() + assert len(exports) == 1 + assert exports[0]["resource_attributes"][0]["project_name"] == expected + finally: + provider.force_flush() + OpenAIInstrumentor().uninstrument() + provider.shutdown() + + def test_usage_absent_is_omitted(journey): with journey() as (receiver, provider, _), mock_client(lambda r: httpx.Response(200, json=completion(usage=False))) as client: client.chat.completions.create(model=MODEL, messages=[{"role": "user", "content": "Usage control"}]) @@ -541,11 +568,14 @@ def child_environment(receiver, fake, directory): return env +@pytest.mark.parametrize("env_project", [None, "doubleword-subprocess-env"], ids=["default-project", "env-project"]) @pytest.mark.parametrize("streaming", [False, True]) -def test_app_subprocess_loopback_chat_and_stream(streaming): +def test_app_subprocess_loopback_chat_and_stream(streaming, env_project): with Receiver() as receiver, FakeOpenAI() as fake, tempfile.TemporaryDirectory(dir=RECIPE, prefix=".test-run-") as temporary: directory = Path(temporary) env = child_environment(receiver, fake, directory) + if env_project: + env["FI_PROJECT_NAME"] = env_project argv = [sys.executable, str(RECIPE / "src/app.py"), "--prompt", "subprocess-prompt-control"] if streaming: argv.append("--stream") @@ -587,7 +617,7 @@ def test_app_subprocess_loopback_chat_and_stream(streaming): assert exports[0]["path"] == "/tracer/v1/traces" assert exports[0]["headers"]["x-api-key"] == FI_KEY assert exports[0]["headers"]["x-secret-key"] == FI_SECRET - assert exports[0]["resource_attributes"][0]["project_name"] == "doubleword-example" + assert exports[0]["resource_attributes"][0]["project_name"] == (env_project or "doubleword-example") assert VENDOR_KEY not in json.dumps(spans + exports) assert VENDOR_KEY not in result.stdout + result.stderr @@ -618,8 +648,10 @@ def test_readme_requirements_and_public_contract(): assert readme.startswith("# Doubleword (OpenAI-compatible) with traceAI") assert app.DEFAULT_BASE_URL in readme assert "provider field says `openai`" in readme - for name in (app.API_KEY_ENV, app.BASE_URL_ENV, app.MODEL_ENV, "FI_API_KEY", "FI_SECRET_KEY", "FI_BASE_URL", "FI_HIDE_INPUTS", "FI_HIDE_OUTPUTS", "FI_HIDE_EMBEDDING_VECTORS"): + for name in (app.API_KEY_ENV, app.BASE_URL_ENV, app.MODEL_ENV, "FI_API_KEY", "FI_SECRET_KEY", "FI_BASE_URL", "FI_PROJECT_NAME", "FI_HIDE_INPUTS", "FI_HIDE_OUTPUTS", "FI_HIDE_EMBEDDING_VECTORS"): assert name in readme + assert "| `FI_PROJECT_NAME` | No | Observe project name; defaults to `doubleword-example`; tracer only. |" in readme + assert 'project_name=os.environ.get("FI_PROJECT_NAME") or "doubleword-example"' in readme assert "gen_ai.request.model" in readme assert "service_tier=\"flex\"" in readme assert "embedding.embeddings" in readme From fa5338f306b1bea1db4ea9aab0709badedeba4c9 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek <8347993+nik13@users.noreply.github.com> Date: Tue, 27 Oct 2026 21:14:11 -0700 Subject: [PATCH 6/7] docs(doubleword): align recipe with official API contracts Keep endpoint, model and auth guidance factual. Add exact-head offline recipe CI for repository source, the SDK floor and published packages. Co-Authored-By: Claude Code --- .../openai-compatible-recipe-tests.yml | 106 ++++++++++++++++++ python/examples/doubleword/README.md | 38 ++++--- python/examples/doubleword/requirements.txt | 2 +- .../tests/test_doubleword_recipe.py | 2 +- 4 files changed, 131 insertions(+), 17 deletions(-) create mode 100644 .github/workflows/openai-compatible-recipe-tests.yml diff --git a/.github/workflows/openai-compatible-recipe-tests.yml b/.github/workflows/openai-compatible-recipe-tests.yml new file mode 100644 index 000000000..0e2587e85 --- /dev/null +++ b/.github/workflows/openai-compatible-recipe-tests.yml @@ -0,0 +1,106 @@ +name: OpenAI-compatible recipe tests + +on: + pull_request: + branches: [main, dev] + paths: + - 'python/examples/**' + - 'python/fi_instrumentation/**' + - 'python/frameworks/openai/**' + - 'python/tests/harness/**' + - '.github/workflows/openai-compatible-recipe-tests.yml' + push: + branches: [dev] + paths: + - 'python/examples/**' + - 'python/fi_instrumentation/**' + - 'python/frameworks/openai/**' + - 'python/tests/harness/**' + - '.github/workflows/openai-compatible-recipe-tests.yml' + workflow_dispatch: + +permissions: + contents: read + +jobs: + plan: + name: Recipe plan + runs-on: ubuntu-24.04 + outputs: + matrix: ${{ steps.plan.outputs.matrix }} + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + ref: ${{ github.event.pull_request.head.sha || github.sha }} + persist-credentials: false + - id: plan + run: | + python3 - <<'PY' + import json, os + from pathlib import Path + matrix = [] + for recipe in ('databricks', 'lepton', 'orcarouter', 'anannas-ai', 'doubleword'): + root = Path('python/examples') / recipe + if not root.exists(): + continue # Recipes may land independently. + assert (root / 'src/app.py').is_file(), f'{recipe}: missing app' + assert (root / 'requirements.txt').is_file(), f'{recipe}: missing requirements' + assert list((root / 'tests').glob('test_*.py')), f'{recipe}: missing tests' + for sdk, mode in (('3.24.0', 'source'), ('1.69.0', 'source'), ('3.24.0', 'published')): + matrix.append({'recipe': recipe, 'sdk': sdk, 'mode': mode}) + assert matrix, 'No recipe tests found' + with open(os.environ['GITHUB_OUTPUT'], 'a') as out: + out.write('matrix=' + json.dumps({'include': matrix}) + '\n') + PY + recipe: + name: Recipe (${{ matrix.recipe }}, openai ${{ matrix.sdk }}, ${{ matrix.mode }}) + needs: plan + runs-on: ubuntu-24.04 + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.plan.outputs.matrix) }} + env: + PYTHONDONTWRITEBYTECODE: '1' + FI_BASE_URL: http://127.0.0.1:9 + FI_GRPC_URL: http://127.0.0.1:9 + RECIPE: ${{ matrix.recipe }} + SDK_VERSION: ${{ matrix.sdk }} + PACKAGE_MODE: ${{ matrix.mode }} + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + ref: ${{ github.event.pull_request.head.sha || github.sha }} + persist-credentials: false + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: '3.11.9' + - name: Install SDK and harness dependencies + run: | + python -m pip install "openai==$SDK_VERSION" httpx 'wrapt<2' \ + opentelemetry-api opentelemetry-sdk opentelemetry-instrumentation \ + opentelemetry-exporter-otlp-proto-http requests protobuf \ + opentelemetry-proto pydantic jsonschema pytest + if [ "$PACKAGE_MODE" = published ]; then + python -m pip install 'traceAI-openai==0.1.10' 'fi-instrumentation-otel==1.1.0' + fi + - name: Run isolated recipe contracts + run: | + if [ "$PACKAGE_MODE" = published ]; then + export PYTHONPATH="python/examples/$RECIPE/src:python/tests" + else + export PYTHONPATH="python/examples/$RECIPE/src:python:python/frameworks/openai:python/tests" + fi + python -m pytest "python/examples/$RECIPE/tests" -q \ + --noconftest -p no:cacheprovider -o addopts='' -rfEs + result: + name: OpenAI-compatible recipes + if: always() + needs: [plan, recipe] + runs-on: ubuntu-24.04 + steps: + - name: Require every planned recipe contract + env: + PLAN_RESULT: ${{ needs.plan.result }} + RECIPE_RESULT: ${{ needs.recipe.result }} + run: test "$PLAN_RESULT" = success && test "$RECIPE_RESULT" = success diff --git a/python/examples/doubleword/README.md b/python/examples/doubleword/README.md index d0a6c66d9..7f6879d24 100644 --- a/python/examples/doubleword/README.md +++ b/python/examples/doubleword/README.md @@ -4,7 +4,7 @@ Trace Chat Completions through the official `openai` Python SDK at Doubleword's ## Install -From this directory, install the tested pins: +From this directory, install the dependency pins: ```bash pip install -r requirements.txt @@ -85,15 +85,17 @@ Cost is unknown when the pricing table has no matching model. If a provider reso ## Provider specifics -Doubleword documents the following tiers in its [API reference](https://doubleword.ai/llms.txt): +This recipe uses synchronous Chat Completions at `https://api.doubleword.ai/v1`. +The OpenAI SDK sends `Authorization: Bearer `. Choose a model ID from the +[model catalog](https://docs.doubleword.ai/inference-api/models) or `client.models.list()`; +pass the exact returned ID, including its organization prefix. For dedicated +realtime deployments, Doubleword supplies your organization-specific model names. -| Tier | What this recipe covers | -|---|---| -| Realtime | OpenAI-compatible request and stream fixtures. Doubleword says the self-serve endpoint is "designed for development and testing" and "not suitable for production workloads". | -| Async | A standard request with `service_tier="flex"`. The fixture confirms the value reaches both the request body and `gen_ai.request.parameters`. | -| Dedicated Realtime | Doubleword's production tier; deployments are outside this recipe. | - -Both tested SDK versions (the 1.69.0 floor and 3.24.0) accept the native `service_tier` parameter and send `flex` unchanged; the test checks the transmitted value and the recorded parameter. Older SDK type annotations may not list `flex`; that does not change the request. +Doubleword documents async inference separately through the Responses API with +`service_tier="flex"`, or through its Autobatcher. The Chat Completions CLI here +does not select an async tier. Self-serve realtime uses shared, concurrency-limited +capacity; dedicated realtime is a separate deployment option. A 429 with +`inflight_limit_exceeded` indicates the account's per-model concurrency limit. The same [API reference](https://doubleword.ai/llms.txt) lists embeddings. An OpenAI-shaped embedding fixture at `/v1/embeddings` produces a `CreateEmbeddingResponse` span with `gen_ai.span.kind=EMBEDDING` and provider `openai` (`doubleword` in later versions). The model id from the response appears in `embedding.model_name`; up to 0.1.10 `gen_ai.request.model` is absent, and later versions set it too. Requested model parameters and input/total token counts are recorded. @@ -110,8 +112,7 @@ Embedding spans also export raw response JSON under `embedding.embeddings`, vect The Batch API is not covered by this recipe ([Doubleword API reference](https://doubleword.ai/llms.txt)). - The Anthropic-compatible Messages API and Dedicated deployments are outside this recipe ([Doubleword API reference](https://doubleword.ai/llms.txt)). -- Responses API, tools, structured outputs, cancellation, console imports, and pricing-table changes are not tested here. -- This recipe was not tested against the live provider (no paid call); the tests use a local fake of the OpenAI API and in-process fixtures. Streaming and embeddings results establish OpenAI-shaped fixture compatibility. +- Responses API, tools, structured outputs, cancellation, console imports, and pricing-table changes are outside this recipe. ## Tests @@ -146,10 +147,17 @@ env -u PYTHONPATH PYTHONDONTWRITEBYTECODE=1 \ pytest python/examples/doubleword/tests -q -p no:cacheprovider --noconftest -o addopts= -rfEs ``` -| Python | `openai` | traceAI packages | Result | +| Python | `openai` | traceAI packages | Local check configuration | |---|---|---|---| -| 3.10, 3.11, 3.12, 3.13 | 3.24.0 | `traceai-openai` and `fi_instrumentation` from this repository | Full suite | -| 3.11 | 1.69.0 (the `traceai-openai` floor) | from this repository | Full suite | -| 3.11 | 3.24.0 | published `traceAI-openai==0.1.10` and `fi-instrumentation-otel==1.1.0` from PyPI | Full suite | +| 3.10, 3.11, 3.12, 3.13 | 3.24.0 | `traceai-openai` and `fi_instrumentation` from this repository | Commands above | +| 3.11 | 1.69.0 (the `traceai-openai` floor) | from this repository | Commands above | +| 3.11 | 3.24.0 | published `traceAI-openai==0.1.10` and `fi-instrumentation-otel==1.1.0` from PyPI | Commands above | The chat fixtures use `deepseek-ai/DeepSeek-V4-Flash`, a model id listed in Doubleword's [API reference](https://doubleword.ai/llms.txt). Its current catalog availability is not asserted. The dated response model and embedding model are synthetic fixture ids. + +## Official API references + +- [Realtime inference](https://docs.doubleword.ai/inference-api/realtime-inference) +- [API keys](https://docs.doubleword.ai/inference-api/creating-an-api-key) +- [Model catalog](https://docs.doubleword.ai/inference-api/models) +- [Async Responses API](https://docs.doubleword.ai/inference-api/async-inference) diff --git a/python/examples/doubleword/requirements.txt b/python/examples/doubleword/requirements.txt index 5ae0f7608..19274c298 100644 --- a/python/examples/doubleword/requirements.txt +++ b/python/examples/doubleword/requirements.txt @@ -1,4 +1,4 @@ -# Tested recipe pins; the suite also covers the OpenAI SDK floor, 1.69.0. +# OpenAI SDK and published traceAI package pins. openai==3.24.0 traceAI-openai==0.1.10 fi-instrumentation-otel==1.1.0 diff --git a/python/examples/doubleword/tests/test_doubleword_recipe.py b/python/examples/doubleword/tests/test_doubleword_recipe.py index 32727c65a..eeee193da 100644 --- a/python/examples/doubleword/tests/test_doubleword_recipe.py +++ b/python/examples/doubleword/tests/test_doubleword_recipe.py @@ -659,7 +659,7 @@ def test_readme_requirements_and_public_contract(): assert "TBD" not in readme assert "https://doubleword.ai/llms.txt" in readme assert "https://app.doubleword.ai/api-keys" in readme - assert "not suitable for production workloads" in readme + assert "shared, concurrency-limited" in readme for py_path in ("python/examples/doubleword/src:python:python/frameworks/openai:python/tests", "python/examples/doubleword/src:python/tests"): assert f'PYTHONPATH="{py_path}"' in readme assert readme.count("pytest python/examples/doubleword/tests -q -p no:cacheprovider --noconftest -o addopts= -rfEs") == 2 From d8f7c90d38d6586b3961991f72c733efe1d1556f Mon Sep 17 00:00:00 2001 From: Nikhil Pareek <8347993+nik13@users.noreply.github.com> Date: Tue, 27 Oct 2026 21:23:07 -0700 Subject: [PATCH 7/7] test(doubleword): pin current and published error text contracts Assert exact status descriptions and qualified exception types across fi-core versions, including the newer redundant-type-prefix cleanup. Co-Authored-By: Claude Code --- .../examples/doubleword/tests/test_doubleword_recipe.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/python/examples/doubleword/tests/test_doubleword_recipe.py b/python/examples/doubleword/tests/test_doubleword_recipe.py index eeee193da..027bf14ca 100644 --- a/python/examples/doubleword/tests/test_doubleword_recipe.py +++ b/python/examples/doubleword/tests/test_doubleword_recipe.py @@ -298,7 +298,14 @@ def handler(request): span, attrs = only_span(receiver, provider) assert span["name"] == "ChatCompletion" assert span["status"]["code"] == "STATUS_CODE_ERROR" - assert error_class.__name__ in span["status"]["message"] + # Current fi-core keeps the error type in exception.type and removes + # its redundant status prefix; published 1.1.0 retains that prefix. + expected_status = str(caught.value) + if not hasattr(fi_instrumentation.TraceConfig, "hide_error_details"): + expected_status = f"{error_class.__name__}: {expected_status}" + assert span["status"]["message"] == expected_status + [exception] = [event for event in span["events"] if event["name"] == "exception"] + assert attributes(exception)["exception.type"] == f"{error_class.__module__}.{error_class.__qualname__}" assert attrs["gen_ai.span.kind"] == "LLM" assert attrs["gen_ai.provider.name"] in PROVIDER_NAMES assert any(event["name"] == "exception" for event in span["events"])