diff --git a/.github/workflows/openai-compatible-recipe-tests.yml b/.github/workflows/openai-compatible-recipe-tests.yml index 0205fc2b0..2a4de3595 100644 --- a/.github/workflows/openai-compatible-recipe-tests.yml +++ b/.github/workflows/openai-compatible-recipe-tests.yml @@ -39,7 +39,7 @@ jobs: import json, os from pathlib import Path matrix = [] - for recipe in ('databricks', 'lepton', 'orcarouter', 'anannas-ai', 'doubleword', 'perplexity'): + for recipe in ('databricks', 'lepton', 'orcarouter', 'anannas-ai', 'doubleword', 'perplexity', 'qwen'): root = Path('python/examples') / recipe if not root.exists(): continue # Recipes may land independently. diff --git a/python/examples/qwen/README.md b/python/examples/qwen/README.md new file mode 100644 index 000000000..fb1457abd --- /dev/null +++ b/python/examples/qwen/README.md @@ -0,0 +1,194 @@ +# Qwen (OpenAI-compatible) with traceAI + +Trace Qwen Chat Completions through the official `openai` Python SDK using Alibaba Cloud Model Studio's [OpenAI-compatible API](https://www.alibabacloud.com/help/en/model-studio/compatibility-of-openai-with-dashscope). This recipe uses the existing `traceai-openai` instrumentor; it needs no Qwen-specific tracing package or DashScope SDK. + +## Install + +From `python/examples/qwen/`, install the dependency pins: + +```bash +pip install -r requirements.txt +``` + +The plain install form is: + +```bash +pip install traceAI-openai fi-instrumentation-otel openai +``` + +## Configure + +| Variable | Required | Purpose | +| --- | --- | --- | +| `FI_API_KEY` | For Future AGI export | Future AGI API key; tracer only. | +| `FI_SECRET_KEY` | For Future AGI export | Future AGI secret key; tracer only. | +| `DASHSCOPE_API_KEY` | Yes | Model Studio API key for the selected region and workspace; OpenAI client only. | +| `DASHSCOPE_MODEL` | Yes, unless `--model` is supplied | A model ID available in your region and workspace. The example uses `qwen-plus`. | +| `DASHSCOPE_BASE_URL` | No | Your console workspace's OpenAI-compatible SDK base URL. Defaults to `https://dashscope-intl.aliyuncs.com/compatible-mode/v1`. | +| `FI_BASE_URL` | No | Future AGI collector origin; defaults to `https://api.futureagi.com`. The tracer adds `/tracer/v1/traces`. | +| `FI_PROJECT_NAME` | No | Observe project name. Defaults to `qwen-openai-recipe`. Goes to the tracer only. | + +The default is the legacy international (Singapore) host. Alibaba recommends a workspace-dedicated domain for production; the legacy hosts are still accepted ([Base URL overview](https://www.alibabacloud.com/help/en/model-studio/base-url)). Copy the API Host from your console workspace, add `/compatible-mode/v1`, and set `DASHSCOPE_BASE_URL`. The API key, workspace, region, and billing plan must match the endpoint. Obtain a key using [Alibaba's API-key instructions](https://www.alibabacloud.com/help/en/model-studio/get-api-key). + +The application exits 2 for a missing or empty model/key or an invalid base URL, before tracing or network calls. It does not validate Future AGI credentials itself. + +## Run + +Run from `python/examples/qwen/`. Replace the key placeholders with your own keys. `qwen-plus` is the model used in Alibaba's [streaming example](https://www.alibabacloud.com/help/en/model-studio/stream); select a model available to your region and account from the [model catalog](https://www.alibabacloud.com/help/en/model-studio/models). + +```bash +export FI_API_KEY="placeholder-futureagi-key" +export FI_SECRET_KEY="placeholder-futureagi-secret" +export DASHSCOPE_API_KEY="placeholder-qwen-key" +export DASHSCOPE_MODEL="qwen-plus" +# Singapore shared endpoint; use your console workspace's SDK base when available. +export DASHSCOPE_BASE_URL="https://dashscope-intl.aliyuncs.com/compatible-mode/v1" +python src/app.py +python src/app.py --stream +``` + +Use `--prompt "What is a rainbow?"` to change the prompt. `--model` overrides `DASHSCOPE_MODEL`. Set `FI_PROJECT_NAME` to send traces to another Observe project; unset or empty, they go to `qwen-openai-recipe`. Streaming requests set `stream_options={"include_usage": True}` and consume the final usage chunk, which can have an empty `choices` list. + +## Code + +Instrument before creating the client. The runnable [src/app.py](src/app.py) also validates the base URL, model, and key before tracing, and handles streaming. This core flow honors the same project and endpoint overrides: + +```python +import os +from openai import OpenAI +from fi_instrumentation import register +from fi_instrumentation.fi_types import ProjectType +from traceai_openai import OpenAIInstrumentor + +provider = register( + project_name=os.environ.get("FI_PROJECT_NAME") or "qwen-openai-recipe", + project_type=ProjectType.OBSERVE, + set_global_tracer_provider=False, + verbose=False, +) +OpenAIInstrumentor().instrument(tracer_provider=provider) +base_url = os.environ.get("DASHSCOPE_BASE_URL") or "https://dashscope-intl.aliyuncs.com/compatible-mode/v1" +try: + with OpenAI(base_url=base_url, api_key=os.environ["DASHSCOPE_API_KEY"]) as client: + response = client.chat.completions.create( + model=os.environ["DASHSCOPE_MODEL"], + messages=[{"role": "user", "content": "What is a rainbow?"}], + ) + print(response.choices[0].message.content or "") +finally: + provider.force_flush() +``` + +## What you see in Future AGI + +The shared instrumentor emits a `ChatCompletion` span with `gen_ai.span.kind=LLM` per completed chat call. Non-streamed `gen_ai.request.model` is the model id the provider returns in its response; `gen_ai.request.parameters` contains the requested ID unless invocation parameters are hidden. + +Up to `traceai-openai` 0.1.10, the provider field says `openai`, including for Qwen. Current source after [#249](https://github.com/future-agi/traceAI/pull/249) uses the explicit host mapping below for `gen_ai.provider.name`, on chat, streamed, and failed calls: + +| Host | Published 0.1.10 | Current source | +| --- | --- | --- | +| `dashscope.aliyuncs.com` (Beijing) | `openai` | `dashscope` | +| `dashscope-intl.aliyuncs.com` (Singapore) | `openai` | `dashscope` | +| `dashscope-us.aliyuncs.com` (Virginia) | `openai` | `dashscope` | +| `cn-hongkong.dashscope.aliyuncs.com` (Hong Kong shared) | `openai` | `openai` | +| `*.maas.aliyuncs.com` (including Hong Kong workspace hosts) | `openai` | `dashscope` | +| Custom proxies and loopback hosts | `openai` | `openai` | + +The Hong Kong shared host is not in the current provider map, so it retains `openai`; its workspace-dedicated host is mapped to `dashscope`. This affects the tracing label, not which endpoint receives the request. + +- Streamed output accumulates in `output.value`. Up to 0.1.10, streamed and failed calls have no `gen_ai.request.model` attribute; later versions record the requested model there. The requested model remains in `gen_ai.request.parameters` unless `FI_HIDE_LLM_INVOCATION_PARAMETERS=true`. +- When the provider supplies usage, the span records `gen_ai.usage.input_tokens`, `gen_ai.usage.output_tokens`, and `gen_ai.usage.total_tokens`. Missing usage is omitted, never set to zero. Without `include_usage`, the OpenAI-compatible stream does not return usage by default; [Alibaba documents](https://www.alibabacloud.com/help/en/model-studio/stream) requesting it in the last chunk. +- Failed calls produce ERROR spans with exception events. The recipe keeps the SDK's default retry policy; the error fixtures disable retries so each case sees one request. + +## Provider specifics + +### Regions and endpoints + +The [Base URL overview](https://www.alibabacloud.com/help/en/model-studio/base-url) and [regional endpoint table](https://www.alibabacloud.com/help/en/model-studio/regions) list the following pay-as-you-go SDK bases. Replace `{WorkspaceId}` with your actual workspace ID from the console; do not pass the placeholder to the SDK. + +| Region | Shared DashScope base | Workspace-dedicated base | +| --- | --- | --- | +| China (Beijing) | `https://dashscope.aliyuncs.com/compatible-mode/v1` | `https://{WorkspaceId}.cn-beijing.maas.aliyuncs.com/compatible-mode/v1` | +| Singapore | `https://dashscope-intl.aliyuncs.com/compatible-mode/v1` | `https://{WorkspaceId}.ap-southeast-1.maas.aliyuncs.com/compatible-mode/v1` | +| US (Virginia) | `https://dashscope-us.aliyuncs.com/compatible-mode/v1` | `https://{WorkspaceId}.us-east-1.maas.aliyuncs.com/compatible-mode/v1` | +| China (Hong Kong) | `https://cn-hongkong.dashscope.aliyuncs.com/compatible-mode/v1` | `https://{WorkspaceId}.cn-hongkong.maas.aliyuncs.com/compatible-mode/v1` | +| Japan (Tokyo) | Use your workspace endpoint | `https://{WorkspaceId}.ap-northeast-1.maas.aliyuncs.com/compatible-mode/v1` | +| Germany (Frankfurt) | Use your workspace endpoint | `https://{WorkspaceId}.eu-central-1.maas.aliyuncs.com/compatible-mode/v1` | + +For Virginia, prefer the workspace-dedicated endpoint from your console. Alibaba's [Base URL overview](https://www.alibabacloud.com/help/en/model-studio/base-url) lists the shared `dashscope-us.aliyuncs.com` endpoint above, while its [regional endpoint table](https://www.alibabacloud.com/help/en/model-studio/regions) marks the Virginia shared domain as unsupported; both list the workspace-dedicated endpoint. + +The SDK appends `/chat/completions` and sends `Authorization: Bearer `. Do not append that endpoint path yourself. Workspace endpoints require a key belonging to that workspace; shared endpoints accept keys from workspaces in the same region. Keys and model availability are region-specific. Use the exact Model Studio model ID, such as `qwen-plus`, rather than a Hugging Face repository name. + +`check_base_url` rejects literal or repeatedly URL-encoded `{WorkspaceId}` placeholders. On `aliyuncs.com` hosts, it requires HTTPS and exactly `/compatible-mode/v1` or `/compatible-mode/v1/`: at most one trailing slash, with no path prefix, endpoint suffix, query, or fragment. All URLs must be absolute HTTP(S) URLs without URL credentials, whitespace, control characters, malformed ports, or invalid host spellings. Unicode dot spellings cannot bypass validation. Uppercase hosts and a single trailing DNS dot are checked the same way. Customer proxies and loopback servers may use their own paths; interior underscores in proxy host labels are allowed. Valid URLs are returned unchanged; the SDK adds a trailing slash to `client.base_url`. + +### Errors + +The [Chat API reference](https://www.alibabacloud.com/help/en/model-studio/compatibility-of-openai-with-dashscope) and [error-code guide](https://www.alibabacloud.com/help/en/model-studio/error-code) describe provider errors: + +- HTTP 400: invalid request; check parameters and model capabilities. +- HTTP 401 `invalid_api_key`: check the key and its region, workspace, and billing-plan match. A cross-region key can produce `Incorrect API key provided`. This comes from Model Studio, not a Future AGI authentication failure; `FI_API_KEY` and `FI_SECRET_KEY` are not involved in that provider request. +- HTTP 404: check the model ID, its availability/permissions, and the endpoint. +- HTTP 429: rate limit or quota/billing limit; inspect the provider message before retrying. +- HTTP 500: provider service error; retry according to the provider's guidance. + +## Privacy + +Set `FI_HIDE_INPUTS=true` to mask prompt attributes or `FI_HIDE_OUTPUTS=true` to mask response attributes in exported spans. You can also pass `config=TraceConfig(hide_inputs=True)` to `OpenAIInstrumentor().instrument(...)`, importing `TraceConfig` from `fi_instrumentation`. + +The provider still receives the prompt. Future AGI masking does not change the provider's own logging. The Model Studio key belongs only to the OpenAI client; Future AGI keys belong only to the tracer. + +## Limits / not covered + +- Qwen-Audio uses the DashScope native protocol; use the DashScope protocol for that model family rather than this Chat Completions recipe. +- The DashScope native SDK/protocol, Responses API, realtime audio, tools, and structured outputs are outside this recipe. +- Choose model availability and workspace access in your Model Studio console. This recipe does not provision workspaces, select billing plans, or modify the pricing table. + +## Tests + +Run from the repository root. Provider traffic uses `httpx.MockTransport` or a loopback server, and OTLP exports go to a loopback receiver. App subprocesses install a socket/DNS guard before imports. Negative controls target the real provider hostname and require the guard to reject it before network access. + +Repository source: + +```bash +env -u PYTHONPATH PYTHONDONTWRITEBYTECODE=1 \ + PYTHONPATH="python/examples/qwen/src:python:python/frameworks/openai:python/tests" \ + uv run --no-project --python 3.11 \ + --with 'openai==3.24.0' --with httpx --with 'wrapt<2' \ + --with opentelemetry-api --with opentelemetry-sdk --with opentelemetry-instrumentation \ + --with opentelemetry-exporter-otlp-proto-http --with requests --with protobuf \ + --with opentelemetry-proto --with pydantic --with jsonschema --with pytest \ + pytest python/examples/qwen/tests -q -p no:cacheprovider --noconftest -o addopts= -rfEs +``` + +For the SDK floor, replace `openai==3.24.0` with `openai==1.69.0`. + +Published packages (keep only the recipe and harness on `PYTHONPATH` so repository source does not shadow the installed packages): + +```bash +env -u PYTHONPATH PYTHONDONTWRITEBYTECODE=1 \ + PYTHONPATH="python/examples/qwen/src:python/tests" \ + uv run --no-project --python 3.11 \ + --with 'openai==3.24.0' --with 'traceAI-openai==0.1.10' --with 'fi-instrumentation-otel==1.1.0' \ + --with httpx --with protobuf --with opentelemetry-proto --with pytest \ + pytest python/examples/qwen/tests -q -p no:cacheprovider --noconftest -o addopts= -rfEs +``` + +[Recipe CI](../../../.github/workflows/openai-compatible-recipe-tests.yml) defines these three Python 3.11 cells: + +| `openai` | traceAI packages | +| --- | --- | +| 3.24.0 | Repository source | +| 1.69.0 | Repository source (SDK floor) | +| 3.24.0 | Published `traceAI-openai==0.1.10` + `fi-instrumentation-otel==1.1.0` | + +The fixtures cover chat, streaming with and without final usage, response-versus-request model IDs, provider errors, privacy controls, project-name precedence, strict URL validation, and CLI subprocesses. Synthetic response IDs and usage counts are fixture data, not provider output. Provider assertions require the exact label for each shared and workspace host, including the Hong Kong shared-host exception, across chat, streaming, and errors. Proxy and look-alike hosts must keep `openai`. Expected provider/model attributes are selected by the imported source or pinned published package, not by accepting either value. + +## Official API references + +- [OpenAI-compatible Chat API](https://www.alibabacloud.com/help/en/model-studio/compatibility-of-openai-with-dashscope) +- [Base URLs by region and plan](https://www.alibabacloud.com/help/en/model-studio/base-url) +- [Regions and endpoints](https://www.alibabacloud.com/help/en/model-studio/regions) +- [Obtain an API key](https://www.alibabacloud.com/help/en/model-studio/get-api-key) +- [Model catalog](https://www.alibabacloud.com/help/en/model-studio/models) +- [Streaming and token usage](https://www.alibabacloud.com/help/en/model-studio/stream) +- [Error codes](https://www.alibabacloud.com/help/en/model-studio/error-code) diff --git a/python/examples/qwen/requirements.txt b/python/examples/qwen/requirements.txt new file mode 100644 index 000000000..cb75780f2 --- /dev/null +++ b/python/examples/qwen/requirements.txt @@ -0,0 +1,4 @@ +# Recipe dependency pins; CI also includes the OpenAI SDK floor, 1.69.0. +openai==3.24.0 +traceAI-openai==0.1.10 +fi-instrumentation-otel==1.1.0 diff --git a/python/examples/qwen/src/app.py b/python/examples/qwen/src/app.py new file mode 100644 index 000000000..8f4db600b --- /dev/null +++ b/python/examples/qwen/src/app.py @@ -0,0 +1,133 @@ +"""Trace Qwen Chat Completions through the official OpenAI SDK.""" + +import argparse +import ipaddress +import os +import re +import sys +from urllib.parse import unquote, urlsplit + +import openai +from fi_instrumentation import register +from fi_instrumentation.fi_types import ProjectType +from traceai_openai import OpenAIInstrumentor + +DEFAULT_BASE_URL = "https://dashscope-intl.aliyuncs.com/compatible-mode/v1" +API_KEY_ENV = "DASHSCOPE_API_KEY" +BASE_URL_ENV = "DASHSCOPE_BASE_URL" +MODEL_ENV = "DASHSCOPE_MODEL" + + +def _fully_unquoted(value: str) -> str: + while (decoded := unquote(value)) != value: + value = decoded + return value + + +def check_base_url(url: str) -> str: + """Validate without rewriting; permit customer proxies and loopback servers.""" + def refuse(reason): + raise ValueError(f"{BASE_URL_ENV}: {reason}") + + if not url or any(c.isspace() or not c.isprintable() for c in url): + refuse("use an absolute URL without whitespace or control characters") + if "{WorkspaceId}" in _fully_unquoted(url): + refuse("Replace {WorkspaceId} with your workspace id from the console.") + try: + parts = urlsplit(url) + host = parts.hostname + parts.port # Reject malformed ports before tracing. + except ValueError: + refuse("use a valid absolute HTTP(S) URL") + if parts.username is not None or parts.password is not None: + refuse("remove URL credentials; supply the key through DASHSCOPE_API_KEY") + if parts.scheme not in ("http", "https") or not host: + refuse("use an absolute HTTP(S) URL with a host") + if not host.isascii(): + refuse("use a plain ASCII host with valid IDNA spelling") + comparable_host = host.lower().rstrip(".") + structural_host = host.lower().removesuffix(".") + try: + ipaddress.ip_address(structural_host) + except ValueError: + try: + structural_host.encode("ascii").decode("idna") + except UnicodeError: + refuse("use a plain ASCII host with valid IDNA spelling") + if len(structural_host) > 253 or not all( + re.fullmatch(r"[a-zA-Z0-9](?:[a-zA-Z0-9_-]{0,61}[a-zA-Z0-9])?", label) + for label in structural_host.split(".") + ): + refuse("use a valid ASCII host") + if comparable_host == "aliyuncs.com" or comparable_host.endswith(".aliyuncs.com"): + if parts.scheme != "https": + refuse("use HTTPS for the Model Studio endpoint") + if "?" in url or "#" in url: + refuse("remove the query or fragment from the SDK base URL") + if parts.path not in ("/compatible-mode/v1", "/compatible-mode/v1/"): + refuse("use exactly /compatible-mode/v1 with at most one trailing slash") + return url + + +def make_client( + base_url: str | None = None, api_key: str | None = None, http_client=None +) -> openai.OpenAI: + url = check_base_url( + base_url if base_url is not None else os.environ.get(BASE_URL_ENV) or DEFAULT_BASE_URL + ) + key = api_key if api_key is not None else os.environ[API_KEY_ENV] + return openai.OpenAI(base_url=url, api_key=key, http_client=http_client) + + +def setup_tracing(project_name: str | None = None): + provider = register( + project_name=project_name or os.environ.get("FI_PROJECT_NAME") or "qwen-openai-recipe", + project_type=ProjectType.OBSERVE, + set_global_tracer_provider=False, + verbose=False, + ) + OpenAIInstrumentor().instrument(tracer_provider=provider) + return provider + + +def main(argv=None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--stream", action="store_true") + parser.add_argument("--prompt", default="What is a rainbow?") + parser.add_argument("--model", help=f"Model id; otherwise use {MODEL_ENV}.") + args = parser.parse_args(argv) + try: + url = check_base_url(os.environ.get(BASE_URL_ENV) or DEFAULT_BASE_URL) + except ValueError as error: + print(str(error), file=sys.stderr) + return 2 + model = args.model or os.environ.get(MODEL_ENV) + if not model: + parser.error(f"Set {MODEL_ENV} or pass --model.") + if not os.environ.get(API_KEY_ENV): + parser.error(f"Set {API_KEY_ENV}.") + + provider = setup_tracing() + try: + with make_client(base_url=url) as client: + response = client.chat.completions.create( + model=model, + messages=[{"role": "user", "content": args.prompt}], + stream=args.stream, + **({"stream_options": {"include_usage": True}} if args.stream else {}), + ) + if args.stream: + with response: + for chunk in response: + if chunk.choices: + print(chunk.choices[0].delta.content or "", end="", flush=True) + print() + else: + print(response.choices[0].message.content or "") + finally: + provider.force_flush() + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/python/examples/qwen/tests/_fake_openai.py b/python/examples/qwen/tests/_fake_openai.py new file mode 100644 index 000000000..dca980fd4 --- /dev/null +++ b/python/examples/qwen/tests/_fake_openai.py @@ -0,0 +1,124 @@ +"""OpenAI-shaped fixtures and a loopback server; never forwards requests.""" + +import copy +import json +import threading +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +ANSWER = "Hello from local Qwen." +MODEL = "qwen-plus" +# Synthetic dated id. Not a documented Model Studio rewrite. No live call. +RESPONSE_MODEL = "qwen-plus-2026-09-01" +USAGE = {"prompt_tokens": 5, "completion_tokens": 7, "total_tokens": 12} +AUTH_ERROR = { + "error": { + "message": "Incorrect API key provided. ", + "type": "invalid_request_error", + "param": None, + "code": "invalid_api_key", + } +} + + +def completion(model=MODEL, include_usage=True): + body = { + "id": "chatcmpl-local-qwen", + "object": "chat.completion", + "created": 1, + "model": model, + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": ANSWER}, + "finish_reason": "stop", + } + ], + } + if include_usage: + body["usage"] = USAGE.copy() + return body + + +def stream_body(model=MODEL, include_usage=False): + chunks = [] + for index, content in enumerate(("Hello ", "from local ", "Qwen.")): + delta = {"content": content} + if index == 0: + delta["role"] = "assistant" + chunks.append( + { + "id": "chatcmpl-local-qwen", + "object": "chat.completion.chunk", + "created": 1, + "model": model, + "choices": [{"index": 0, "delta": delta, "finish_reason": None}], + } + ) + chunks.append( + { + "id": "chatcmpl-local-qwen", + "object": "chat.completion.chunk", + "created": 1, + "model": model, + "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}], + } + ) + if include_usage: + chunks.append({**chunks[-1], "choices": [], "usage": USAGE.copy()}) + return ("".join(f"data: {json.dumps(chunk)}\n\n" for chunk in chunks) + "data: [DONE]\n\n").encode() + + +class FakeOpenAI: + def __init__(self, error=False): + self._requests = [] + self._lock = threading.Lock() + owner = self + + class Handler(BaseHTTPRequestHandler): + def do_POST(self): + body = json.loads(self.rfile.read(int(self.headers["Content-Length"]))) + with owner._lock: + owner._requests.append( + { + "path": self.path, + "headers": {key.lower(): value for key, value in self.headers.items()}, + "body": body, + } + ) + if self.path != "/compatible-mode/v1/chat/completions": + self.send_error(404) + return + if error: + status, content_type, payload = 401, "application/json", json.dumps(AUTH_ERROR).encode() + elif body.get("stream"): + status, content_type = 200, "text/event-stream" + payload = stream_body( + body["model"], body.get("stream_options", {}).get("include_usage", False) + ) + else: + status, content_type, payload = 200, "application/json", json.dumps(completion(body["model"])).encode() + self.send_response(status) + self.send_header("Content-Type", content_type) + self.send_header("Content-Length", str(len(payload))) + self.end_headers() + self.wfile.write(payload) + + def log_message(self, _format, *args): + pass + + self._server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + self.base_url = f"http://127.0.0.1:{self._server.server_port}/compatible-mode/v1" + self._thread = threading.Thread(target=self._server.serve_forever, daemon=True) + + def __enter__(self): + self._thread.start() + return self + + def __exit__(self, *_args): + self._server.shutdown() + self._server.server_close() + self._thread.join() + + def requests(self): + with self._lock: + return copy.deepcopy(self._requests) diff --git a/python/examples/qwen/tests/loopback_guard/sitecustomize.py b/python/examples/qwen/tests/loopback_guard/sitecustomize.py new file mode 100644 index 000000000..8ff625122 --- /dev/null +++ b/python/examples/qwen/tests/loopback_guard/sitecustomize.py @@ -0,0 +1,45 @@ +"""Install before application imports and block non-loopback DNS/connections.""" + +import os +import socket +from pathlib import Path + +_getaddrinfo = socket.getaddrinfo +_connect = socket.socket.connect +_connect_ex = socket.socket.connect_ex + + +def _check(host, operation): + if isinstance(host, bytes): + host = host.decode("ascii", errors="replace") + if host not in ("127.0.0.1", "localhost"): + log = os.environ.get("LOOPBACK_GUARD_LOG") + if log: + with open(log, "a", encoding="utf-8") as handle: + handle.write(f"{operation}: {host}\n") + raise RuntimeError(f"Loopback guard refused {operation} to {host}") + + +def _guarded_getaddrinfo(host, *args, **kwargs): + _check(host, "getaddrinfo") + return _getaddrinfo(host, *args, **kwargs) + + +def _guarded_connect(self, address): + _check(address[0] if isinstance(address, tuple) else address, "connect") + return _connect(self, address) + + +def _guarded_connect_ex(self, address): + _check(address[0] if isinstance(address, tuple) else address, "connect_ex") + return _connect_ex(self, address) + + +socket.getaddrinfo = _guarded_getaddrinfo +socket.socket.connect = _guarded_connect +socket.socket.connect_ex = _guarded_connect_ex + +if log := os.environ.get("LOOPBACK_GUARD_LOG"): + Path(log).touch() +if ready := os.environ.get("LOOPBACK_GUARD_READY"): + Path(ready).write_text("installed\n", encoding="utf-8") diff --git a/python/examples/qwen/tests/test_qwen_recipe.py b/python/examples/qwen/tests/test_qwen_recipe.py new file mode 100644 index 000000000..327bfa562 --- /dev/null +++ b/python/examples/qwen/tests/test_qwen_recipe.py @@ -0,0 +1,751 @@ +"""Qwen recipe contracts: mocked provider traffic and real loopback OTLP exports.""" + +import atexit +import importlib.util +import json +import os +import re +import signal +import socket +import subprocess +import sys +import tempfile +import uuid +from contextlib import contextmanager +from importlib.metadata import version +from pathlib import Path +from urllib.parse import urlsplit + +import fi_instrumentation +import harness +import httpx +import openai +import pytest +import traceai_openai +from harness import Receiver +from opentelemetry import trace +from traceai_openai import OpenAIInstrumentor + +import app + +RECIPE = Path(__file__).resolve().parents[1] +# Identify the matrix implementation by its import path, not by inspecting the +# provider map or the exported value: a source regression must not pass as 0.1.10. +SOURCE_INSTRUMENTOR = Path(traceai_openai.__file__).resolve() == ( + RECIPE.parents[1] / "frameworks/openai/traceai_openai/__init__.py" +) +_fake_spec = importlib.util.spec_from_file_location( + "qwen_recipe_fake", RECIPE / "tests" / "_fake_openai.py" +) +_fake = importlib.util.module_from_spec(_fake_spec) +_fake_spec.loader.exec_module(_fake) +ANSWER = _fake.ANSWER +AUTH_ERROR = _fake.AUTH_ERROR +MODEL = _fake.MODEL +RESPONSE_MODEL = _fake.RESPONSE_MODEL +USAGE = _fake.USAGE +FakeOpenAI = _fake.FakeOpenAI +completion = _fake.completion +stream_body = _fake.stream_body +VENDOR_KEY = "placeholder-qwen-key" +FI_KEY = "placeholder-futureagi-key" +FI_SECRET = "placeholder-futureagi-secret" +DOCUMENTED_BASE = "https://dashscope-intl.aliyuncs.com/compatible-mode/v1" +WORKSPACE = "ws-" + "test0001" # Syntactic example, never a real workspace. +REGION_BASES = { + "Beijing": "https://{WorkspaceId}.cn-beijing.maas.aliyuncs.com/compatible-mode/v1", + "US (Virginia)": "https://{WorkspaceId}.us-east-1.maas.aliyuncs.com/compatible-mode/v1", + "Singapore": "https://{WorkspaceId}.ap-southeast-1.maas.aliyuncs.com/compatible-mode/v1", + "Japan (Tokyo)": "https://{WorkspaceId}.ap-northeast-1.maas.aliyuncs.com/compatible-mode/v1", + "China (Hong Kong)": "https://{WorkspaceId}.cn-hongkong.maas.aliyuncs.com/compatible-mode/v1", + "Germany (Frankfurt)": "https://{WorkspaceId}.eu-central-1.maas.aliyuncs.com/compatible-mode/v1", +} +# Explicit source expectations for each documented shared endpoint. Hong Kong's +# shared host is not mapped by #249; its workspace-dedicated host is mapped. +SHARED_PROVIDER_CASES = [ + (DOCUMENTED_BASE, "dashscope"), + ("https://dashscope.aliyuncs.com/compatible-mode/v1", "dashscope"), + ("https://dashscope-us.aliyuncs.com/compatible-mode/v1", "dashscope"), + ("https://cn-hongkong.dashscope.aliyuncs.com/compatible-mode/v1", "openai"), +] +SHARED_BASES = [base for base, _ in SHARED_PROVIDER_CASES] +REFUSED_URLS = [ + (REGION_BASES["Beijing"], "workspace id from the console"), + (REGION_BASES["Singapore"].replace("{WorkspaceId}", "%7BWorkspaceId%7D"), "workspace id from the console"), + (REGION_BASES["Japan (Tokyo)"].replace("{WorkspaceId}", "%7bWorkspaceId%7d"), "workspace id from the console"), + (DOCUMENTED_BASE + "?workspace={WorkspaceId}", "workspace id from the console"), + ("https://dashscope.aliyuncs.com", "/compatible-mode/v1"), + ("https://dashscope-intl.aliyuncs.com", "/compatible-mode/v1"), + ("https://cn-hongkong.dashscope.aliyuncs.com", "/compatible-mode/v1"), + (f"https://{WORKSPACE}.cn-beijing.maas.aliyuncs.com/v1", "/compatible-mode/v1"), + (DOCUMENTED_BASE + "/chat/completions", "/compatible-mode/v1"), + (DOCUMENTED_BASE + "//", "/compatible-mode/v1"), + ("https://aliyuncs.com/v1", "/compatible-mode/v1"), + # Host and encoding spellings must not bypass the checks. + ("https://dashscope.aliyuncs.com./", "/compatible-mode/v1"), + ("https://DashScope-Intl.AliyunCS.com.:443/v1", "/compatible-mode/v1"), + (REGION_BASES["Beijing"].replace("{WorkspaceId}", "%257BWorkspaceId%257D"), "workspace id from the console"), + # Exact path, not a suffix match; no extra trailing slashes or URL metadata. + (DOCUMENTED_BASE + "///", "/compatible-mode/v1"), + (DOCUMENTED_BASE.replace("/compatible-mode", "/prefix/compatible-mode"), "/compatible-mode/v1"), + (DOCUMENTED_BASE + "?", "query or fragment"), + (DOCUMENTED_BASE + "?api_key=placeholder", "query or fragment"), + (DOCUMENTED_BASE + "#", "query or fragment"), + (DOCUMENTED_BASE + "#section", "query or fragment"), + (DOCUMENTED_BASE.replace("https:", "http:"), "HTTPS"), + ("https://dashscope-intl。aliyuncs.com/v1", "ASCII host"), + ("https://dashscope-intl.aliyuncs.com/v1", "ASCII host"), + ("https://dashscope-intl。aliyuncs.com/v1", "ASCII host"), + ("https://dashscope-intl.aliyuncs.com../compatible-mode/v1", "valid ASCII host"), + ("https://user:placeholder@dashscope-intl.aliyuncs.com/compatible-mode/v1", "URL credentials"), + ("https://user:placeholder@example.invalid/custom", "URL credentials"), + ("", "absolute URL"), + ("/compatible-mode/v1", "absolute HTTP(S) URL"), + ("ftp://example.invalid/custom", "absolute HTTP(S) URL"), + ("https:///compatible-mode/v1", "absolute HTTP(S) URL"), + (" https://example.invalid/v1", "whitespace"), + ("https://example.invalid/\npath", "control characters"), + ("https://example.invalid/\x00path", "control characters"), + ("https://example.invalid:wrong/v1", "valid absolute HTTP(S) URL"), + ("https://example.invalid:70000/v1", "valid absolute HTTP(S) URL"), + ("http://[::1/v1", "valid absolute HTTP(S) URL"), + ("https://bad%2eexample.invalid/v1", "valid ASCII host"), + ("https://xn--/v1", "valid IDNA spelling"), + ("https://-bad.example/v1", "valid ASCII host"), + ("https://bad..example/v1", "valid ASCII host"), +] +ALLOWED_URLS = [ + *SHARED_BASES, + DOCUMENTED_BASE + "/", + *(base.replace("{WorkspaceId}", WORKSPACE) for base in REGION_BASES.values()), + f"https://{WORKSPACE}.cn-hongkong.maas.aliyuncs.com/compatible-mode/v1/", + "http://127.0.0.1:12345/compatible-mode/v1", + "http://localhost:12345/arbitrary", + "https://example.invalid/arbitrary", + "https://aliyuncs.com.example.invalid/arbitrary", + "https://DashScope-Intl.AliyunCS.com.:443/compatible-mode/v1/", + "http://[::1]:12345/custom/v1", + "http://myproj_proxy_1:12345/custom/v1", +] + +TEST_COMMAND = """env -u PYTHONPATH PYTHONDONTWRITEBYTECODE=1 \\ + PYTHONPATH="python/examples/qwen/src:python:python/frameworks/openai:python/tests" \\ + uv run --no-project --python 3.11 \\ + --with 'openai==3.24.0' --with httpx --with 'wrapt<2' \\ + --with opentelemetry-api --with opentelemetry-sdk --with opentelemetry-instrumentation \\ + --with opentelemetry-exporter-otlp-proto-http --with requests --with protobuf \\ + --with opentelemetry-proto --with pydantic --with jsonschema --with pytest \\ + pytest python/examples/qwen/tests -q -p no:cacheprovider --noconftest -o addopts= -rfEs""" + + +@pytest.fixture(autouse=True) +def isolated_environment(monkeypatch): + # Clear inherited tracing settings and proxies without displaying any values. + for name in tuple(os.environ): + if name.startswith(("FI_", "OTEL_", "DASHSCOPE_", "OPENAI_")) or name.lower() in { + "http_proxy", "https_proxy", "all_proxy", "no_proxy" + }: + monkeypatch.delenv(name, raising=False) + monkeypatch.setenv("FI_API_KEY", FI_KEY) + monkeypatch.setenv("FI_SECRET_KEY", FI_SECRET) + monkeypatch.setenv(app.API_KEY_ENV, VENDOR_KEY) + monkeypatch.setenv(app.MODEL_ENV, MODEL) + monkeypatch.setenv("PYTHONDONTWRITEBYTECODE", "1") + original_dns = socket.getaddrinfo + original_connect = socket.socket.connect + original_connect_ex = socket.socket.connect_ex + + def check(host): + if isinstance(host, bytes): + host = host.decode("ascii") + if host not in ("127.0.0.1", "localhost"): + raise AssertionError(f"Test refused non-loopback network access to {host}") + + def guarded_dns(host, *args, **kwargs): + check(host) + return original_dns(host, *args, **kwargs) + + def guarded_connect(sock, address): + check(address[0]) + return original_connect(sock, address) + + def guarded_connect_ex(sock, address): + check(address[0]) + return original_connect_ex(sock, address) + + monkeypatch.setattr(socket, "getaddrinfo", guarded_dns) + monkeypatch.setattr(socket.socket, "connect", guarded_connect) + monkeypatch.setattr(socket.socket, "connect_ex", guarded_connect_ex) + saved_signals = {number: signal.getsignal(number) for number in (signal.SIGINT, signal.SIGTERM)} + yield + instrumentor = OpenAIInstrumentor() + if instrumentor.is_instrumented_by_opentelemetry: + instrumentor.uninstrument() + for number, handler in saved_signals.items(): + signal.signal(number, handler) + + +@contextmanager +def trace_session(monkeypatch, **privacy): + for name, value in privacy.items(): + monkeypatch.setenv(name, str(value).lower()) + with Receiver() as receiver: + monkeypatch.setenv("FI_BASE_URL", receiver.origin) + project = f"qwen-recipe-{uuid.uuid4().hex}" + provider = app.setup_tracing(project) + try: + yield receiver, provider, project + finally: + OpenAIInstrumentor().uninstrument() + provider.shutdown() + atexit.unregister(provider.shutdown) + + +@pytest.fixture +def traced(monkeypatch): + with trace_session(monkeypatch) as session: + yield session + + +def attributes(span): + return { + entry["key"]: next(iter(entry["value"].values())) + for entry in span.get("attributes", []) + } + + +def assert_llm(receiver, project, model=MODEL, *, source_provider="dashscope"): + [span] = receiver.spans() + attrs = attributes(span) + assert span["name"] == "ChatCompletion" + assert attrs["gen_ai.span.kind"] == "LLM" + if model is None: + # 0.1.10 records the model only from non-streamed responses; later versions record the requested one. + assert attrs.get("gen_ai.request.model") == (MODEL if SOURCE_INSTRUMENTOR else None) + else: + assert attrs["gen_ai.request.model"] == model + assert attrs["gen_ai.provider.name"] == (source_provider if SOURCE_INSTRUMENTOR else "openai") + for request in receiver.requests(): + assert request["path"] == "/tracer/v1/traces" + assert request["headers"]["x-api-key"] == FI_KEY + assert request["headers"]["x-secret-key"] == FI_SECRET + [resource] = request["resource_attributes"] + assert resource["project_name"] == project + assert resource["project_type"] == "observe" + assert VENDOR_KEY not in json.dumps([receiver.spans(), receiver.requests()]) + return span, attrs + + +def assert_usage(attrs): + assert {name: int(value) for name, value in attrs.items() if name.startswith("gen_ai.usage.")} == { + "gen_ai.usage.input_tokens": USAGE["prompt_tokens"], + "gen_ai.usage.output_tokens": USAGE["completion_tokens"], + "gen_ai.usage.total_tokens": USAGE["total_tokens"], + } + + +def assert_vendor_request(request): + assert str(request.url) == DOCUMENTED_BASE + "/chat/completions" + assert request.headers["authorization"] == f"Bearer {VENDOR_KEY}" + assert "x-api-key" not in request.headers + assert "x-secret-key" not in request.headers + assert FI_KEY not in str(request.headers) + assert FI_SECRET not in str(request.headers) + + +def mock_client(body, *, status=200, streaming=False, echo_request_model=True): + requests = [] + + def handler(request): + requests.append(request) + if streaming: + return httpx.Response(status, content=body, headers={"Content-Type": "text/event-stream"}) + response_body = body + if status == 200 and body.get("object") == "chat.completion" and echo_request_model: + response_body = {**body, "model": json.loads(request.content)["model"]} + return httpx.Response(status, json=response_body) + + client = app.make_client(http_client=httpx.Client(transport=httpx.MockTransport(handler))) + return client, requests + + +def child_environment(directory, receiver=None, fake=None): + child_env = os.environ.copy() + packages = [fi_instrumentation, traceai_openai, harness, openai] + # Preserve SDK dependencies loaded from an offline cache as well. + packages.extend(sys.modules[name] for name in ("distro", "sniffio") if name in sys.modules) + paths = [ + RECIPE / "tests" / "loopback_guard", + RECIPE / "src", + *(Path(package.__file__).resolve().parent.parent for package in packages), + ] + child_env["PYTHONPATH"] = os.pathsep.join(dict.fromkeys(str(path) for path in paths)) + child_env["PYTHONDONTWRITEBYTECODE"] = "1" + child_env["LOOPBACK_GUARD_LOG"] = str(directory / "guard.log") + child_env["LOOPBACK_GUARD_READY"] = str(directory / "guard.ready") + if receiver: + child_env["FI_BASE_URL"] = receiver.origin + if fake: + child_env[app.BASE_URL_ENV] = fake.base_url + return child_env + + +def test_instrumentor_matches_matrix_cell(): + mode = "source" if SOURCE_INSTRUMENTOR else "published" + assert os.environ.get("PACKAGE_MODE", mode) == mode + if not SOURCE_INSTRUMENTOR: + assert version("traceAI-openai") == "0.1.10" + + +def test_documented_default_base_url(monkeypatch): + assert app.DEFAULT_BASE_URL == DOCUMENTED_BASE + assert DOCUMENTED_BASE in (RECIPE / "README.md").read_text() + monkeypatch.delenv(app.BASE_URL_ENV, raising=False) + with app.make_client(http_client=httpx.Client(transport=httpx.MockTransport(lambda _: pytest.fail("unexpected request")))) as client: + assert str(client.base_url) == DOCUMENTED_BASE + "/" + + +def test_chat_exports_usage_and_keeps_keys_separate(traced): + receiver, provider, project = traced + client, requests = mock_client(completion()) + with client: + response = client.chat.completions.create(model=MODEL, messages=[{"role": "user", "content": "Say hello."}]) + assert response.choices[0].message.content == ANSWER + assert response.model == MODEL + assert provider.force_flush() + [request] = requests + assert_vendor_request(request) + assert json.loads(request.content)["model"] == MODEL + span, attrs = assert_llm(receiver, project) + assert span["status"]["code"] == "STATUS_CODE_OK" + assert attrs["output.value"] == ANSWER + assert_usage(attrs) + + +@pytest.mark.parametrize("env_project, explicit, expected", [ + (None, None, "qwen-openai-recipe"), + ("", None, "qwen-openai-recipe"), + ("qwen-from-env", None, "qwen-from-env"), + ("qwen-from-env", "qwen-explicit", "qwen-explicit"), +], ids=["default", "empty", "env", "explicit"]) +def test_setup_tracing_project_name(monkeypatch, env_project, explicit, expected): + # FI_PROJECT_NAME overrides the recipe default; an explicit argument wins over both. + # Never fall through to register()'s generic DEFAULT_PROJECT_NAME. + if env_project is not None: + monkeypatch.setenv("FI_PROJECT_NAME", env_project) + with Receiver() as receiver: + monkeypatch.setenv("FI_BASE_URL", receiver.origin) + provider = app.setup_tracing(explicit) + try: + client, requests = mock_client(completion()) + with client: + client.chat.completions.create(model=MODEL, messages=[{"role": "user", "content": "Project control."}]) + assert provider.force_flush() + assert len(requests) == 1 + assert_llm(receiver, expected) + finally: + OpenAIInstrumentor().uninstrument() + provider.shutdown() + atexit.unregister(provider.shutdown) + + +def test_response_model_differs_from_request(traced): + """Span model is the response field, not the request id.""" + assert RESPONSE_MODEL != MODEL + receiver, provider, project = traced + client, requests = mock_client(completion(RESPONSE_MODEL), echo_request_model=False) + with client: + response = client.chat.completions.create( + model=MODEL, messages=[{"role": "user", "content": "Say hello."}] + ) + assert response.model == RESPONSE_MODEL + assert provider.force_flush() + assert json.loads(requests[0].content)["model"] == MODEL + _, attrs = assert_llm(receiver, project, model=RESPONSE_MODEL) + assert attrs["gen_ai.request.model"] != MODEL + body = completion(RESPONSE_MODEL) + assert body["object"] == "chat.completion" + assert set(body["usage"]) == {"prompt_tokens", "completion_tokens", "total_tokens"} + + +def test_missing_usage_is_omitted(traced): + receiver, provider, project = traced + client, _ = mock_client(completion(include_usage=False)) + with client: + client.chat.completions.create(model=MODEL, messages=[{"role": "user", "content": "Say hello."}]) + assert provider.force_flush() + _, attrs = assert_llm(receiver, project) + assert not any(name.startswith("gen_ai.usage.") for name in attrs) + + +@pytest.mark.parametrize("include_usage", [False, True], ids=["without-usage", "final-chunk-usage"]) +def test_stream_accumulates_text_and_optional_usage(traced, include_usage): + receiver, provider, project = traced + client, requests = mock_client(stream_body(include_usage=include_usage), streaming=True) + options = {"stream_options": {"include_usage": True}} if include_usage else {} + with client: + stream = client.chat.completions.create( + model=MODEL, messages=[{"role": "user", "content": "Say hello."}], stream=True, **options + ) + chunks = list(stream) + assert "".join(chunk.choices[0].delta.content or "" for chunk in chunks if chunk.choices) == ANSWER + if include_usage: + assert chunks[-1].choices == [] + assert chunks[-1].usage.total_tokens == USAGE["total_tokens"] + assert provider.force_flush() + [request] = requests + assert_vendor_request(request) + body = json.loads(request.content) + assert body["stream"] is True + assert body.get("stream_options") == options.get("stream_options") + span, attrs = assert_llm(receiver, project, model=None) + assert span["status"]["code"] == "STATUS_CODE_OK" + assert attrs["output.value"] == ANSWER + # The requested model is still exported inside the request parameters. + assert json.loads(attrs["gen_ai.request.parameters"])["model"] == MODEL + if include_usage: + assert_usage(attrs) + else: + assert not any(name.startswith("gen_ai.usage.") for name in attrs) + + +@pytest.mark.parametrize("base, source_provider", [ + *SHARED_PROVIDER_CASES, + *((base.replace("{WorkspaceId}", WORKSPACE), "dashscope") for base in REGION_BASES.values()), + ("https://qwen-proxy.example.invalid/custom/v1", "openai"), + ("https://dashscope-intl.aliyuncs.com.example.invalid/custom/v1", "openai"), + ("https://example.maas.aliyuncs.com.example.invalid/custom/v1", "openai"), +]) +@pytest.mark.parametrize("operation", ["chat", "stream", "error"]) +def test_regional_base_routes_bearer_auth_and_provider(traced, base, source_provider, operation): + receiver, provider, project = traced + requests = [] + + def handler(request): + requests.append(request) + if operation == "stream": + return httpx.Response(200, content=stream_body(include_usage=True), headers={"Content-Type": "text/event-stream"}) + if operation == "error": + return httpx.Response(401, json=AUTH_ERROR) + return httpx.Response(200, json=completion()) + + with app.make_client(base_url=base, http_client=httpx.Client(transport=httpx.MockTransport(handler))) as client: + client.max_retries = 0 + kwargs = {"model": MODEL, "messages": [{"role": "user", "content": "Local region fixture."}]} + if operation == "error": + with pytest.raises(openai.AuthenticationError): + client.chat.completions.create(**kwargs) + elif operation == "stream": + stream = client.chat.completions.create(**kwargs, stream=True, stream_options={"include_usage": True}) + chunks = list(stream) + assert "".join(chunk.choices[0].delta.content or "" for chunk in chunks if chunk.choices) == ANSWER + else: + assert client.chat.completions.create(**kwargs).choices[0].message.content == ANSWER + assert provider.force_flush() + [request] = requests + assert str(request.url) == base + "/chat/completions" + assert request.headers["authorization"] == f"Bearer {VENDOR_KEY}" + assert "x-api-key" not in request.headers + assert "x-secret-key" not in request.headers + span, _ = assert_llm( + receiver, project, model=MODEL if operation == "chat" else None, source_provider=source_provider, + ) + assert span["status"]["code"] == ("STATUS_CODE_ERROR" if operation == "error" else "STATUS_CODE_OK") + + +@pytest.mark.parametrize("status, code, message, error_class", [ + (400, "invalid_value", "-1 is lesser than the minimum of 0 - 'seed'", openai.BadRequestError), + (404, "model_not_found", "The model does not exist or you do not have access to it.", openai.NotFoundError), + (429, "limit_requests", "Requests rate limit exceeded, please try again later.", openai.RateLimitError), + (429, "insufficient_quota", "You exceeded your current quota, please check your plan and billing details.", openai.RateLimitError), + (500, "internal_error", "An internal error has occured, please try again later or contact service support.", openai.InternalServerError), +]) +def test_provider_error_statuses(traced, status, code, message, error_class): + receiver, provider, project = traced + body = {"error": {"message": message, "type": "invalid_request_error", "param": None, "code": code}} + client, requests = mock_client(body, status=status) + client.max_retries = 0 + with client, pytest.raises(error_class) as caught: + client.chat.completions.create(model=MODEL, messages=[{"role": "user", "content": "Local error fixture."}]) + assert caught.value.status_code == status + assert caught.value.code == code + assert provider.force_flush() + [request] = requests + assert_vendor_request(request) + span, attrs = assert_llm(receiver, project, model=None) + assert span["status"]["code"] == "STATUS_CODE_ERROR" + assert json.loads(attrs["gen_ai.request.parameters"])["model"] == MODEL + [event] = [event for event in span["events"] if event["name"] == "exception"] + assert attributes(event)["exception.type"] == f"openai.{error_class.__name__}" + assert code in attributes(event)["exception.message"] + + +def test_cross_region_401_records_authentication_error(traced): + receiver, provider, project = traced + client, requests = mock_client(AUTH_ERROR, status=401) + with client, pytest.raises(openai.AuthenticationError) as caught: + client.chat.completions.create(model=MODEL, messages=[{"role": "user", "content": "Say hello."}]) + assert type(caught.value) is openai.AuthenticationError + assert caught.value.status_code == 401 + assert caught.value.code == "invalid_api_key" + assert provider.force_flush() + [request] = requests + assert_vendor_request(request) + span, attrs = assert_llm(receiver, project, model=None) + assert json.loads(attrs["gen_ai.request.parameters"])["model"] == MODEL + assert span["status"]["code"] == "STATUS_CODE_ERROR" + # Source fi-core normalizes redundant exception-type prefixes in status text. + assert "invalid_api_key" in span["status"]["message"] + [event] = [event for event in span["events"] if event["name"] == "exception"] + assert attributes(event)["exception.type"] == "openai.AuthenticationError" + assert "invalid_api_key" in attributes(event)["exception.message"] + assert VENDOR_KEY not in json.dumps(span) + + +@pytest.mark.parametrize("hidden", [False, True], ids=["visible-control", "hidden"]) +def test_hide_inputs_removes_prompt_from_export(monkeypatch, hidden): + marker = "qwen-private-prompt-" + uuid.uuid4().hex + with trace_session(monkeypatch, FI_HIDE_INPUTS=hidden) as (receiver, provider, project): + client, requests = mock_client(completion()) + with client: + client.chat.completions.create(model=MODEL, messages=[{"role": "user", "content": marker}]) + assert marker in requests[0].content.decode() + assert provider.force_flush() + assert_llm(receiver, project) + assert (marker in json.dumps([receiver.spans(), receiver.requests()])) is (not hidden) + + +@pytest.mark.parametrize("hidden", [False, True], ids=["visible-control", "hidden"]) +def test_hide_outputs_removes_answer_from_export(monkeypatch, hidden): + with trace_session(monkeypatch, FI_HIDE_OUTPUTS=hidden) as (receiver, provider, project): + client, _ = mock_client(completion()) + with client: + response = client.chat.completions.create(model=MODEL, messages=[{"role": "user", "content": "Say hello."}]) + assert response.choices[0].message.content == ANSWER + assert provider.force_flush() + assert_llm(receiver, project) + assert (ANSWER in json.dumps([receiver.spans(), receiver.requests()])) is (not hidden) + + +@pytest.mark.parametrize("env_project", [None, "qwen-subprocess-env"], ids=["default-project", "env-project"]) +@pytest.mark.parametrize("streaming", [False, True], ids=["chat", "stream"]) +def test_app_subprocess_exports_one_span(monkeypatch, streaming, env_project): + with tempfile.TemporaryDirectory(prefix="qwen-recipe-") as temporary: + directory = Path(temporary) + with Receiver() as receiver, FakeOpenAI() as fake: + child_env = child_environment(directory, receiver, fake) + if env_project: + child_env["FI_PROJECT_NAME"] = env_project + argv = [sys.executable, str(RECIPE / "src" / "app.py"), "--prompt", "Local subprocess prompt."] + if streaming: + argv.append("--stream") + result = subprocess.run(argv, env=child_env, capture_output=True, text=True, timeout=30) + assert result.returncode == 0, result.stderr + assert result.stdout == ANSWER + "\n" + assert all(key not in result.stdout + result.stderr for key in (VENDOR_KEY, FI_KEY, FI_SECRET)) + assert (directory / "guard.ready").read_text() == "installed\n" + assert (directory / "guard.log").read_text() == "" + [request] = fake.requests() + assert request["path"] == "/compatible-mode/v1/chat/completions" + assert request["headers"]["authorization"] == f"Bearer {VENDOR_KEY}" + assert "x-api-key" not in request["headers"] + assert "x-secret-key" not in request["headers"] + assert request["body"]["model"] == MODEL + assert request["body"]["messages"] == [{"role": "user", "content": "Local subprocess prompt."}] + assert request["body"]["stream"] is streaming + if streaming: + assert request["body"]["stream_options"] == {"include_usage": True} + [span] = receiver.spans() + project = receiver.requests()[0]["resource_attributes"][0]["project_name"] + assert project == (env_project or "qwen-openai-recipe") + _, attrs = assert_llm(receiver, project, model=None if streaming else MODEL, source_provider="openai") + assert attrs["output.value"] == ANSWER + assert_usage(attrs) + + +@pytest.mark.parametrize("operation", ["getaddrinfo", "connect", "connect_ex"]) +def test_guard_refuses_provider_before_dns(operation): + host = urlsplit(DOCUMENTED_BASE).hostname + calls = { + "getaddrinfo": f"socket.getaddrinfo({host!r}, 443)", + "connect": f"socket.socket().connect(({host!r}, 443))", + "connect_ex": f"socket.socket().connect_ex(({host!r}, 443))", + } + with tempfile.TemporaryDirectory(prefix="qwen-guard-") as temporary: + directory = Path(temporary) + result = subprocess.run( + [sys.executable, "-c", "import socket; " + calls[operation]], + env=child_environment(directory), capture_output=True, text=True, timeout=10, + ) + assert result.returncode != 0 + assert "Loopback guard refused" in result.stderr + assert (directory / "guard.ready").read_text() == "installed\n" + assert (directory / "guard.log").read_text() == f"{operation}: {host}\n" + + +@pytest.mark.parametrize("url, reason", REFUSED_URLS) +def test_check_base_url_refuses_documented_mistakes(url, reason): + with pytest.raises(ValueError) as caught: + app.check_base_url(url) + assert reason in str(caught.value) + assert "\n" not in str(caught.value) + + +@pytest.mark.parametrize("url", ALLOWED_URLS) +def test_check_base_url_preserves_allowed_urls(url): + assert app.check_base_url(url) == url + + +@pytest.mark.parametrize("url, reason", [(url, reason) for url, reason in REFUSED_URLS if url and "\x00" not in url]) +def test_main_rejects_url_before_client_or_tracing(monkeypatch, capsys, url, reason): + monkeypatch.setenv(app.BASE_URL_ENV, url) + monkeypatch.delenv(app.API_KEY_ENV, raising=False) + + def forbidden(*_args, **_kwargs): + pytest.fail("Invalid URL must be rejected before tracing or client setup") + + monkeypatch.setattr(app, "setup_tracing", forbidden) + monkeypatch.setattr(app, "make_client", forbidden) + assert app.main([]) == 2 + captured = capsys.readouterr() + assert captured.out == "" + assert reason in captured.err + assert VENDOR_KEY not in captured.err + + +def test_make_client_env_override_and_explicit_arguments(monkeypatch): + monkeypatch.setenv(app.BASE_URL_ENV, SHARED_BASES[2]) + with app.make_client() as client: + assert str(client.base_url) == SHARED_BASES[2] + "/" + assert client.api_key == VENDOR_KEY + monkeypatch.setenv(app.BASE_URL_ENV, "https://dashscope.aliyuncs.com") + monkeypatch.delenv(app.API_KEY_ENV) + with app.make_client(base_url=DOCUMENTED_BASE, api_key="placeholder-explicit-qwen-key") as client: + assert str(client.base_url) == DOCUMENTED_BASE + "/" + assert client.api_key == "placeholder-explicit-qwen-key" + with pytest.raises(ValueError, match="/compatible-mode/v1"): + app.make_client() + monkeypatch.setenv(app.BASE_URL_ENV, "") + monkeypatch.setenv(app.API_KEY_ENV, VENDOR_KEY) + with app.make_client() as client: + assert str(client.base_url) == DOCUMENTED_BASE + "/" + + +def test_make_client_missing_key_names_variable(monkeypatch): + monkeypatch.delenv(app.API_KEY_ENV) + with pytest.raises(KeyError) as caught: + app.make_client() + assert caught.value.args == ("DASHSCOPE_API_KEY",) + + +def test_main_missing_model_names_variable_before_tracing(monkeypatch, capsys): + monkeypatch.delenv(app.MODEL_ENV) + monkeypatch.setattr(app, "setup_tracing", lambda *_args: pytest.fail("Missing model must fail before tracing")) + with pytest.raises(SystemExit) as caught: + app.main([]) + assert caught.value.code == 2 + assert "DASHSCOPE_MODEL" in capsys.readouterr().err + + +def test_main_missing_key_names_variable_before_tracing(monkeypatch, capsys): + monkeypatch.delenv(app.API_KEY_ENV) + monkeypatch.setattr(app, "setup_tracing", lambda *_args: pytest.fail("Missing key must fail before tracing")) + with pytest.raises(SystemExit) as caught: + app.main([]) + assert caught.value.code == 2 + assert "DASHSCOPE_API_KEY" in capsys.readouterr().err + + +def test_cli_model_overrides_environment(): + with tempfile.TemporaryDirectory(prefix="qwen-model-") as temporary: + directory = Path(temporary) + with Receiver() as receiver, FakeOpenAI() as fake: + child_env = child_environment(directory, receiver, fake) + result = subprocess.run( + [sys.executable, str(RECIPE / "src" / "app.py"), "--model", "qwen-cli-test"], + env=child_env, capture_output=True, text=True, timeout=30, + ) + assert result.returncode == 0, result.stderr + assert result.stdout == ANSWER + "\n" + [request] = fake.requests() + assert request["body"]["model"] == "qwen-cli-test" + assert request["body"]["messages"] == [{"role": "user", "content": "What is a rainbow?"}] + assert (directory / "guard.ready").exists() + assert (directory / "guard.log").read_text() == "" + assert_llm(receiver, "qwen-openai-recipe", model="qwen-cli-test", source_provider="openai") + + +def test_setup_tracing_keeps_global_provider(monkeypatch): + original = trace.get_tracer_provider() + with trace_session(monkeypatch): + assert trace.get_tracer_provider() is original + assert OpenAIInstrumentor().is_instrumented_by_opentelemetry + assert not OpenAIInstrumentor().is_instrumented_by_opentelemetry + + +def test_fake_server_error_fixture(): + with FakeOpenAI(error=True) as fake: + with app.make_client(base_url=fake.base_url) as client: + with pytest.raises(openai.AuthenticationError) as caught: + client.chat.completions.create(model=MODEL, messages=[{"role": "user", "content": "Local error."}]) + assert caught.value.status_code == 401 + assert caught.value.code == "invalid_api_key" + [request] = fake.requests() + assert request["path"] == "/compatible-mode/v1/chat/completions" + assert request["headers"]["authorization"] == f"Bearer {VENDOR_KEY}" + + +def test_readme_and_requirements_pin_contract(): + readme = (RECIPE / "README.md").read_text() + source = (RECIPE / "src" / "app.py").read_text() + headings = [ + "## Install", "## Configure", "## Run", "## Code", "## What you see in Future AGI", + "## Provider specifics", "## Privacy", "## Limits / not covered", "## Tests", + ] + positions = [readme.index(heading) for heading in headings] + assert positions == sorted(positions) + assert readme.startswith("# Qwen (OpenAI-compatible) with traceAI\n") + assert DOCUMENTED_BASE in readme + assert "provider field says `openai`" in readme + for base, expected_provider in SHARED_PROVIDER_CASES: + host = urlsplit(base).hostname + assert re.search( + rf"^\| `{re.escape(host)}` [^|]+ \| `openai` \| `{expected_provider}` \|$", + readme, re.MULTILINE, + ) + assert "| `*.maas.aliyuncs.com` (including Hong Kong workspace hosts) | `openai` | `dashscope` |" in readme + assert "The Hong Kong shared host is not in the current provider map" in readme + assert "For Virginia, prefer the workspace-dedicated endpoint from your console." in readme + for name in (app.API_KEY_ENV, app.BASE_URL_ENV, app.MODEL_ENV, "FI_API_KEY", "FI_SECRET_KEY", "FI_BASE_URL", "FI_PROJECT_NAME", "FI_HIDE_INPUTS", "FI_HIDE_OUTPUTS"): + assert name in readme + assert "| `FI_PROJECT_NAME` | No | Observe project name. Defaults to `qwen-openai-recipe`. Goes to the tracer only. |" in readme + assert 'project_name=os.environ.get("FI_PROJECT_NAME") or "qwen-openai-recipe"' in readme + for region, base in REGION_BASES.items(): + assert region in readme + assert base in readme + for phrase in ( + "Hong Kong", "https://dashscope.aliyuncs.com", "https://dashscope-intl.aliyuncs.com", + "https://cn-hongkong.dashscope.aliyuncs.com", "https://{WorkspaceId}.cn-hongkong.maas.aliyuncs.com", + "legacy international (Singapore)", "HTTP 401", "invalid_api_key", "Qwen-Audio", + "DashScope native", "Responses API", 'stream_options={"include_usage": True}', + "Without `include_usage`", "1.69.0", "3.24.0", + "at most one trailing slash", + "model id the provider returns in its response", "no `gen_ai.request.model` attribute", + "gen_ai.request.parameters", "Authorization: Bearer ", + "not a Future AGI authentication failure", "legacy hosts are still accepted", + "use the DashScope protocol", 'PYTHONPATH="python/examples/qwen/src:python/tests"', + ): + assert phrase in readme + for page in ("compatibility-of-openai-with-dashscope", "base-url", "regions", "stream", "error-code", "models", "get-api-key"): + assert f"https://www.alibabacloud.com/help/en/model-studio/{page}" in readme + run_section = readme.split("## Run\n", 1)[1].split("## Code\n", 1)[0] + assert 'export DASHSCOPE_MODEL="qwen-plus"' in run_section + assert "openai-compatible-recipe-tests.yml" in readme + assert TEST_COMMAND in readme + assert not re.search(r"sk-[A-Za-z0-9]{8,}", readme + source) + assert not re.search(r"ws-[A-Za-z0-9]+", readme + source) + requirements = (RECIPE / "requirements.txt").read_text().splitlines() + assert requirements[0].startswith("#") + assert [line for line in requirements if line and not line.startswith("#")] == [ + "openai==3.24.0", "traceAI-openai==0.1.10", "fi-instrumentation-otel==1.1.0" + ]