diff --git a/python/examples/azure-ai-foundry/README.md b/python/examples/azure-ai-foundry/README.md new file mode 100644 index 000000000..852b80ba9 --- /dev/null +++ b/python/examples/azure-ai-foundry/README.md @@ -0,0 +1,246 @@ +# Azure AI Foundry (OpenAI-compatible) with traceAI + +This recipe traces Chat Completions through the official `openai` Python SDK at your Azure resource's `/openai/v1/` endpoint; model is your deployment name, not a model id. +It uses `traceai-openai`, with no separate provider package. + +## Install + +Requires Python 3.10 or later. + +From this example directory, install the pinned dependencies: + +```bash +pip install -r requirements.txt +``` + +The plain package form is: + +```bash +pip install traceAI-openai fi-instrumentation-otel openai +``` + +## Configure + +| Variable | Required | Purpose | +|---|---|---| +| `FI_API_KEY` | Yes | Future AGI API key; tracer only. | +| `FI_SECRET_KEY` | Yes | Future AGI secret key; tracer only. | +| `AZURE_OPENAI_API_KEY` | Yes | Azure resource API key; OpenAI client only. | +| `AZURE_OPENAI_DEPLOYMENT` | Yes, unless `--model` is used | Your deployment name, not an underlying model id. | +| `AZURE_OPENAI_BASE_URL` | Yes | Your resource-specific inference base URL. There is no public default. | +| `FI_BASE_URL` | No | Future AGI collector origin; defaults to `https://api.futureagi.com`. | +| `FI_PROJECT_NAME` | No | Observe project name; defaults to `azure-ai-foundry`; tracer only. | +| `FI_HIDE_INPUTS` | No | Set to `true` to hide prompt text in exported spans. | +| `FI_HIDE_OUTPUTS` | No | Set to `true` to hide response text in exported spans. | + +The client receives the Azure key. The tracer receives the Future AGI keys. +The example uses the Future AGI project name `azure-ai-foundry` unless `FI_PROJECT_NAME` is set. + +## Run + +Replace the placeholders before running. A template URL that still contains +`YOUR-RESOURCE-NAME` or angle brackets fails before tracing or a network call. + +```bash +export FI_API_KEY="placeholder-futureagi-key" +export FI_SECRET_KEY="placeholder-futureagi-secret" +export AZURE_OPENAI_API_KEY="placeholder-azure-key" +export AZURE_OPENAI_BASE_URL="https://YOUR-RESOURCE-NAME.openai.azure.com/openai/v1/" +export AZURE_OPENAI_DEPLOYMENT="your-deployment-name" +python src/app.py +python src/app.py --stream +``` + +Use `--prompt "What is a rainbow?"` to change the question or +`--model your-deployment-name` to override the deployment environment variable. +Set `FI_PROJECT_NAME` to send the traces to another Observe project; unset or +empty, they go to `azure-ai-foundry`. +Missing Azure configuration and refused URLs return exit code 2 before tracing. + +## Code + +Instrument before creating the client. [src/app.py](src/app.py) contains the CLI +and the URL validation used here. + +```python +import os +from openai import OpenAI +from fi_instrumentation import register +from fi_instrumentation.fi_types import ProjectType +from traceai_openai import OpenAIInstrumentor +from app import check_base_url +base_url = check_base_url(os.environ["AZURE_OPENAI_BASE_URL"]) +provider = register(project_name=os.environ.get("FI_PROJECT_NAME") or "azure-ai-foundry", # as in src/app.py + project_type=ProjectType.OBSERVE, set_global_tracer_provider=False, verbose=False) +OpenAIInstrumentor().instrument(tracer_provider=provider) +client = OpenAI(api_key=os.environ["AZURE_OPENAI_API_KEY"], base_url=base_url) +response = client.chat.completions.create(model=os.environ["AZURE_OPENAI_DEPLOYMENT"], + messages=[{"role": "user", "content": "What is a rainbow?"}]) +print(response.choices[0].message.content) +provider.force_flush() +``` + +## What you see in Future AGI + +The fixture tests produce one LLM span per call: `ChatCompletion` for Chat +Completions and `Response` for the Responses API. Up to `traceai-openai` 0.1.10, the provider field is `openai` +(`gen_ai.provider.name`). This is the shared instrumentor's label for both Azure +host forms; the span does not say `azure`. Later versions (future-agi/traceAI#249) +say `azure` for both host forms; proxy and loopback base URLs keep `openai`. + +For a successful non-streamed response, `gen_ai.request.model` is the model id +the provider returns in its response. The fixtures return the deployment name +that was requested. Azure's `model` request parameter is your deployment name. +Azure may return the underlying model name rather than your deployment name; +`gen_ai.request.model` then shows the returned name, and your deployment name is +in `gen_ai.request.parameters`. Cost lookup keyed on one name may miss the other. + +Token usage is recorded only when the response has `usage`. Missing usage +attributes are omitted, never replaced with 0. The Chat Completions stream is +accumulated by the same instrumentor. Its default test stream has no usage +attributes. With `stream_options={"include_usage": True}` and a final usage +chunk, the instrumentor records input, output and total token counts. + +Up to `traceai-openai` 0.1.10, streamed and failed Chat Completions calls omit `gen_ai.request.model`; later versions record the requested deployment there. +On the Responses API, a failed call records the requested deployment, and a successful Responses call records the model the response returns. +The requested deployment remains in the +`gen_ai.request.parameters` JSON unless `FI_HIDE_LLM_INVOCATION_PARAMETERS=true`. +Errors set the span status to ERROR and record an exception event. + +Up to 0.1.10, the non-streamed Responses fixture records the full parsed response as Python +dictionary text in `output.value`, including its output text; later versions write it as +JSON (`output.mime_type=application/json`) led by `output_text`. It also records +the model the response returns (the fixture returns a different model from the +deployment requested) and usage, including the supplied reasoning and cache +token counts. Responses streaming is not tested here. + +## Live vendor verification (2026-10-19) + +Run against a real Azure OpenAI resource (East US, pay-as-you-go subscription, `gpt-4.1-nano` deployment named `th8103-nano`, Global Standard), using the `.openai.azure.com/openai/v1/` form, `openai` 3.24.0, `traceAI-openai` 0.1.10 and `fi-instrumentation-otel` 1.1.0, exactly as in "Run" above, exported to a self-hosted Future AGI stack: + +| Call | What Azure and the span showed | +|---|---| +| Chat | Succeeded. The `openai` SDK's `Authorization: Bearer ` is accepted by the v1 endpoint. One `llm` span, `gen_ai.request.model` = `gpt-4.1-nano-2025-04-14` (the underlying model Azure returns, **not** the deployment name `th8103-nano`), 12 input and 119 output tokens, provider `openai`. | +| Stream | Succeeded. One `llm` span with output text, no model and no token counts (no `stream_options`), provider `openai`. | +| Error (unknown deployment) | Azure answered `404 DeploymentNotFound`. One `llm` span with status ERROR and the `NotFoundError` text, no model. | + +This confirms the model-name, streaming and error statements in "What you see in Future AGI". Not exercised live: the `services.ai.azure.com` form, the Responses API, Entra ID auth, and `traceai-openai` after future-agi/traceAI#249. + +## Provider specifics + +Microsoft documents both of these resource-specific base URL forms: + +```text +https://YOUR-RESOURCE-NAME.openai.azure.com/openai/v1/ +https://YOUR-RESOURCE-NAME.services.ai.azure.com/openai/v1/ +``` + +Both end in `/openai/v1/`. The SDK joins `chat/completions` or `responses` to +that path. If the slash is omitted after `/openai/v1`, the SDK appends it. +The example's validator returns allowed URLs exactly as supplied; it does not +rewrite them. Azure hosts must use `https://` and have the path `/openai/v1` or +`/openai/v1/`, with no query string (the v1 API needs no `api-version`), no +fragment and no percent-encoded path. Credentials in the URL are refused. +Proxy hosts and loopback URLs are also allowed. + +`model` means your model deployment name, for example `my-gpt-deployment`, +not a model id. Read the API key from `AZURE_OPENAI_API_KEY`. +Microsoft recommends the Responses API for Azure OpenAI models. Chat +Completions remains available for models that support it on the same base URL: + +```python +response = client.responses.create( + model=os.environ["AZURE_OPENAI_DEPLOYMENT"], input="What is a rainbow?" +) +print(response.output_text) +provider.force_flush() +``` + +For Microsoft Entra ID, use Microsoft's token-provider sample with +`azure.identity` and the same `base_url` on the +[v1 API page](https://learn.microsoft.com/en-us/azure/foundry/openai/api-version-lifecycle). +This recipe does not implement token acquisition. Tracing uses the same +instrumentation because it is the same OpenAI client. + +The tests use these syntactic test hosts, not real Azure resources: + +```text +https://test-resource-0000.openai.azure.com/openai/v1/ +https://test-resource-0000.services.ai.azure.com/openai/v1/ +``` + +MockTransport handles those URLs in-process without DNS or an Azure connection. + +### Not this page: Foundry Agent Service + +Foundry agents are not this page. Agent Service, agent threads and evaluations +are separate surfaces. A project endpoint such as +`https://YOUR-RESOURCE-NAME.services.ai.azure.com/api/projects/YOUR-PROJECT-NAME` +is not a model inference base URL. + +`AzureAIOpenTelemetryTracer` exports to Application Insights. That setup does +not send agent spans to Future AGI through this recipe. This example does not +retarget that tracer or implement agent tracing. + +## Privacy + +Set `FI_HIDE_INPUTS=true` and/or `FI_HIDE_OUTPUTS=true` to keep prompt or response +text out of exported spans. The tests include visible control runs and verify +both flags. You can also pass `TraceConfig(hide_inputs=True)` as the instrumentor's +`config`. `FI_HIDE_LLM_INVOCATION_PARAMETERS=true` omits request parameters. + +On non-streamed Responses calls, `output.value` holds the whole response object, +which can echo request fields such as `instructions`. `FI_HIDE_INPUTS` does not cover +that field; set `FI_HIDE_OUTPUTS=true` as well if those must stay out of the export. + +Azure still receives the prompt. Future AGI masking does not change Azure's +own logging or data handling. + +## Limits / not covered + +- The `AzureOpenAI` client with `api-version`, including older deployment-scoped URLs. +- The `azure-ai-inference` SDK and its `/models` endpoint. +- Foundry agents, threads, evaluations and Application Insights export. +- A bare Azure resource root or the project endpoint as an inference base URL. +- Responses streaming, async calls and other SDK surfaces in this example's tests. +- Live Azure calls: chat, stream and one error were run live on 2026-10-19 (see "Live vendor verification"); the automated tests still use a local fake of the OpenAI API. + +The recipe does not change the shared instrumentor's `openai` provider label. + +## Tests + +Run from the repository root. The suite uses MockTransport and local servers +on `127.0.0.1`. Child processes load a loopback guard that rejects other hosts +before DNS. Its negative control proves the guard is active. + +Repository source: + +```bash +env -u PYTHONPATH PYTHONDONTWRITEBYTECODE=1 \ + PYTHONPATH="python/examples/azure-ai-foundry/src:python:python/frameworks/openai:python/tests" \ + uv run --no-project --python 3.11 \ + --with 'openai==3.24.0' --with httpx --with 'wrapt<2' \ + --with opentelemetry-api --with opentelemetry-sdk --with opentelemetry-instrumentation \ + --with opentelemetry-exporter-otlp-proto-http --with requests --with protobuf \ + --with opentelemetry-proto --with pydantic --with jsonschema --with pytest \ + pytest python/examples/azure-ai-foundry/tests -q -p no:cacheprovider --noconftest -o addopts= -rfEs +``` + +Published packages: + +```bash +env -u PYTHONPATH PYTHONDONTWRITEBYTECODE=1 \ + PYTHONPATH="python/examples/azure-ai-foundry/src:python/tests" \ + uv run --no-project --python 3.11 \ + --with 'openai==3.24.0' --with 'traceAI-openai==0.1.10' --with 'fi-instrumentation-otel==1.1.0' \ + --with httpx --with 'wrapt<2' --with pytest \ + pytest python/examples/azure-ai-foundry/tests -q -p no:cacheprovider --noconftest -o addopts= -rfEs +``` + +For the floor row, change `openai==3.24.0` to `openai==1.69.0` in the repository-source command. + +| Python | `openai` | traceAI packages | Result | +|---|---|---|---| +| 3.10, 3.11, 3.12, 3.13 | 3.24.0 | `traceai-openai` and `fi_instrumentation` from this repository | Full suite | +| 3.11 | 1.69.0 (the `traceai-openai` floor) | from this repository | Full suite | +| 3.11 | 3.24.0 | published `traceAI-openai==0.1.10` and `fi-instrumentation-otel==1.1.0` from PyPI | Full suite | diff --git a/python/examples/azure-ai-foundry/requirements.txt b/python/examples/azure-ai-foundry/requirements.txt new file mode 100644 index 000000000..2e2d809c3 --- /dev/null +++ b/python/examples/azure-ai-foundry/requirements.txt @@ -0,0 +1,4 @@ +# Pinned recipe dependencies; see README for the compatibility test commands. +openai==3.24.0 +traceAI-openai==0.1.10 +fi-instrumentation-otel==1.1.0 diff --git a/python/examples/azure-ai-foundry/src/app.py b/python/examples/azure-ai-foundry/src/app.py new file mode 100644 index 000000000..dd758ed49 --- /dev/null +++ b/python/examples/azure-ai-foundry/src/app.py @@ -0,0 +1,122 @@ +"""Trace Azure OpenAI v1 Chat Completions with the OpenAI client.""" + +from __future__ import annotations + +import argparse +import os +import sys +from urllib.parse import unquote, urlsplit + +import openai +from fi_instrumentation import register +from fi_instrumentation.fi_types import ProjectType +from traceai_openai import OpenAIInstrumentor + +DEFAULT_BASE_URL = None +DOCUMENTED_BASE_URL_FORMS = ( + "https://YOUR-RESOURCE-NAME.openai.azure.com/openai/v1/", + "https://YOUR-RESOURCE-NAME.services.ai.azure.com/openai/v1/", +) +API_KEY_ENV = "AZURE_OPENAI_API_KEY" +BASE_URL_ENV = "AZURE_OPENAI_BASE_URL" +MODEL_ENV = "AZURE_OPENAI_DEPLOYMENT" + + +def check_base_url(url: str) -> str: + if not url: + raise ValueError(f"Set {BASE_URL_ENV} to your resource's /openai/v1/ base URL.") + decoded = url + while (next_value := unquote(decoded)) != decoded: + decoded = next_value + if "your-resource-name" in decoded.lower() or "<" in decoded or ">" in decoded: + raise ValueError(f"Replace placeholders in {BASE_URL_ENV} with your resource name.") + try: + # Structural checks use the URL as given; only the placeholder check decodes. + parsed = urlsplit(url) + host = (parsed.hostname or "").lower().rstrip(".") + except ValueError: + raise ValueError(f"Set {BASE_URL_ENV} to a valid HTTP(S) base URL.") from None + if parsed.scheme not in ("http", "https") or not host: + raise ValueError(f"Set {BASE_URL_ENV} to a valid HTTP(S) base URL.") + if parsed.username is not None or parsed.password is not None: + raise ValueError(f"Do not put credentials in {BASE_URL_ENV}.") + if host.endswith((".openai.azure.com", ".services.ai.azure.com", ".cognitiveservices.azure.com")): + if parsed.scheme != "https": + raise ValueError(f"Azure hosts must use https; set {BASE_URL_ENV} to your resource's /openai/v1/ URL.") + if "%" in parsed.path: + raise ValueError(f"Percent-encoded paths are not accepted for Azure hosts in {BASE_URL_ENV}.") + if parsed.path not in ("/openai/v1", "/openai/v1/"): + if parsed.path.startswith("/api/projects"): + reason = "The project endpoint is for agents, not model inference." + elif parsed.path.startswith("/openai"): + reason = "Older AzureOpenAI deployment/api-version URLs are not covered." + elif parsed.path.startswith("/models"): + reason = "The azure-ai-inference SDK /models endpoint is not covered." + else: + reason = "A bare resource root or other Azure path is not an inference base URL." + raise ValueError(f"{reason} Set {BASE_URL_ENV} to your resource's /openai/v1/ URL.") + if parsed.query or parsed.fragment: + raise ValueError(f"Remove the query string or fragment from {BASE_URL_ENV}; the v1 API needs no api-version.") + return url + + +def make_client(base_url: str | None = None, api_key: str | None = None, http_client=None) -> openai.OpenAI: + resolved_url = check_base_url(base_url if base_url is not None else os.environ.get(BASE_URL_ENV) or DEFAULT_BASE_URL) + key = api_key if api_key is not None else os.environ[API_KEY_ENV] + return openai.OpenAI(base_url=resolved_url, api_key=key, http_client=http_client) + + +def setup_tracing(project_name: str | None = None): + provider = register( + project_name=project_name or os.environ.get("FI_PROJECT_NAME") or "azure-ai-foundry", + project_type=ProjectType.OBSERVE, + set_global_tracer_provider=False, + verbose=False, + ) + OpenAIInstrumentor().instrument(tracer_provider=provider) + return provider + + +def main(argv=None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--stream", action="store_true") + parser.add_argument("--prompt", default="What is a rainbow?") + parser.add_argument("--model", help="Your Azure model deployment name") + args = parser.parse_args(argv) + try: + base_url = check_base_url(os.environ.get(BASE_URL_ENV) or DEFAULT_BASE_URL) + model = args.model or os.environ[MODEL_ENV] + api_key = os.environ[API_KEY_ENV] + if not model: + raise ValueError(f"Set {MODEL_ENV} to your deployment name or use --model.") + if not api_key: + raise ValueError(f"Set {API_KEY_ENV} to your Azure API key.") + except KeyError as error: + print(f"Set {error.args[0]} before running this example.", file=sys.stderr) + return 2 + except ValueError as error: + print(str(error), file=sys.stderr) + return 2 + + provider = setup_tracing() + try: + with make_client(base_url=base_url, api_key=api_key) as client: + response = client.chat.completions.create( + model=model, + messages=[{"role": "user", "content": args.prompt}], + stream=args.stream, + ) + if args.stream: + for chunk in response: + if chunk.choices: + print(chunk.choices[0].delta.content or "", end="", flush=True) + print() + else: + print(response.choices[0].message.content or "") + finally: + provider.force_flush() + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/python/examples/azure-ai-foundry/tests/_fake_openai.py b/python/examples/azure-ai-foundry/tests/_fake_openai.py new file mode 100644 index 000000000..0c1058204 --- /dev/null +++ b/python/examples/azure-ai-foundry/tests/_fake_openai.py @@ -0,0 +1,114 @@ +"""OpenAI-shaped fixtures and a local server; no upstream requests.""" + +import copy +import json +import threading +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +ANSWER = "A rainbow is light split into colors." +CHAT_USAGE = {"prompt_tokens": 7, "completion_tokens": 9, "total_tokens": 16} +ERROR = {"error": {"code": "401", "message": "Access denied due to invalid subscription key or wrong API endpoint."}} + + +def chat_response(model, usage=True): + response = { + "id": "chatcmpl-fixture", "object": "chat.completion", "created": 1, + "model": model, + "choices": [{"index": 0, "message": {"role": "assistant", "content": ANSWER}, "finish_reason": "stop"}], + } + if usage: + response["usage"] = dict(CHAT_USAGE) + return response + + +def chat_stream(model, include_usage=False): + chunks = [] + for index, text in enumerate(("A rainbow is ", "light split into colors.")): + chunks.append({ + "id": "chatcmpl-fixture", "object": "chat.completion.chunk", "created": 1, + "model": model, + "choices": [{"index": 0, "delta": {"content": text, **({"role": "assistant"} if index == 0 else {})}, "finish_reason": None}], + }) + chunks.append({ + "id": "chatcmpl-fixture", "object": "chat.completion.chunk", "created": 1, + "model": model, "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}], + }) + if include_usage: + chunks.append({ + "id": "chatcmpl-fixture", "object": "chat.completion.chunk", "created": 1, + "model": model, "choices": [], "usage": dict(CHAT_USAGE), + }) + return ("".join("data: " + json.dumps(chunk) + "\n\n" for chunk in chunks) + "data: [DONE]\n\n").encode() + + +def responses_response(model): + return { + "id": "resp-fixture", "object": "response", "created_at": 1, + "status": "completed", "error": None, "incomplete_details": None, + "instructions": None, "model": model, + "output": [{ + "id": "msg-fixture", "type": "message", "role": "assistant", "status": "completed", + "content": [{"type": "output_text", "text": ANSWER, "annotations": []}], + }], + "parallel_tool_calls": True, "tools": [], "tool_choice": "auto", + "temperature": 1.0, "top_p": 1.0, + "usage": { + "input_tokens": 7, "output_tokens": 9, "total_tokens": 16, + "input_tokens_details": {"cached_tokens": 0}, + "output_tokens_details": {"reasoning_tokens": 0}, + }, + } + + +class FakeOpenAI: + def __init__(self): + self._requests = [] + self._lock = threading.Lock() + owner = self + + class Handler(BaseHTTPRequestHandler): + def do_POST(self): + body = json.loads(self.rfile.read(int(self.headers["Content-Length"]))) + with owner._lock: + owner._requests.append({ + "path": self.path, + "headers": {key.lower(): value for key, value in self.headers.items()}, + "body": body, + }) + if self.path not in ("/openai/v1/chat/completions", "/openai/v1/responses"): + self.send_error(404) + return + if body.get("model") == "error-deployment": + status, content_type, payload = 401, "application/json", json.dumps(ERROR).encode() + elif self.path.endswith("/responses"): + status, content_type, payload = 200, "application/json", json.dumps(responses_response(body["model"])).encode() + elif body.get("stream"): + include_usage = body.get("stream_options", {}).get("include_usage", False) + status, content_type, payload = 200, "text/event-stream", chat_stream(body["model"], include_usage) + else: + status, content_type, payload = 200, "application/json", json.dumps(chat_response(body["model"])).encode() + self.send_response(status) + self.send_header("Content-Type", content_type) + self.send_header("Content-Length", str(len(payload))) + self.end_headers() + self.wfile.write(payload) + + def log_message(self, _format, *args): + return + + self._server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + self.base_url = f"http://127.0.0.1:{self._server.server_port}/openai/v1/" + self._thread = threading.Thread(target=self._server.serve_forever, daemon=True) + + def __enter__(self): + self._thread.start() + return self + + def __exit__(self, *_args): + self._server.shutdown() + self._server.server_close() + self._thread.join() + + def requests(self): + with self._lock: + return copy.deepcopy(self._requests) diff --git a/python/examples/azure-ai-foundry/tests/loopback_guard/sitecustomize.py b/python/examples/azure-ai-foundry/tests/loopback_guard/sitecustomize.py new file mode 100644 index 000000000..4de633cb7 --- /dev/null +++ b/python/examples/azure-ai-foundry/tests/loopback_guard/sitecustomize.py @@ -0,0 +1,47 @@ +"""Refuse non-loopback networking before DNS; loaded by child Python processes.""" + +import os +import socket +from pathlib import Path + + +def require_loopback(host): + if isinstance(host, bytes): + host = host.decode("ascii", errors="replace") + if host not in ("127.0.0.1", "localhost"): + message = f"Loopback guard refused host: {host!r}" + log = os.environ.get("LOOPBACK_GUARD_LOG") + if log: + with open(log, "a", encoding="utf-8") as output: + output.write(message + "\n") + raise RuntimeError(message) + + +_getaddrinfo = socket.getaddrinfo +_connect = socket.socket.connect +_connect_ex = socket.socket.connect_ex + + +def guarded_getaddrinfo(host, *args, **kwargs): + require_loopback(host) + return _getaddrinfo(host, *args, **kwargs) + + +def guarded_connect(sock, address): + require_loopback(address[0]) + return _connect(sock, address) + + +def guarded_connect_ex(sock, address): + require_loopback(address[0]) + return _connect_ex(sock, address) + + +socket.getaddrinfo = guarded_getaddrinfo +socket.socket.connect = guarded_connect +socket.socket.connect_ex = guarded_connect_ex + +if ready := os.environ.get("LOOPBACK_GUARD_READY"): + Path(ready).write_text("loopback guard installed\n", encoding="utf-8") +if log := os.environ.get("LOOPBACK_GUARD_LOG"): + Path(log).touch() diff --git a/python/examples/azure-ai-foundry/tests/test_azure_ai_foundry_recipe.py b/python/examples/azure-ai-foundry/tests/test_azure_ai_foundry_recipe.py new file mode 100644 index 000000000..91440a60f --- /dev/null +++ b/python/examples/azure-ai-foundry/tests/test_azure_ai_foundry_recipe.py @@ -0,0 +1,606 @@ +"""Fixture-only Azure v1 recipe contract; syntactic test hosts are not real hosts.""" + +import json +import os +import re +import socket +import subprocess +import sys +import uuid +from contextlib import contextmanager +from pathlib import Path + +import app +import fi_instrumentation +import httpx +import openai +import pytest +import traceai_openai +from harness import Receiver +from traceai_openai import OpenAIInstrumentor + +from _fake_openai import ( + ANSWER, ERROR, FakeOpenAI, chat_response, chat_stream, responses_response, +) + +# traceai-openai 0.1.10 and earlier label every span "openai"; later versions +# (future-agi/traceAI#249) use the vendor's name for its documented API host. +# They also omit gen_ai.request.model on streamed and failed calls, where later +# versions record the requested model. Accept both so this recipe passes on either version. +PROVIDER_NAMES = ("openai", "azure") + +RECIPE = Path(__file__).resolve().parents[1] +# Syntactic test hosts, not real Azure resources; MockTransport never resolves them. +TEST_BASE_URLS = ( + "https://test-resource-0000.openai.azure.com/openai/v1/", + "https://test-resource-0000.services.ai.azure.com/openai/v1/", +) +MODEL = "my-gpt-deployment" +VENDOR_KEY = "placeholder-azure-key" +FI_KEY = "placeholder-futureagi-key" +FI_SECRET = "placeholder-futureagi-secret" +USAGE = { + "gen_ai.usage.input_tokens": 7, + "gen_ai.usage.output_tokens": 9, + "gen_ai.usage.total_tokens": 16, +} +SOURCE_COMMAND = """env -u PYTHONPATH PYTHONDONTWRITEBYTECODE=1 \\ + PYTHONPATH="python/examples/azure-ai-foundry/src:python:python/frameworks/openai:python/tests" \\ + uv run --no-project --python 3.11 \\ + --with 'openai==3.24.0' --with httpx --with 'wrapt<2' \\ + --with opentelemetry-api --with opentelemetry-sdk --with opentelemetry-instrumentation \\ + --with opentelemetry-exporter-otlp-proto-http --with requests --with protobuf \\ + --with opentelemetry-proto --with pydantic --with jsonschema --with pytest \\ + pytest python/examples/azure-ai-foundry/tests -q -p no:cacheprovider --noconftest -o addopts= -rfEs""" +PUBLISHED_COMMAND = """env -u PYTHONPATH PYTHONDONTWRITEBYTECODE=1 \\ + PYTHONPATH="python/examples/azure-ai-foundry/src:python/tests" \\ + uv run --no-project --python 3.11 \\ + --with 'openai==3.24.0' --with 'traceAI-openai==0.1.10' --with 'fi-instrumentation-otel==1.1.0' \\ + --with httpx --with 'wrapt<2' --with pytest \\ + pytest python/examples/azure-ai-foundry/tests -q -p no:cacheprovider --noconftest -o addopts= -rfEs""" + + +@pytest.fixture(autouse=True) +def isolate_environment_and_network(monkeypatch): + # No inherited credentials, hiding flags, collector settings or proxy routes. + for name in tuple(os.environ): + if name.startswith(("FI_", "OTEL_", "OPENAI_", "AZURE_")) or name.lower().endswith("_proxy"): + monkeypatch.delenv(name) + monkeypatch.setenv("FI_API_KEY", FI_KEY) + monkeypatch.setenv("FI_SECRET_KEY", FI_SECRET) + monkeypatch.setenv(app.API_KEY_ENV, VENDOR_KEY) + monkeypatch.setenv(app.BASE_URL_ENV, TEST_BASE_URLS[0]) + monkeypatch.setenv(app.MODEL_ENV, MODEL) + monkeypatch.setenv("NO_PROXY", "*") + original_dns = socket.getaddrinfo + original_connect = socket.socket.connect + original_connect_ex = socket.socket.connect_ex + + def require_loopback(host): + if host not in ("127.0.0.1", "localhost", b"127.0.0.1", b"localhost"): + raise RuntimeError("Tests refuse non-loopback networking before DNS") + + def guarded_dns(host, *args, **kwargs): + require_loopback(host) + return original_dns(host, *args, **kwargs) + + def guarded_connect(sock, address): + require_loopback(address[0]) + return original_connect(sock, address) + + def guarded_connect_ex(sock, address): + require_loopback(address[0]) + return original_connect_ex(sock, address) + + monkeypatch.setattr(socket, "getaddrinfo", guarded_dns) + monkeypatch.setattr(socket.socket, "connect", guarded_connect) + monkeypatch.setattr(socket.socket, "connect_ex", guarded_connect_ex) + + +@pytest.fixture +def receiver(monkeypatch): + with Receiver() as receiver: + monkeypatch.setenv("FI_BASE_URL", receiver.origin) + yield receiver + + +@pytest.fixture +def tracing(): + @contextmanager + def start(): + project = "azure-recipe-" + uuid.uuid4().hex + provider = app.setup_tracing(project) + try: + yield provider, project + finally: + provider.force_flush() + OpenAIInstrumentor().uninstrument() + provider.shutdown() + return start + + +def attributes(span): + return {item["key"]: next(iter(item["value"].values())) for item in span["attributes"]} + + +def assert_usage(attrs, expected): + actual = {key: int(value) for key, value in attrs.items() if key.startswith("gen_ai.usage.")} + assert actual == expected + + +def assert_export(receiver, project, name="ChatCompletion"): + spans, exports = receiver.spans(), receiver.requests() + assert len(spans) == 1 + assert len(exports) == 1 + span = spans[0] + attrs = attributes(span) + assert span["name"] == name + assert attrs["gen_ai.span.kind"] == "LLM" + assert attrs["gen_ai.provider.name"] in PROVIDER_NAMES + export = exports[0] + assert export["path"] == "/tracer/v1/traces" + assert "x-api-key" in export["headers"] and "x-secret-key" in export["headers"] + assert export["headers"]["x-api-key"] == FI_KEY + assert export["headers"]["x-secret-key"] == FI_SECRET + assert export["resource_attributes"][0]["project_name"] == project + assert export["resource_attributes"][0]["project_type"] == "observe" + # The populated span/export and bearer assertion at each call site are controls. + assert VENDOR_KEY not in json.dumps({"spans": spans, "exports": exports}) + return span, attrs + + +def assert_vendor_request(request, base_url, suffix, model=MODEL): + assert str(request.url) == base_url + suffix + assert request.headers["authorization"] == "Bearer " + VENDOR_KEY + assert "x-api-key" not in request.headers and "x-secret-key" not in request.headers + assert FI_KEY not in str(request.headers) and FI_SECRET not in str(request.headers) + assert json.loads(request.content)["model"] == model + + +def mock_client(base_url, payload, requests, status=200): + def handler(request): + requests.append(request) + if isinstance(payload, bytes): + return httpx.Response(status, content=payload, headers={"Content-Type": "text/event-stream"}) + return httpx.Response(status, json=payload) + return app.make_client(base_url=base_url, http_client=httpx.Client(transport=httpx.MockTransport(handler))) + + +def complete(client, prompt="What is a rainbow?", **kwargs): + return client.chat.completions.create(model=MODEL, messages=[{"role": "user", "content": prompt}], **kwargs) + + +def test_documented_forms_require_resource_and_base_url(monkeypatch): + assert app.DEFAULT_BASE_URL is None + assert app.DOCUMENTED_BASE_URL_FORMS == ( + "https://YOUR-RESOURCE-NAME.openai.azure.com/openai/v1/", + "https://YOUR-RESOURCE-NAME.services.ai.azure.com/openai/v1/", + ) + readme = (RECIPE / "README.md").read_text() + for form in app.DOCUMENTED_BASE_URL_FORMS: + assert form in readme + with pytest.raises(ValueError, match="placeholders"): + app.check_base_url(form) + monkeypatch.delenv(app.BASE_URL_ENV) + with pytest.raises(ValueError, match=app.BASE_URL_ENV): + app.make_client() + + +def test_sdk_appends_trailing_slash_without_validator_rewriting(): + supplied = TEST_BASE_URLS[0].rstrip("/") + assert app.check_base_url(supplied) == supplied + with app.make_client(base_url=supplied) as client: + assert str(client.base_url) == TEST_BASE_URLS[0] + + +@pytest.mark.parametrize("base_url", TEST_BASE_URLS) +def test_chat_both_host_forms_and_key_separation(base_url, receiver, tracing, monkeypatch): + monkeypatch.setenv(app.BASE_URL_ENV, base_url) + requests = [] + with tracing() as (provider, project): + with mock_client(base_url, chat_response(MODEL), requests) as client: + assert str(client.base_url) == base_url + assert complete(client).choices[0].message.content == ANSWER + # The environment default is identical to the explicit client base URL. + with app.make_client() as default_client: + assert str(default_client.base_url) == base_url + provider.force_flush() + span, attrs = assert_export(receiver, project) + assert len(requests) == 1 + assert_vendor_request(requests[0], base_url, "chat/completions") + assert attrs["gen_ai.request.model"] == MODEL + assert ANSWER in attrs["output.value"] + assert span["status"]["code"] == "STATUS_CODE_OK" + assert_usage(attrs, USAGE) + + +def test_chat_response_model_differs_from_deployment(receiver, tracing): + """Chat span stores the response model, not the deployment name.""" + assert RETURNED_MODEL != MODEL + requests = [] + with tracing() as (provider, project): + with mock_client(TEST_BASE_URLS[0], chat_response(RETURNED_MODEL), requests) as client: + response = complete(client) + provider.force_flush() + _, attrs = assert_export(receiver, project) + assert response.model == RETURNED_MODEL + assert_vendor_request(requests[0], TEST_BASE_URLS[0], "chat/completions", model=MODEL) + assert attrs["gen_ai.request.model"] == RETURNED_MODEL + body = chat_response(RETURNED_MODEL) + assert body["object"] == "chat.completion" + assert body["model"] == RETURNED_MODEL + assert set(body["usage"]) >= {"prompt_tokens", "completion_tokens", "total_tokens"} + + +@pytest.mark.parametrize("env_project, explicit, expected", [ + (None, None, "azure-ai-foundry"), + ("", None, "azure-ai-foundry"), + ("azure-from-env", None, "azure-from-env"), + ("azure-from-env", "azure-explicit", "azure-explicit"), +], ids=["default", "empty", "env", "explicit"]) +def test_setup_tracing_project_name(env_project, explicit, expected, receiver, monkeypatch): + # FI_PROJECT_NAME overrides the recipe default; an explicit argument wins over both. + # Never fall through to register()'s generic DEFAULT_PROJECT_NAME. + if env_project is not None: + monkeypatch.setenv("FI_PROJECT_NAME", env_project) + requests = [] + provider = app.setup_tracing(explicit) + try: + with mock_client(TEST_BASE_URLS[0], chat_response(MODEL), requests) as client: + complete(client) + provider.force_flush() + assert len(requests) == 1 + assert_export(receiver, expected) + finally: + provider.force_flush() + OpenAIInstrumentor().uninstrument() + provider.shutdown() + + +def test_usage_absent_is_omitted(receiver, tracing): + requests = [] + with tracing() as (provider, project): + with mock_client(TEST_BASE_URLS[0], chat_response(MODEL, usage=False), requests) as client: + assert complete(client).choices[0].message.content == ANSWER + provider.force_flush() + _, attrs = assert_export(receiver, project) + assert_vendor_request(requests[0], TEST_BASE_URLS[0], "chat/completions") + assert attrs["gen_ai.request.model"] == MODEL + assert_usage(attrs, {}) + # The same fixture's normal usage is verified in test_chat_both_host_forms_and_key_separation. + + +@pytest.mark.parametrize("include_usage", [False, True], ids=["default", "include-usage"]) +def test_stream_output_model_gap_and_usage(include_usage, receiver, tracing): + requests = [] + with tracing() as (provider, project): + with mock_client(TEST_BASE_URLS[0], chat_stream(MODEL, include_usage), requests) as client: + options = {"stream_options": {"include_usage": True}} if include_usage else {} + text = "".join(chunk.choices[0].delta.content or "" for chunk in complete(client, stream=True, **options) if chunk.choices) + assert text == ANSWER + provider.force_flush() + span, attrs = assert_export(receiver, project) + assert_vendor_request(requests[0], TEST_BASE_URLS[0], "chat/completions") + params = json.loads(attrs["gen_ai.request.parameters"]) + assert params["model"] == MODEL and params["stream"] is True + assert json.loads(requests[0].content).get("stream_options") == ({"include_usage": True} if include_usage else None) + # 0.1.10 records the model only from non-streamed responses; later versions record the requested one. + assert attrs.get("gen_ai.request.model") in (None, MODEL) + assert attrs["output.value"] == ANSWER + assert span["status"]["code"] == "STATUS_CODE_OK" + assert_usage(attrs, USAGE if include_usage else {}) + + +def test_authentication_error_records_exception_and_model_gap(receiver, tracing): + requests = [] + with tracing() as (provider, project): + with mock_client(TEST_BASE_URLS[0], ERROR, requests, status=401) as client: + with pytest.raises(openai.AuthenticationError) as error: + complete(client) + provider.force_flush() + span, attrs = assert_export(receiver, project) + assert len(requests) == 1 + assert_vendor_request(requests[0], TEST_BASE_URLS[0], "chat/completions") + assert span["status"]["code"] == "STATUS_CODE_ERROR" + assert "invalid subscription key" in str(error.value) + assert any(event["name"] == "exception" for event in span["events"]) + assert "AuthenticationError" in json.dumps(span["events"]) + assert "invalid subscription key" in json.dumps(span) + # 0.1.10 records the model only from non-streamed responses; later versions record the requested one. + assert attrs.get("gen_ai.request.model") in (None, MODEL) + assert json.loads(attrs["gen_ai.request.parameters"])["model"] == MODEL + assert_usage(attrs, {}) + assert VENDOR_KEY not in str(error.value) + + +@pytest.mark.parametrize("flag,marker", [("FI_HIDE_INPUTS", "unique-private-prompt-marker"), ("FI_HIDE_OUTPUTS", ANSWER)]) +def test_privacy_hiding_has_visible_control(flag, marker, receiver, tracing, monkeypatch): + for hidden in (False, True): + receiver.clear() + monkeypatch.setenv(flag, str(hidden).lower()) + requests = [] + with tracing() as (provider, project): + with mock_client(TEST_BASE_URLS[0], chat_response(MODEL), requests) as client: + assert complete(client, prompt="unique-private-prompt-marker").choices[0].message.content == ANSWER + provider.force_flush() + assert_export(receiver, project) + assert_vendor_request(requests[0], TEST_BASE_URLS[0], "chat/completions") + assert json.loads(requests[0].content)["messages"][0]["content"] == "unique-private-prompt-marker" + assert (marker in json.dumps(receiver.spans())) is (not hidden) + + +RETURNED_MODEL = "gpt-test-model-2026-09-01" + + +def test_responses_api_span_and_exact_url(receiver, tracing): + requests = [] + with tracing() as (provider, project): + # The response names a different (underlying) model than the deployment requested. + with mock_client(TEST_BASE_URLS[0], responses_response(RETURNED_MODEL), requests) as client: + response = client.responses.create(model=MODEL, input="What is a rainbow?") + assert response.output_text == ANSWER + provider.force_flush() + span, attrs = assert_export(receiver, project, name="Response") + assert len(requests) == 1 + assert_vendor_request(requests[0], TEST_BASE_URLS[0], "responses") + assert requests[0].url.path == "/openai/v1/responses" + assert json.loads(requests[0].content)["input"] == "What is a rainbow?" + # On a successful Responses call the instrumentor records the model the response returns; + # the requested deployment stays in the request parameters. + assert attrs["gen_ai.request.model"] == RETURNED_MODEL + assert json.loads(attrs["gen_ai.request.parameters"])["model"] == MODEL + # Non-streamed Responses records the full parsed response's Python dict text. + # traceai-openai 0.1.10: Python repr of the response; later versions: JSON led by output_text. + if attrs["output.value"] != str(response.model_dump()): + body = json.loads(attrs["output.value"]) + assert body["output_text"] == ANSWER + assert body["output"][0]["content"][0]["text"] == ANSWER + assert attrs["output.mime_type"] == "application/json" + assert response.model_dump()["output"][0]["content"][0]["text"] == ANSWER + assert span["status"]["code"] == "STATUS_CODE_OK" + assert_usage(attrs, {**USAGE, "gen_ai.usage.output_tokens.reasoning": 0, "gen_ai.usage.input_tokens.cache_read": 0}) + + +WRONG_PATHS = ( + ("/api/projects/my-project", "project endpoint"), + ("/openai/deployments/my-gpt-deployment", "AzureOpenAI"), + ("/openai?api-version=2024-01-01", "AzureOpenAI"), + ("/", "bare resource root"), + ("", "bare resource root"), + ("/models", "azure-ai-inference"), +) +AZURE_HOSTS = ( + "test-resource-0000.openai.azure.com", + "TEST-RESOURCE-0000.SERVICES.AI.AZURE.COM", + "test-resource-0000.cognitiveservices.azure.com.", + "TEST-RESOURCE-0000.OPENAI.AZURE.COM.", +) + + +HARDENING_REFUSALS = ( + # The SDK folds the path into a query string, so the request would miss /chat/completions. + ("https://test-resource-0000.openai.azure.com/openai/v1/?api-version=preview", "query string"), + ("https://test-resource-0000.openai.azure.com/openai/v1?api-version=preview", "query string"), + ("https://test-resource-0000.openai.azure.com/openai/v1/#frag", "query string"), + # Encoded paths would pass a decoded check but go on the wire as a different path. + ("https://test-resource-0000.openai.azure.com/openai%2Fv1/", "Percent-encoded paths"), + ("https://test-resource-0000.openai.azure.com/openai/v1%3F/api/projects/x", "Percent-encoded paths"), + # Plain http would send the Azure key in cleartext. + ("http://test-resource-0000.openai.azure.com/openai/v1/", "must use https"), + ("http://test-resource-0000.services.ai.azure.com/openai/v1/", "must use https"), + # Credentials in the URL; this one only looks like Azure and would go to another host. + ("https://test-resource-0000.openai.azure.com@proxy.example/openai/v1/", "credentials"), + ("https://user:pass@test-resource-0000.openai.azure.com/openai/v1/", "credentials"), +) + + +@pytest.mark.parametrize("url,reason", HARDENING_REFUSALS) +def test_refuse_query_encoded_http_and_credentials(url, reason): + with pytest.raises(ValueError, match=reason) as error: + app.check_base_url(url) + assert app.BASE_URL_ENV in str(error.value) + assert "\n" not in str(error.value) + + +@pytest.mark.parametrize("host", AZURE_HOSTS) +@pytest.mark.parametrize("path,reason", WRONG_PATHS) +def test_refuse_wrong_azure_surfaces(host, path, reason): + with pytest.raises(ValueError, match=reason) as error: + app.check_base_url("https://" + host + path) + assert app.BASE_URL_ENV in str(error.value) + assert "\n" not in str(error.value) + + +@pytest.mark.parametrize("url", [ + *app.DOCUMENTED_BASE_URL_FORMS, + "https://your-resource-name.OPENAI.AZURE.COM/openai/v1/", + "https://.services.ai.azure.com/openai/v1/", + "https://%3Cresource%3E.openai.azure.com/openai/v1/", + "https://%253Cresource%253E.openai.azure.com/openai/v1/", + "https://%59OUR-RESOURCE-NAME.openai.azure.com/openai/v1/", + "https://%2559OUR-RESOURCE-NAME.openai.azure.com/openai/v1/", + "https://test-resource-0000.services.ai.azure.com/api/projects/", + "https://test-resource-0000.openai.azure.com/openai/deployments/", +]) +def test_refuse_literal_encoded_and_double_encoded_placeholders(url): + with pytest.raises(ValueError, match="Replace placeholders"): + app.check_base_url(url) + + +@pytest.mark.parametrize("url", [ + *TEST_BASE_URLS, + "https://test-resource-0000.openai.azure.com/openai/v1", + "https://TEST-RESOURCE-0000.OPENAI.AZURE.COM./openai/v1/", + "https://test-resource-0000.services.ai.azure.com./openai/v1/", + "https://test-resource-0000.cognitiveservices.azure.com/openai/v1/", + "https://proxy.example/custom/inference/", + "https://proxy.example/models", + "http://127.0.0.1:12345/openai/v1/", + "http://localhost:12345/custom/", +]) +def test_allowed_urls_are_returned_unchanged(url): + assert app.check_base_url(url) == url + + +@pytest.mark.parametrize("missing", [app.BASE_URL_ENV, app.MODEL_ENV, app.API_KEY_ENV]) +def test_missing_configuration_exits_before_tracing(missing, receiver, monkeypatch, capsys): + monkeypatch.delenv(missing) + monkeypatch.setattr(app, "setup_tracing", lambda *args, **kwargs: pytest.fail("Tracing ran before validation")) + assert app.main([]) == 2 + assert missing in capsys.readouterr().err + assert receiver.spans() == [] and receiver.requests() == [] + if missing == app.API_KEY_ENV: + with pytest.raises(KeyError, match=app.API_KEY_ENV): + app.make_client() + + +def test_refused_url_main_exits_without_requests_or_spans(receiver, monkeypatch, capsys): + # The setup_tracing and make_client traps are the controls: no tracer, no client, no request. + refused = "https://TEST-RESOURCE-0000.SERVICES.AI.AZURE.COM./api/projects/my-project" + monkeypatch.setenv(app.BASE_URL_ENV, refused) + monkeypatch.setattr(app, "setup_tracing", lambda *args, **kwargs: pytest.fail("Tracing ran for refused URL")) + monkeypatch.setattr(app, "make_client", lambda *args, **kwargs: pytest.fail("Client created for refused URL")) + assert app.main([]) == 2 + assert "project endpoint" in capsys.readouterr().err + assert receiver.spans() == [] and receiver.requests() == [] + + +def child_environment(tmp_path, receiver, base_url, project): + # Resolve actual imported packages so this also works with published wheels. + python_paths = [ + RECIPE / "tests" / "loopback_guard", RECIPE / "src", + Path(fi_instrumentation.__file__).resolve().parent.parent, + Path(traceai_openai.__file__).resolve().parent.parent, + Path(sys.modules["harness"].__file__).resolve().parent.parent, + ] + return { + "PATH": os.defpath, + "PYTHONDONTWRITEBYTECODE": "1", + "PYTHONPATH": os.pathsep.join(dict.fromkeys(str(path) for path in python_paths)), + "NO_PROXY": "*", + "FI_API_KEY": FI_KEY, "FI_SECRET_KEY": FI_SECRET, + "FI_BASE_URL": receiver.origin, "FI_PROJECT_NAME": project, + app.API_KEY_ENV: VENDOR_KEY, app.BASE_URL_ENV: base_url, app.MODEL_ENV: MODEL, + "LOOPBACK_GUARD_READY": str(tmp_path / "guard-ready"), + "LOOPBACK_GUARD_LOG": str(tmp_path / "guard.log"), + } + + +def assert_child_guard(env): + assert Path(env["LOOPBACK_GUARD_READY"]).read_text() == "loopback guard installed\n" + assert Path(env["LOOPBACK_GUARD_LOG"]).read_text() == "" + + +SUBPROCESS_JOURNEYS = [ + ("chat", True), ("chat", False), ("stream", True), ("stream", False), + ("model-override", True), ("responses", True), +] + + +@pytest.mark.parametrize("mode, env_project", SUBPROCESS_JOURNEYS, + ids=[f"{mode}-{'env' if env else 'default'}-project" for mode, env in SUBPROCESS_JOURNEYS]) +def test_loopback_subprocess_journeys(mode, env_project, tmp_path, receiver): + with FakeOpenAI() as fake: + env = child_environment(tmp_path, receiver, fake.base_url, "azure-subprocess-" + uuid.uuid4().hex) + if not env_project: + # Unset, app.py keeps the recipe default (never register()'s generic one). + env.pop("FI_PROJECT_NAME") + project = env.get("FI_PROJECT_NAME", "azure-ai-foundry") + model = MODEL + if mode == "responses": + command = [sys.executable, "-c", "import os, app; p = app.setup_tracing(os.environ['FI_PROJECT_NAME']); c = app.make_client(); print(c.responses.create(model=os.environ[app.MODEL_ENV], input='What is a rainbow?').output_text); c.close(); p.force_flush()"] + else: + command = [sys.executable, str(RECIPE / "src" / "app.py")] + if mode == "stream": + command.append("--stream") + elif mode == "model-override": + model = "override-deployment" + env.pop(app.MODEL_ENV) + command.extend(["--model", model]) + result = subprocess.run(command, env=env, capture_output=True, text=True, timeout=30) + assert result.returncode == 0, result.stderr + assert result.stdout.strip() == ANSWER + assert VENDOR_KEY not in result.stdout + result.stderr + assert_child_guard(env) + _, attrs = assert_export(receiver, project, name="Response" if mode == "responses" else "ChatCompletion") + requests = fake.requests() + assert len(requests) == 1 + assert requests[0]["path"] == ("/openai/v1/responses" if mode == "responses" else "/openai/v1/chat/completions") + assert requests[0]["headers"]["authorization"] == "Bearer " + VENDOR_KEY + assert "x-api-key" not in requests[0]["headers"] and "x-secret-key" not in requests[0]["headers"] + assert requests[0]["body"]["model"] == model + assert ANSWER in attrs["output.value"] + if mode == "stream": + # 0.1.10 records the model only from non-streamed responses; later versions record the requested one. + assert attrs.get("gen_ai.request.model") in (None, MODEL) + assert json.loads(attrs["gen_ai.request.parameters"])["model"] == model + assert_usage(attrs, {}) + else: + assert attrs["gen_ai.request.model"] == model + + +def test_guard_refuses_before_dns_with_loopback_control(tmp_path, receiver): + env = child_environment(tmp_path, receiver, TEST_BASE_URLS[0], "guard-control") + host = "test-resource-0000.openai.azure.com" + code = """import socket, sys, sitecustomize +from pathlib import Path +assert Path(sys.argv[1]).is_file() +assert socket.getaddrinfo('localhost', 80) +with socket.create_connection(('127.0.0.1', int(sys.argv[2])), timeout=5): + pass +def must_not_reach_network(*args, **kwargs): + raise AssertionError('Non-loopback operation reached DNS or connect') +sitecustomize._getaddrinfo = must_not_reach_network +sitecustomize._connect = must_not_reach_network +sitecustomize._connect_ex = must_not_reach_network +for operation in (lambda: socket.getaddrinfo(sys.argv[3], 443), + lambda: socket.socket().connect((sys.argv[3], 443)), + lambda: socket.socket().connect_ex((sys.argv[3], 443))): + try: + operation() + except RuntimeError as error: + assert 'Loopback guard refused' in str(error) + else: + raise AssertionError('Guard permitted non-loopback host') +print('guard controls passed') +""" + result = subprocess.run([sys.executable, "-c", code, env["LOOPBACK_GUARD_READY"], receiver.origin.rsplit(":", 1)[1], host], env=env, capture_output=True, text=True, timeout=15) + assert result.returncode == 0, result.stderr + assert result.stdout.strip() == "guard controls passed" + log = Path(env["LOOPBACK_GUARD_LOG"]).read_text().splitlines() + assert len(log) == 3 and all(host in line and "refused" in line for line in log) + assert receiver.requests() == [] + + +def test_readme_and_requirement_pins(): + readme = (RECIPE / "README.md").read_text() + for value in (*app.DOCUMENTED_BASE_URL_FORMS, app.API_KEY_ENV, app.BASE_URL_ENV, app.MODEL_ENV, "FI_API_KEY", "FI_SECRET_KEY", "FI_BASE_URL", "FI_PROJECT_NAME", "FI_HIDE_INPUTS", "FI_HIDE_OUTPUTS"): + assert value in readme + assert "| `FI_PROJECT_NAME` | No | Observe project name; defaults to `azure-ai-foundry`; tracer only. |" in readme + assert 'project_name=os.environ.get("FI_PROJECT_NAME") or "azure-ai-foundry"' in readme + assert "model is your deployment name" in readme + assert "provider field is `openai`" in readme + assert "Not this page: Foundry Agent Service" in readme + assert "agents are not this page" in readme + assert "AzureAIOpenTelemetryTracer" in readme and "Application Insights" in readme + assert SOURCE_COMMAND in readme and PUBLISHED_COMMAND in readme + assert "https://learn.microsoft.com/en-us/azure/foundry/openai/api-version-lifecycle" in readme + assert "syntactic test hosts" in readme and "not real" in readme + assert "gen_ai.request.model" in readme and "TBD" not in readme + assert "streamed and failed Chat Completions calls omit `gen_ai.request.model`" in readme + assert "a successful Responses call records the model the response returns" in readme + assert "may return the underlying model name rather than your deployment name" in readme + assert "Requires Python 3.10 or later" in readme + assert "Azure hosts must use `https://`" in readme + assert "`FI_HIDE_INPUTS` does not cover" in readme + assert "published `traceAI-openai==0.1.10` and `fi-instrumentation-otel==1.1.0`" in readme + exports = dict(re.findall(r'^export (\w+)="([^"]*)"$', readme, re.MULTILINE)) + assert exports["FI_API_KEY"] == FI_KEY + assert exports["FI_SECRET_KEY"] == FI_SECRET + assert exports[app.API_KEY_ENV] == VENDOR_KEY + assert not re.search(r"\b[A-Z]{2,5}-\d{3,}\b", readme), "internal ticket ids do not belong in the README" + for forbidden in ("prepared environment", "sk-", "all models", "every call"): + assert forbidden not in readme + assert (RECIPE / "requirements.txt").read_text().splitlines()[1:] == [ + "openai==3.24.0", "traceAI-openai==0.1.10", "fi-instrumentation-otel==1.1.0", + ] diff --git a/python/tests/harness/__init__.py b/python/tests/harness/__init__.py new file mode 100644 index 000000000..cf79047f1 --- /dev/null +++ b/python/tests/harness/__init__.py @@ -0,0 +1,238 @@ +"""Small, loopback-only utilities for OTLP contract tests.""" + +from __future__ import annotations + +import copy +import difflib +import json +import subprocess +import threading +from http import HTTPStatus +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from typing import Any, Mapping, Optional, Sequence, Union +from urllib.parse import urlsplit +from urllib.request import Request, urlopen + + +class Receiver: + """An in-memory OTLP/HTTP receiver for tests. + + The receiver only listens on loopback and never forwards received spans. + """ + + def __init__(self) -> None: + self._spans: list[dict[str, Any]] = [] + self._requests: list[dict[str, Any]] = [] + self._lock = threading.Lock() + owner = self + + class Handler(BaseHTTPRequestHandler): + def _read_body(self) -> bytes: + # The Node OTLP exporter streams with Transfer-Encoding: chunked + # and sends no Content-Length, so both framings are accepted. + if "chunked" in self.headers.get("Transfer-Encoding", "").lower(): + chunks = [] + while True: + size_line = self.rfile.readline().split(b";", 1)[0].strip() + size = int(size_line, 16) + if size == 0: + while self.rfile.readline() not in (b"\r\n", b"\n", b""): + pass + return b"".join(chunks) + chunks.append(self.rfile.read(size)) + self.rfile.readline() + length = int(self.headers.get("Content-Length", "0")) + return self.rfile.read(length) + + def do_POST(self) -> None: # noqa: N802 - required by BaseHTTPRequestHandler + path = urlsplit(self.path).path + if path not in ("/v1/traces", "/tracer/v1/traces"): + self.send_error(HTTPStatus.NOT_FOUND) + return + + try: + body = self._read_body() + content_type = self.headers.get("Content-Type", "").lower() + + if content_type.startswith("application/json"): + request = json.loads(body.decode("utf-8")) + elif content_type.startswith(("application/x-protobuf", "application/protobuf")): + from google.protobuf.json_format import MessageToDict + from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ( + ExportTraceServiceRequest, + ) + + protobuf_request = ExportTraceServiceRequest() + protobuf_request.ParseFromString(body) + request = MessageToDict(protobuf_request) + else: + self.send_error(HTTPStatus.UNSUPPORTED_MEDIA_TYPE) + return + + spans: list[dict[str, Any]] = [] + resource_attributes: list[dict[str, Any]] = [] + for resource_spans in request.get( + "resourceSpans", request.get("resource_spans", []) + ): + resource_attributes.append( + _flatten_attributes( + resource_spans.get("resource", {}).get("attributes", []) + ) + ) + for scope_spans in resource_spans.get( + "scopeSpans", resource_spans.get("scope_spans", []) + ): + spans.extend(scope_spans.get("spans", [])) + except (ImportError, TypeError, UnicodeDecodeError, ValueError) as error: + self.send_error(HTTPStatus.BAD_REQUEST, str(error)) + return + + with owner._lock: + owner._spans.extend(copy.deepcopy(spans)) + owner._requests.append( + { + "path": path, + "headers": { + key.lower(): value for key, value in self.headers.items() + }, + "resource_attributes": resource_attributes, + } + ) + self.send_response(HTTPStatus.OK) + self.send_header("Content-Length", "0") + self.end_headers() + + def log_message(self, _format: str, *args: Any) -> None: + return + + self._server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + self.origin = "http://127.0.0.1:{0}".format(self._server.server_port) + self.endpoint = "{0}/v1/traces".format(self.origin) + self.collector_endpoint = "{0}/tracer/v1/traces".format(self.origin) + self._thread = threading.Thread(target=self._server.serve_forever, daemon=True) + self._thread.start() + + def __enter__(self) -> "Receiver": + return self + + def __exit__(self, _type: Any, _value: Any, _traceback: Any) -> None: + self._server.shutdown() + self._server.server_close() + self._thread.join() + + def spans(self) -> list[dict[str, Any]]: + """Return a copy of the decoded spans received so far.""" + with self._lock: + return copy.deepcopy(self._spans) + + def requests(self) -> list[dict[str, Any]]: + """Return one record per accepted export: path, lower-cased headers, resource attributes.""" + with self._lock: + return copy.deepcopy(self._requests) + + def clear(self) -> None: + """Remove every decoded span and request record received so far.""" + with self._lock: + self._spans.clear() + self._requests.clear() + + +def _flatten_attributes(attributes: Sequence[Mapping[str, Any]]) -> dict[str, Any]: + """Turn OTLP key/value attributes into a plain dict of scalar values.""" + flat: dict[str, Any] = {} + for attribute in attributes: + value = attribute.get("value", {}) + for kind in ("stringValue", "boolValue", "intValue", "doubleValue"): + if kind in value: + flat[attribute.get("key", "")] = value[kind] + break + else: + flat[attribute.get("key", "")] = value + return flat + + +def run( + argv: Sequence[str], + env: Mapping[str, str], + stdin: Optional[bytes], + timeout: float, +) -> subprocess.CompletedProcess[bytes]: + """Run one process and return its captured output, including after a timeout.""" + process = subprocess.Popen( + argv, + env=dict(env), + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + timed_out = False + try: + stdout, stderr = process.communicate(input=stdin, timeout=timeout) + except subprocess.TimeoutExpired: + timed_out = True + process.kill() + stdout, stderr = process.communicate() + + result = subprocess.CompletedProcess(argv, process.returncode, stdout, stderr) + result.timed_out = timed_out + return result + + +def post_otlp(spans: dict[str, Any], endpoint: str) -> int: + """Post one JSON-encoded OTLP request to a loopback receiver.""" + parsed = urlsplit(endpoint) + if parsed.scheme != "http" or parsed.hostname != "127.0.0.1": + raise ValueError("post_otlp only sends to a loopback HTTP endpoint") + + request = Request( + endpoint, + data=json.dumps(spans).encode("utf-8"), + headers={"Content-Type": "application/json"}, + method="POST", + ) + with urlopen(request, timeout=5) as response: + return response.status + + +def compare(actual: Sequence[dict[str, Any]], golden_path: Union[str, Path]) -> None: + """Fail with a diff unless span names, attributes, and statuses match a golden.""" + with Path(golden_path).open(encoding="utf-8") as golden_file: + expected = json.load(golden_file) + + def canonical(spans: Sequence[dict[str, Any]]) -> list[dict[str, Any]]: + normalized = [] + for span in spans: + attributes = span.get("attributes", []) + if isinstance(attributes, dict): + attributes = [ + {"key": key, "value": value} for key, value in attributes.items() + ] + normalized.append( + { + "name": span.get("name", ""), + "attributes": sorted( + attributes, + key=lambda attribute: json.dumps( + attribute, sort_keys=True, separators=(",", ":") + ), + ), + "status": span.get("status", {}), + } + ) + return normalized + + actual_normalized = canonical(actual) + expected_normalized = canonical(expected) + if actual_normalized != expected_normalized: + diff = "\n".join( + difflib.unified_diff( + json.dumps(expected_normalized, indent=2, sort_keys=True).splitlines(), + json.dumps(actual_normalized, indent=2, sort_keys=True).splitlines(), + fromfile=str(golden_path), + tofile="actual", + lineterm="", + ) + ) + print(diff) + raise AssertionError("spans do not match golden") diff --git a/python/tests/harness/test_harness.py b/python/tests/harness/test_harness.py new file mode 100644 index 000000000..bd61d5adc --- /dev/null +++ b/python/tests/harness/test_harness.py @@ -0,0 +1,189 @@ +"""Contract tests for the shared loopback-only harness.""" + +import json +import sys + +import pytest + +from tests.harness import Receiver, compare, post_otlp, run + + +def test_receiver_stores_posted_span_and_can_clear() -> None: + payload = { + "resourceSpans": [ + { + "scopeSpans": [ + { + "spans": [ + { + "traceId": "trace-id", + "spanId": "span-id", + "name": "harness.receiver", + "attributes": [ + {"key": "component", "value": {"stringValue": "test"}} + ], + "status": {"code": 1}, + } + ] + } + ] + } + ] + } + + with Receiver() as receiver: + assert post_otlp(payload, receiver.endpoint) == 200 + assert receiver.spans() == payload["resourceSpans"][0]["scopeSpans"][0]["spans"] + receiver.clear() + assert receiver.spans() == [] + + +def test_receiver_accepts_the_collector_path() -> None: + payload = { + "resourceSpans": [ + {"scopeSpans": [{"spans": [{"name": "collector.path"}]}]} + ] + } + + with Receiver() as receiver: + assert post_otlp(payload, receiver.collector_endpoint) == 200 + assert receiver.spans() == [{"name": "collector.path"}] + + +def _post_chunked(url: str, body: bytes, headers: dict) -> int: + """Send a body with Transfer-Encoding: chunked, as the Node OTLP exporter does.""" + import http.client + from urllib.parse import urlsplit + + parts = urlsplit(url) + connection = http.client.HTTPConnection(parts.hostname, parts.port, timeout=5) + try: + chunks = [body[: len(body) // 2], body[len(body) // 2 :]] + connection.request( + "POST", + parts.path, + body=iter(chunks), + headers={**headers, "Transfer-Encoding": "chunked"}, + encode_chunked=True, + ) + return connection.getresponse().status + finally: + connection.close() + + +def test_receiver_decodes_a_chunked_body() -> None: + payload = { + "resourceSpans": [ + {"scopeSpans": [{"spans": [{"name": "chunked.body"}]}]} + ] + } + + with Receiver() as receiver: + status = _post_chunked( + receiver.collector_endpoint, + json.dumps(payload).encode("utf-8"), + {"Content-Type": "application/json"}, + ) + assert status == 200 + assert receiver.spans() == [{"name": "chunked.body"}] + + +def test_receiver_records_path_headers_and_resource_attributes() -> None: + payload = { + "resourceSpans": [ + { + "resource": { + "attributes": [ + {"key": "project_name", "value": {"stringValue": "harness"}}, + {"key": "project_type", "value": {"stringValue": "observe"}}, + ] + }, + "scopeSpans": [{"spans": [{"name": "with.resource"}]}], + } + ] + } + + with Receiver() as receiver: + status = _post_chunked( + receiver.collector_endpoint, + json.dumps(payload).encode("utf-8"), + { + "Content-Type": "application/json", + "X-Api-Key": "test-key", + "X-Secret-Key": "test-secret", + }, + ) + assert status == 200 + [request] = receiver.requests() + assert request["path"] == "/tracer/v1/traces" + assert request["headers"]["x-api-key"] == "test-key" + assert request["headers"]["x-secret-key"] == "test-secret" + assert "authorization" not in request["headers"] + assert request["resource_attributes"] == [ + {"project_name": "harness", "project_type": "observe"} + ] + receiver.clear() + assert receiver.requests() == [] + + +def test_run_captures_output_and_kills_timed_out_process() -> None: + completed = run( + [ + sys.executable, + "-c", + "import sys; print('stdout'); sys.stderr.write('stderr\\n')", + ], + env={}, + stdin=b"", + timeout=1, + ) + assert completed.returncode == 0 + assert completed.stdout == b"stdout\n" + assert completed.stderr == b"stderr\n" + assert completed.timed_out is False + + timed_out = run( + [sys.executable, "-c", "import time; time.sleep(30)"], + env={}, + stdin=b"", + timeout=0.1, + ) + assert timed_out.timed_out is True + assert timed_out.returncode != 0 + + +def test_compare_accepts_normalized_match_and_prints_mismatch_diff(tmp_path, capsys) -> None: + golden = [ + { + "traceId": "golden-trace", + "spanId": "golden-span", + "startTimeUnixNano": "1", + "endTimeUnixNano": "2", + "name": "harness.compare", + "attributes": [ + {"key": "alpha", "value": {"stringValue": "a"}}, + {"key": "beta", "value": {"intValue": "2"}}, + ], + "status": {"code": 1}, + } + ] + golden_path = tmp_path / "spans.golden.json" + golden_path.write_text(json.dumps(golden), encoding="utf-8") + + actual = [ + { + "traceId": "other-trace", + "spanId": "other-span", + "startTimeUnixNano": "3", + "endTimeUnixNano": "4", + "name": "harness.compare", + "attributes": list(reversed(golden[0]["attributes"])), + "status": {"code": 1}, + } + ] + compare(actual, golden_path) + + actual[0]["name"] = "harness.mismatch" + with pytest.raises(AssertionError, match="spans do not match golden"): + compare(actual, golden_path) + assert "---" in capsys.readouterr().out