From 5936e0960f01a23e842d3df90b7c101874ed8149 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Sat, 3 Oct 2026 18:58:31 -0700 Subject: [PATCH 01/10] test(harness): add shared OTLP harness for TH-8103 contract tests --- python/tests/harness/__init__.py | 184 +++++++++++++++++++++++++++ python/tests/harness/test_harness.py | 101 +++++++++++++++ 2 files changed, 285 insertions(+) create mode 100644 python/tests/harness/__init__.py create mode 100644 python/tests/harness/test_harness.py diff --git a/python/tests/harness/__init__.py b/python/tests/harness/__init__.py new file mode 100644 index 000000000..dfd9198e4 --- /dev/null +++ b/python/tests/harness/__init__.py @@ -0,0 +1,184 @@ +"""Small, loopback-only utilities for OTLP contract tests.""" + +from __future__ import annotations + +import copy +import difflib +import json +import subprocess +import threading +from http import HTTPStatus +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from typing import Any, Mapping, Optional, Sequence, Union +from urllib.parse import urlsplit +from urllib.request import Request, urlopen + + +class Receiver: + """An in-memory OTLP/HTTP receiver for tests. + + The receiver only listens on loopback and never forwards received spans. + """ + + def __init__(self) -> None: + self._spans: list[dict[str, Any]] = [] + self._lock = threading.Lock() + owner = self + + class Handler(BaseHTTPRequestHandler): + def do_POST(self) -> None: # noqa: N802 - required by BaseHTTPRequestHandler + if urlsplit(self.path).path != "/v1/traces": + self.send_error(HTTPStatus.NOT_FOUND) + return + + try: + length = int(self.headers.get("Content-Length", "0")) + body = self.rfile.read(length) + content_type = self.headers.get("Content-Type", "").lower() + + if content_type.startswith("application/json"): + request = json.loads(body.decode("utf-8")) + elif content_type.startswith(("application/x-protobuf", "application/protobuf")): + from google.protobuf.json_format import MessageToDict + from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ( + ExportTraceServiceRequest, + ) + + protobuf_request = ExportTraceServiceRequest() + protobuf_request.ParseFromString(body) + request = MessageToDict(protobuf_request) + else: + self.send_error(HTTPStatus.UNSUPPORTED_MEDIA_TYPE) + return + + spans: list[dict[str, Any]] = [] + for resource_spans in request.get( + "resourceSpans", request.get("resource_spans", []) + ): + for scope_spans in resource_spans.get( + "scopeSpans", resource_spans.get("scope_spans", []) + ): + spans.extend(scope_spans.get("spans", [])) + except (ImportError, TypeError, UnicodeDecodeError, ValueError) as error: + self.send_error(HTTPStatus.BAD_REQUEST, str(error)) + return + + with owner._lock: + owner._spans.extend(copy.deepcopy(spans)) + self.send_response(HTTPStatus.OK) + self.send_header("Content-Length", "0") + self.end_headers() + + def log_message(self, _format: str, *args: Any) -> None: + return + + self._server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + self.origin = "http://127.0.0.1:{0}".format(self._server.server_port) + self.endpoint = "{0}/v1/traces".format(self.origin) + self._thread = threading.Thread(target=self._server.serve_forever, daemon=True) + self._thread.start() + + def __enter__(self) -> "Receiver": + return self + + def __exit__(self, _type: Any, _value: Any, _traceback: Any) -> None: + self._server.shutdown() + self._server.server_close() + self._thread.join() + + def spans(self) -> list[dict[str, Any]]: + """Return a copy of the decoded spans received so far.""" + with self._lock: + return copy.deepcopy(self._spans) + + def clear(self) -> None: + """Remove every decoded span received so far.""" + with self._lock: + self._spans.clear() + + +def run( + argv: Sequence[str], + env: Mapping[str, str], + stdin: Optional[bytes], + timeout: float, +) -> subprocess.CompletedProcess[bytes]: + """Run one process and return its captured output, including after a timeout.""" + process = subprocess.Popen( + argv, + env=dict(env), + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + timed_out = False + try: + stdout, stderr = process.communicate(input=stdin, timeout=timeout) + except subprocess.TimeoutExpired: + timed_out = True + process.kill() + stdout, stderr = process.communicate() + + result = subprocess.CompletedProcess(argv, process.returncode, stdout, stderr) + result.timed_out = timed_out + return result + + +def post_otlp(spans: dict[str, Any], endpoint: str) -> int: + """Post one JSON-encoded OTLP request to a loopback receiver.""" + parsed = urlsplit(endpoint) + if parsed.scheme != "http" or parsed.hostname != "127.0.0.1": + raise ValueError("post_otlp only sends to a loopback HTTP endpoint") + + request = Request( + endpoint, + data=json.dumps(spans).encode("utf-8"), + headers={"Content-Type": "application/json"}, + method="POST", + ) + with urlopen(request, timeout=5) as response: + return response.status + + +def compare(actual: Sequence[dict[str, Any]], golden_path: Union[str, Path]) -> None: + """Fail with a diff unless span names, attributes, and statuses match a golden.""" + with Path(golden_path).open(encoding="utf-8") as golden_file: + expected = json.load(golden_file) + + def canonical(spans: Sequence[dict[str, Any]]) -> list[dict[str, Any]]: + normalized = [] + for span in spans: + attributes = span.get("attributes", []) + if isinstance(attributes, dict): + attributes = [ + {"key": key, "value": value} for key, value in attributes.items() + ] + normalized.append( + { + "name": span.get("name", ""), + "attributes": sorted( + attributes, + key=lambda attribute: json.dumps( + attribute, sort_keys=True, separators=(",", ":") + ), + ), + "status": span.get("status", {}), + } + ) + return normalized + + actual_normalized = canonical(actual) + expected_normalized = canonical(expected) + if actual_normalized != expected_normalized: + diff = "\n".join( + difflib.unified_diff( + json.dumps(expected_normalized, indent=2, sort_keys=True).splitlines(), + json.dumps(actual_normalized, indent=2, sort_keys=True).splitlines(), + fromfile=str(golden_path), + tofile="actual", + lineterm="", + ) + ) + print(diff) + raise AssertionError("spans do not match golden") diff --git a/python/tests/harness/test_harness.py b/python/tests/harness/test_harness.py new file mode 100644 index 000000000..4e76ceabe --- /dev/null +++ b/python/tests/harness/test_harness.py @@ -0,0 +1,101 @@ +"""Contract tests for the shared loopback-only harness.""" + +import json +import sys + +import pytest + +from tests.harness import Receiver, compare, post_otlp, run + + +def test_receiver_stores_posted_span_and_can_clear() -> None: + payload = { + "resourceSpans": [ + { + "scopeSpans": [ + { + "spans": [ + { + "traceId": "trace-id", + "spanId": "span-id", + "name": "harness.receiver", + "attributes": [ + {"key": "component", "value": {"stringValue": "test"}} + ], + "status": {"code": 1}, + } + ] + } + ] + } + ] + } + + with Receiver() as receiver: + assert post_otlp(payload, receiver.endpoint) == 200 + assert receiver.spans() == payload["resourceSpans"][0]["scopeSpans"][0]["spans"] + receiver.clear() + assert receiver.spans() == [] + + +def test_run_captures_output_and_kills_timed_out_process() -> None: + completed = run( + [ + sys.executable, + "-c", + "import sys; print('stdout'); sys.stderr.write('stderr\\n')", + ], + env={}, + stdin=b"", + timeout=1, + ) + assert completed.returncode == 0 + assert completed.stdout == b"stdout\n" + assert completed.stderr == b"stderr\n" + assert completed.timed_out is False + + timed_out = run( + [sys.executable, "-c", "import time; time.sleep(30)"], + env={}, + stdin=b"", + timeout=0.1, + ) + assert timed_out.timed_out is True + assert timed_out.returncode != 0 + + +def test_compare_accepts_normalized_match_and_prints_mismatch_diff(tmp_path, capsys) -> None: + golden = [ + { + "traceId": "golden-trace", + "spanId": "golden-span", + "startTimeUnixNano": "1", + "endTimeUnixNano": "2", + "name": "harness.compare", + "attributes": [ + {"key": "alpha", "value": {"stringValue": "a"}}, + {"key": "beta", "value": {"intValue": "2"}}, + ], + "status": {"code": 1}, + } + ] + golden_path = tmp_path / "spans.golden.json" + golden_path.write_text(json.dumps(golden), encoding="utf-8") + + actual = [ + { + "traceId": "other-trace", + "spanId": "other-span", + "startTimeUnixNano": "3", + "endTimeUnixNano": "4", + "name": "harness.compare", + "attributes": list(reversed(golden[0]["attributes"])), + "status": {"code": 1}, + } + ] + compare(actual, golden_path) + + actual[0]["name"] = "harness.mismatch" + with pytest.raises(AssertionError, match="spans do not match golden"): + compare(actual, golden_path) + assert "---" in capsys.readouterr().out From 32f5c9d2b3b80f97a70e00b3f3ca1b408b14dc5c Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Sat, 3 Oct 2026 23:21:29 -0700 Subject: [PATCH 02/10] test(harness): accept the collector /tracer/v1/traces path --- python/tests/harness/__init__.py | 3 ++- python/tests/harness/test_harness.py | 12 ++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/python/tests/harness/__init__.py b/python/tests/harness/__init__.py index dfd9198e4..cb510ad10 100644 --- a/python/tests/harness/__init__.py +++ b/python/tests/harness/__init__.py @@ -28,7 +28,7 @@ def __init__(self) -> None: class Handler(BaseHTTPRequestHandler): def do_POST(self) -> None: # noqa: N802 - required by BaseHTTPRequestHandler - if urlsplit(self.path).path != "/v1/traces": + if urlsplit(self.path).path not in ("/v1/traces", "/tracer/v1/traces"): self.send_error(HTTPStatus.NOT_FOUND) return @@ -76,6 +76,7 @@ def log_message(self, _format: str, *args: Any) -> None: self._server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) self.origin = "http://127.0.0.1:{0}".format(self._server.server_port) self.endpoint = "{0}/v1/traces".format(self.origin) + self.collector_endpoint = "{0}/tracer/v1/traces".format(self.origin) self._thread = threading.Thread(target=self._server.serve_forever, daemon=True) self._thread.start() diff --git a/python/tests/harness/test_harness.py b/python/tests/harness/test_harness.py index 4e76ceabe..6a32c2af8 100644 --- a/python/tests/harness/test_harness.py +++ b/python/tests/harness/test_harness.py @@ -38,6 +38,18 @@ def test_receiver_stores_posted_span_and_can_clear() -> None: assert receiver.spans() == [] +def test_receiver_accepts_the_collector_path() -> None: + payload = { + "resourceSpans": [ + {"scopeSpans": [{"spans": [{"name": "collector.path"}]}]} + ] + } + + with Receiver() as receiver: + assert post_otlp(payload, receiver.collector_endpoint) == 200 + assert receiver.spans() == [{"name": "collector.path"}] + + def test_run_captures_output_and_kills_timed_out_process() -> None: completed = run( [ From 3eaadc8bdbf428a472db6779091bbce3d6d89015 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Sun, 4 Oct 2026 15:59:06 -0700 Subject: [PATCH 03/10] fix(harness): decode chunked OTLP bodies and record request metadata The Node OTLP exporter streams with Transfer-Encoding: chunked and sends no Content-Length. The Receiver read Content-Length only, so every Node export came back 400 with zero spans; two TH-8103 children had to put a de-chunking relay in front of it. Receiver now accepts both framings. It also records one entry per accepted export (path, lower-cased headers, flattened resource attributes) via requests(), so a contract test can assert X-Api-Key/X-Secret-Key and project_name through the shared harness instead of a private recorder. Verified: 6 harness tests pass, and the TanStack example's real Node exporter delivered 4 chunked exports (4 spans, collector path, both auth headers, project_name/project_type) with no relay. Refs: TH-8339, TH-8103 --- python/tests/harness/__init__.py | 61 ++++++++++++++++++++-- python/tests/harness/test_harness.py | 76 ++++++++++++++++++++++++++++ 2 files changed, 133 insertions(+), 4 deletions(-) diff --git a/python/tests/harness/__init__.py b/python/tests/harness/__init__.py index cb510ad10..cf79047f1 100644 --- a/python/tests/harness/__init__.py +++ b/python/tests/harness/__init__.py @@ -23,18 +23,36 @@ class Receiver: def __init__(self) -> None: self._spans: list[dict[str, Any]] = [] + self._requests: list[dict[str, Any]] = [] self._lock = threading.Lock() owner = self class Handler(BaseHTTPRequestHandler): + def _read_body(self) -> bytes: + # The Node OTLP exporter streams with Transfer-Encoding: chunked + # and sends no Content-Length, so both framings are accepted. + if "chunked" in self.headers.get("Transfer-Encoding", "").lower(): + chunks = [] + while True: + size_line = self.rfile.readline().split(b";", 1)[0].strip() + size = int(size_line, 16) + if size == 0: + while self.rfile.readline() not in (b"\r\n", b"\n", b""): + pass + return b"".join(chunks) + chunks.append(self.rfile.read(size)) + self.rfile.readline() + length = int(self.headers.get("Content-Length", "0")) + return self.rfile.read(length) + def do_POST(self) -> None: # noqa: N802 - required by BaseHTTPRequestHandler - if urlsplit(self.path).path not in ("/v1/traces", "/tracer/v1/traces"): + path = urlsplit(self.path).path + if path not in ("/v1/traces", "/tracer/v1/traces"): self.send_error(HTTPStatus.NOT_FOUND) return try: - length = int(self.headers.get("Content-Length", "0")) - body = self.rfile.read(length) + body = self._read_body() content_type = self.headers.get("Content-Type", "").lower() if content_type.startswith("application/json"): @@ -53,9 +71,15 @@ def do_POST(self) -> None: # noqa: N802 - required by BaseHTTPRequestHandler return spans: list[dict[str, Any]] = [] + resource_attributes: list[dict[str, Any]] = [] for resource_spans in request.get( "resourceSpans", request.get("resource_spans", []) ): + resource_attributes.append( + _flatten_attributes( + resource_spans.get("resource", {}).get("attributes", []) + ) + ) for scope_spans in resource_spans.get( "scopeSpans", resource_spans.get("scope_spans", []) ): @@ -66,6 +90,15 @@ def do_POST(self) -> None: # noqa: N802 - required by BaseHTTPRequestHandler with owner._lock: owner._spans.extend(copy.deepcopy(spans)) + owner._requests.append( + { + "path": path, + "headers": { + key.lower(): value for key, value in self.headers.items() + }, + "resource_attributes": resource_attributes, + } + ) self.send_response(HTTPStatus.OK) self.send_header("Content-Length", "0") self.end_headers() @@ -93,10 +126,30 @@ def spans(self) -> list[dict[str, Any]]: with self._lock: return copy.deepcopy(self._spans) + def requests(self) -> list[dict[str, Any]]: + """Return one record per accepted export: path, lower-cased headers, resource attributes.""" + with self._lock: + return copy.deepcopy(self._requests) + def clear(self) -> None: - """Remove every decoded span received so far.""" + """Remove every decoded span and request record received so far.""" with self._lock: self._spans.clear() + self._requests.clear() + + +def _flatten_attributes(attributes: Sequence[Mapping[str, Any]]) -> dict[str, Any]: + """Turn OTLP key/value attributes into a plain dict of scalar values.""" + flat: dict[str, Any] = {} + for attribute in attributes: + value = attribute.get("value", {}) + for kind in ("stringValue", "boolValue", "intValue", "doubleValue"): + if kind in value: + flat[attribute.get("key", "")] = value[kind] + break + else: + flat[attribute.get("key", "")] = value + return flat def run( diff --git a/python/tests/harness/test_harness.py b/python/tests/harness/test_harness.py index 6a32c2af8..bd61d5adc 100644 --- a/python/tests/harness/test_harness.py +++ b/python/tests/harness/test_harness.py @@ -50,6 +50,82 @@ def test_receiver_accepts_the_collector_path() -> None: assert receiver.spans() == [{"name": "collector.path"}] +def _post_chunked(url: str, body: bytes, headers: dict) -> int: + """Send a body with Transfer-Encoding: chunked, as the Node OTLP exporter does.""" + import http.client + from urllib.parse import urlsplit + + parts = urlsplit(url) + connection = http.client.HTTPConnection(parts.hostname, parts.port, timeout=5) + try: + chunks = [body[: len(body) // 2], body[len(body) // 2 :]] + connection.request( + "POST", + parts.path, + body=iter(chunks), + headers={**headers, "Transfer-Encoding": "chunked"}, + encode_chunked=True, + ) + return connection.getresponse().status + finally: + connection.close() + + +def test_receiver_decodes_a_chunked_body() -> None: + payload = { + "resourceSpans": [ + {"scopeSpans": [{"spans": [{"name": "chunked.body"}]}]} + ] + } + + with Receiver() as receiver: + status = _post_chunked( + receiver.collector_endpoint, + json.dumps(payload).encode("utf-8"), + {"Content-Type": "application/json"}, + ) + assert status == 200 + assert receiver.spans() == [{"name": "chunked.body"}] + + +def test_receiver_records_path_headers_and_resource_attributes() -> None: + payload = { + "resourceSpans": [ + { + "resource": { + "attributes": [ + {"key": "project_name", "value": {"stringValue": "harness"}}, + {"key": "project_type", "value": {"stringValue": "observe"}}, + ] + }, + "scopeSpans": [{"spans": [{"name": "with.resource"}]}], + } + ] + } + + with Receiver() as receiver: + status = _post_chunked( + receiver.collector_endpoint, + json.dumps(payload).encode("utf-8"), + { + "Content-Type": "application/json", + "X-Api-Key": "test-key", + "X-Secret-Key": "test-secret", + }, + ) + assert status == 200 + [request] = receiver.requests() + assert request["path"] == "/tracer/v1/traces" + assert request["headers"]["x-api-key"] == "test-key" + assert request["headers"]["x-secret-key"] == "test-secret" + assert "authorization" not in request["headers"] + assert request["resource_attributes"] == [ + {"project_name": "harness", "project_type": "observe"} + ] + receiver.clear() + assert receiver.requests() == [] + + def test_run_captures_output_and_kills_timed_out_process() -> None: completed = run( [ From b61fc881f7d3fec4d9a817e00d86c14f804427f9 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Mon, 5 Oct 2026 20:09:51 -0700 Subject: [PATCH 04/10] feat(examples): Azure AI Foundry model-inference recipe on traceai-openai Adds python/examples/azure-ai-foundry/: the official openai SDK pointed at an Azure resource's v1 endpoint (https://YOUR-RESOURCE-NAME.openai.azure.com/openai/v1/ or https://YOUR-RESOURCE-NAME.services.ai.azure.com/openai/v1/), traced by the existing traceai-openai instrumentor. No provider package. - AZURE_OPENAI_BASE_URL is required (no public default); template URLs that still contain YOUR-RESOURCE-NAME or angle brackets (literal, encoded or double-encoded) are refused before tracing. - check_base_url refuses Azure hosts whose path is not /openai/v1[/] (project endpoint, deployment-scoped api-version URLs, bare root, /models), including upper-case and trailing-dot spellings; it never rewrites a URL. main() exits 2 before tracing on a refused URL or a missing base URL, deployment or key. - Azure key only on the OpenAI client; Future AGI keys only on the tracer. - Pins current traceai-openai behaviour: the span's provider label is openai for both host forms (not azure); streamed and failed Chat Completions spans have no gen_ai.request.model (the deployment stays in gen_ai.request.parameters); one Responses API case at the same base URL records the model. - Loopback only: MockTransport, a local OpenAI-shaped fake, the shared harness receiver and a socket guard with a negative control. approved: Nikhil 2026-10-03 blanket Refs: TH-8317 --- python/examples/azure-ai-foundry/README.md | 217 ++++++++ .../azure-ai-foundry/requirements.txt | 4 + python/examples/azure-ai-foundry/src/app.py | 111 ++++ .../azure-ai-foundry/tests/_fake_openai.py | 114 ++++ .../tests/loopback_guard/sitecustomize.py | 47 ++ .../tests/test_azure_ai_foundry_recipe.py | 504 ++++++++++++++++++ 6 files changed, 997 insertions(+) create mode 100644 python/examples/azure-ai-foundry/README.md create mode 100644 python/examples/azure-ai-foundry/requirements.txt create mode 100644 python/examples/azure-ai-foundry/src/app.py create mode 100644 python/examples/azure-ai-foundry/tests/_fake_openai.py create mode 100644 python/examples/azure-ai-foundry/tests/loopback_guard/sitecustomize.py create mode 100644 python/examples/azure-ai-foundry/tests/test_azure_ai_foundry_recipe.py diff --git a/python/examples/azure-ai-foundry/README.md b/python/examples/azure-ai-foundry/README.md new file mode 100644 index 000000000..0fedd3c25 --- /dev/null +++ b/python/examples/azure-ai-foundry/README.md @@ -0,0 +1,217 @@ +# Azure AI Foundry (OpenAI-compatible) with traceAI + +This recipe traces Chat Completions through the official `openai` Python SDK at your Azure resource's `/openai/v1/` endpoint; model is your deployment name, not a model id. +It uses `traceai-openai`, with no separate provider package. + +## Install + +From this example directory, install the pinned dependencies: + +```bash +pip install -r requirements.txt +``` + +The plain package form is: + +```bash +pip install traceAI-openai fi-instrumentation-otel openai +``` + +## Configure + +| Variable | Required | Purpose | +|---|---|---| +| `FI_API_KEY` | Yes | Future AGI API key; tracer only. | +| `FI_SECRET_KEY` | Yes | Future AGI secret key; tracer only. | +| `AZURE_OPENAI_API_KEY` | Yes | Azure resource API key; OpenAI client only. | +| `AZURE_OPENAI_DEPLOYMENT` | Yes, unless `--model` is used | Your deployment name, not an underlying model id. | +| `AZURE_OPENAI_BASE_URL` | Yes | Your resource-specific inference base URL. There is no public default. | +| `FI_BASE_URL` | No | Future AGI collector origin; defaults to `https://api.futureagi.com`. | +| `FI_HIDE_INPUTS` | No | Set to `true` to hide prompt text in exported spans. | +| `FI_HIDE_OUTPUTS` | No | Set to `true` to hide response text in exported spans. | + +The client receives the Azure key. The tracer receives the Future AGI keys. +The example uses the Future AGI project name `azure-ai-foundry`. + +## Run + +Replace the placeholders before running. A template URL that still contains +`YOUR-RESOURCE-NAME` or angle brackets fails before tracing or a network call. + +```bash +export FI_API_KEY="placeholder-futureagi-key" +export FI_SECRET_KEY="placeholder-futureagi-secret" +export AZURE_OPENAI_API_KEY="placeholder-azure-key" +export AZURE_OPENAI_BASE_URL="https://YOUR-RESOURCE-NAME.openai.azure.com/openai/v1/" +export AZURE_OPENAI_DEPLOYMENT="your-deployment-name" +python src/app.py +python src/app.py --stream +``` + +Use `--prompt "What is a rainbow?"` to change the question or +`--model your-deployment-name` to override the deployment environment variable. +Missing Azure configuration and refused URLs return exit code 2 before tracing. + +## Code + +Instrument before creating the client. [src/app.py](src/app.py) contains the CLI +and the URL validation used here. + +```python +import os +from openai import OpenAI +from fi_instrumentation import register +from fi_instrumentation.fi_types import ProjectType +from traceai_openai import OpenAIInstrumentor +from app import check_base_url +base_url = check_base_url(os.environ["AZURE_OPENAI_BASE_URL"]) +provider = register(project_name="azure-ai-foundry", project_type=ProjectType.OBSERVE, + set_global_tracer_provider=False, verbose=False) +OpenAIInstrumentor().instrument(tracer_provider=provider) +client = OpenAI(api_key=os.environ["AZURE_OPENAI_API_KEY"], base_url=base_url) +response = client.chat.completions.create(model=os.environ["AZURE_OPENAI_DEPLOYMENT"], + messages=[{"role": "user", "content": "What is a rainbow?"}]) +print(response.choices[0].message.content) +provider.force_flush() +``` + +## What you see in Future AGI + +The fixture tests produce one LLM span per call: `ChatCompletion` for Chat +Completions and `Response` for the Responses API. The provider field is `openai` +(`gen_ai.provider.name`). This is the shared instrumentor's label for both Azure +host forms. The span does not say `azure`. + +For a successful non-streamed response, `gen_ai.request.model` is the model id +the provider returns in its response. The fixtures return the deployment name +that was requested. Azure's `model` request parameter is your deployment name; +the returned name can differ from an underlying catalog model id. + +Token usage is recorded only when the response has `usage`. Missing usage +attributes are omitted, never replaced with 0. The Chat Completions stream is +accumulated by the same instrumentor. Its default test stream has no usage +attributes. With `stream_options={"include_usage": True}` and a final usage +chunk, the instrumentor records input, output and total token counts. + +In the current `traceai-openai` version, streamed Chat Completions and failed +calls omit `gen_ai.request.model`. The requested deployment remains in the +`gen_ai.request.parameters` JSON unless `FI_HIDE_LLM_INVOCATION_PARAMETERS=true`. +Errors set the span status to ERROR and record an exception event. + +The non-streamed Responses fixture records the full parsed response as Python +dictionary text in `output.value`, including its output text. It also records +the response's model and usage, including the supplied reasoning and cache +token counts. Responses streaming is not tested here. + +## Provider specifics + +Microsoft documents both of these resource-specific base URL forms: + +```text +https://YOUR-RESOURCE-NAME.openai.azure.com/openai/v1/ +https://YOUR-RESOURCE-NAME.services.ai.azure.com/openai/v1/ +``` + +Both end in `/openai/v1/`. The SDK joins `chat/completions` or `responses` to +that path. If the slash is omitted after `/openai/v1`, the SDK appends it. +The example's validator returns allowed URLs exactly as supplied; it does not +rewrite them. Azure hosts must have the path `/openai/v1` or `/openai/v1/`. +Proxy hosts and loopback URLs are also allowed. + +`model` means your model deployment name, for example `my-gpt-deployment`, +not a model id. Read the API key from `AZURE_OPENAI_API_KEY`. +Microsoft recommends the Responses API for Azure OpenAI models. Chat +Completions remains available for models that support it on the same base URL: + +```python +response = client.responses.create( + model=os.environ["AZURE_OPENAI_DEPLOYMENT"], input="What is a rainbow?" +) +print(response.output_text) +provider.force_flush() +``` + +For Microsoft Entra ID, use Microsoft's token-provider sample with +`azure.identity` and the same `base_url` on the +[v1 API page](https://learn.microsoft.com/en-us/azure/foundry/openai/api-version-lifecycle). +This recipe does not implement token acquisition. Tracing uses the same +instrumentation because it is the same OpenAI client. + +The tests use these syntactic test hosts, not real Azure resources: + +```text +https://test-resource-0000.openai.azure.com/openai/v1/ +https://test-resource-0000.services.ai.azure.com/openai/v1/ +``` + +MockTransport handles those URLs in-process without DNS or an Azure connection. + +### Not this page: Foundry Agent Service + +Foundry agents are not this page. Agent Service, agent threads and evaluations +are separate surfaces. A project endpoint such as +`https://YOUR-RESOURCE-NAME.services.ai.azure.com/api/projects/YOUR-PROJECT-NAME` +is not a model inference base URL. + +`AzureAIOpenTelemetryTracer` exports to Application Insights. That setup does +not send agent spans to Future AGI through this recipe. This example does not +retarget that tracer or implement agent tracing. + +## Privacy + +Set `FI_HIDE_INPUTS=true` and/or `FI_HIDE_OUTPUTS=true` to keep prompt or response +text out of exported spans. The tests include visible control runs and verify +both flags. You can also pass `TraceConfig(hide_inputs=True)` as the instrumentor's +`config`. `FI_HIDE_LLM_INVOCATION_PARAMETERS=true` omits request parameters. + +Azure still receives the prompt. Future AGI masking does not change Azure's +own logging or data handling. + +## Limits / not covered + +- The `AzureOpenAI` client with `api-version`, including older deployment-scoped URLs. +- The `azure-ai-inference` SDK and its `/models` endpoint. +- Foundry agents, threads, evaluations and Application Insights export. +- A bare Azure resource root or the project endpoint as an inference base URL. +- Responses streaming, async calls and other SDK surfaces in this example's tests. +- Live Azure calls: not tested against the live provider (no paid call); the tests use a local fake of the OpenAI API. + +The recipe does not change the shared instrumentor's `openai` provider label. + +## Tests + +Run from the repository root. The suite uses MockTransport and local servers +on `127.0.0.1`. Child processes load a loopback guard that rejects other hosts +before DNS. Its negative control proves the guard is active. + +Repository source: + +```bash +env -u PYTHONPATH PYTHONDONTWRITEBYTECODE=1 \ + PYTHONPATH="python/examples/azure-ai-foundry/src:python:python/frameworks/openai:python/tests" \ + uv run --no-project --python 3.11 \ + --with 'openai==3.24.0' --with httpx --with 'wrapt<2' \ + --with opentelemetry-api --with opentelemetry-sdk --with opentelemetry-instrumentation \ + --with opentelemetry-exporter-otlp-proto-http --with requests --with protobuf \ + --with opentelemetry-proto --with pydantic --with jsonschema --with pytest \ + pytest python/examples/azure-ai-foundry/tests -q -p no:cacheprovider --noconftest -o addopts= -rfEs +``` + +Published packages: + +```bash +env -u PYTHONPATH PYTHONDONTWRITEBYTECODE=1 \ + PYTHONPATH="python/examples/azure-ai-foundry/src:python/tests" \ + uv run --no-project --python 3.11 \ + --with 'openai==3.24.0' --with 'traceAI-openai==0.1.10' --with 'fi-instrumentation-otel==1.1.0' \ + --with httpx --with 'wrapt<2' --with pytest \ + pytest python/examples/azure-ai-foundry/tests -q -p no:cacheprovider --noconftest -o addopts= -rfEs +``` + +For the floor row, change `openai==3.24.0` to `openai==1.69.0` in the repository-source command. + +| Python | `openai` | traceAI packages | Result | +|---|---|---|---| +| 3.10, 3.11, 3.12, 3.13 | 3.24.0 | `traceai-openai` and `fi_instrumentation` from this repository | Full suite | +| 3.11 | 1.69.0 (the `traceai-openai` floor) | from this repository | Full suite | +| 3.11 | 3.24.0 | published `traceAI-openai==0.1.10` and `fi-instrumentation-otel==1.1.0` from PyPI | Full suite | diff --git a/python/examples/azure-ai-foundry/requirements.txt b/python/examples/azure-ai-foundry/requirements.txt new file mode 100644 index 000000000..2e2d809c3 --- /dev/null +++ b/python/examples/azure-ai-foundry/requirements.txt @@ -0,0 +1,4 @@ +# Pinned recipe dependencies; see README for the compatibility test commands. +openai==3.24.0 +traceAI-openai==0.1.10 +fi-instrumentation-otel==1.1.0 diff --git a/python/examples/azure-ai-foundry/src/app.py b/python/examples/azure-ai-foundry/src/app.py new file mode 100644 index 000000000..771184c10 --- /dev/null +++ b/python/examples/azure-ai-foundry/src/app.py @@ -0,0 +1,111 @@ +"""Trace Azure OpenAI v1 Chat Completions with the OpenAI client.""" + +import argparse +import os +import sys +from urllib.parse import unquote, urlsplit + +import openai +from fi_instrumentation import register +from fi_instrumentation.fi_types import ProjectType +from traceai_openai import OpenAIInstrumentor + +DEFAULT_BASE_URL = None +DOCUMENTED_BASE_URL_FORMS = ( + "https://YOUR-RESOURCE-NAME.openai.azure.com/openai/v1/", + "https://YOUR-RESOURCE-NAME.services.ai.azure.com/openai/v1/", +) +API_KEY_ENV = "AZURE_OPENAI_API_KEY" +BASE_URL_ENV = "AZURE_OPENAI_BASE_URL" +MODEL_ENV = "AZURE_OPENAI_DEPLOYMENT" + + +def check_base_url(url: str) -> str: + if not url: + raise ValueError(f"Set {BASE_URL_ENV} to your resource's /openai/v1/ base URL.") + decoded = url + while (next_value := unquote(decoded)) != decoded: + decoded = next_value + if "your-resource-name" in decoded.lower() or "<" in decoded or ">" in decoded: + raise ValueError(f"Replace placeholders in {BASE_URL_ENV} with your resource name.") + try: + parsed = urlsplit(decoded) + host = (parsed.hostname or "").lower().rstrip(".") + except ValueError: + raise ValueError(f"Set {BASE_URL_ENV} to a valid HTTP(S) base URL.") from None + if parsed.scheme not in ("http", "https") or not host: + raise ValueError(f"Set {BASE_URL_ENV} to a valid HTTP(S) base URL.") + if host.endswith((".openai.azure.com", ".services.ai.azure.com", ".cognitiveservices.azure.com")): + if parsed.path not in ("/openai/v1", "/openai/v1/"): + if parsed.path.startswith("/api/projects"): + reason = "The project endpoint is for agents, not model inference." + elif parsed.path.startswith("/openai"): + reason = "Older AzureOpenAI deployment/api-version URLs are not covered." + elif parsed.path.startswith("/models"): + reason = "The azure-ai-inference SDK /models endpoint is not covered." + else: + reason = "A bare resource root or other Azure path is not an inference base URL." + raise ValueError(f"{reason} Set {BASE_URL_ENV} to your resource's /openai/v1/ URL.") + return url + + +def make_client(base_url: str | None = None, api_key: str | None = None, http_client=None) -> openai.OpenAI: + resolved_url = check_base_url(base_url if base_url is not None else os.environ.get(BASE_URL_ENV) or DEFAULT_BASE_URL) + key = api_key if api_key is not None else os.environ[API_KEY_ENV] + return openai.OpenAI(base_url=resolved_url, api_key=key, http_client=http_client) + + +def setup_tracing(project_name: str | None = None): + provider = register( + project_name=project_name or "azure-ai-foundry", + project_type=ProjectType.OBSERVE, + set_global_tracer_provider=False, + verbose=False, + ) + OpenAIInstrumentor().instrument(tracer_provider=provider) + return provider + + +def main(argv=None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--stream", action="store_true") + parser.add_argument("--prompt", default="What is a rainbow?") + parser.add_argument("--model", help="Your Azure model deployment name") + args = parser.parse_args(argv) + try: + base_url = check_base_url(os.environ.get(BASE_URL_ENV) or DEFAULT_BASE_URL) + model = args.model or os.environ[MODEL_ENV] + api_key = os.environ[API_KEY_ENV] + if not model: + raise ValueError(f"Set {MODEL_ENV} to your deployment name or use --model.") + if not api_key: + raise ValueError(f"Set {API_KEY_ENV} to your Azure API key.") + except KeyError as error: + print(f"Set {error.args[0]} before running this example.", file=sys.stderr) + return 2 + except ValueError as error: + print(str(error), file=sys.stderr) + return 2 + + provider = setup_tracing() + try: + with make_client(base_url=base_url, api_key=api_key) as client: + response = client.chat.completions.create( + model=model, + messages=[{"role": "user", "content": args.prompt}], + stream=args.stream, + ) + if args.stream: + for chunk in response: + if chunk.choices: + print(chunk.choices[0].delta.content or "", end="", flush=True) + print() + else: + print(response.choices[0].message.content or "") + finally: + provider.force_flush() + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/python/examples/azure-ai-foundry/tests/_fake_openai.py b/python/examples/azure-ai-foundry/tests/_fake_openai.py new file mode 100644 index 000000000..0c1058204 --- /dev/null +++ b/python/examples/azure-ai-foundry/tests/_fake_openai.py @@ -0,0 +1,114 @@ +"""OpenAI-shaped fixtures and a local server; no upstream requests.""" + +import copy +import json +import threading +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +ANSWER = "A rainbow is light split into colors." +CHAT_USAGE = {"prompt_tokens": 7, "completion_tokens": 9, "total_tokens": 16} +ERROR = {"error": {"code": "401", "message": "Access denied due to invalid subscription key or wrong API endpoint."}} + + +def chat_response(model, usage=True): + response = { + "id": "chatcmpl-fixture", "object": "chat.completion", "created": 1, + "model": model, + "choices": [{"index": 0, "message": {"role": "assistant", "content": ANSWER}, "finish_reason": "stop"}], + } + if usage: + response["usage"] = dict(CHAT_USAGE) + return response + + +def chat_stream(model, include_usage=False): + chunks = [] + for index, text in enumerate(("A rainbow is ", "light split into colors.")): + chunks.append({ + "id": "chatcmpl-fixture", "object": "chat.completion.chunk", "created": 1, + "model": model, + "choices": [{"index": 0, "delta": {"content": text, **({"role": "assistant"} if index == 0 else {})}, "finish_reason": None}], + }) + chunks.append({ + "id": "chatcmpl-fixture", "object": "chat.completion.chunk", "created": 1, + "model": model, "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}], + }) + if include_usage: + chunks.append({ + "id": "chatcmpl-fixture", "object": "chat.completion.chunk", "created": 1, + "model": model, "choices": [], "usage": dict(CHAT_USAGE), + }) + return ("".join("data: " + json.dumps(chunk) + "\n\n" for chunk in chunks) + "data: [DONE]\n\n").encode() + + +def responses_response(model): + return { + "id": "resp-fixture", "object": "response", "created_at": 1, + "status": "completed", "error": None, "incomplete_details": None, + "instructions": None, "model": model, + "output": [{ + "id": "msg-fixture", "type": "message", "role": "assistant", "status": "completed", + "content": [{"type": "output_text", "text": ANSWER, "annotations": []}], + }], + "parallel_tool_calls": True, "tools": [], "tool_choice": "auto", + "temperature": 1.0, "top_p": 1.0, + "usage": { + "input_tokens": 7, "output_tokens": 9, "total_tokens": 16, + "input_tokens_details": {"cached_tokens": 0}, + "output_tokens_details": {"reasoning_tokens": 0}, + }, + } + + +class FakeOpenAI: + def __init__(self): + self._requests = [] + self._lock = threading.Lock() + owner = self + + class Handler(BaseHTTPRequestHandler): + def do_POST(self): + body = json.loads(self.rfile.read(int(self.headers["Content-Length"]))) + with owner._lock: + owner._requests.append({ + "path": self.path, + "headers": {key.lower(): value for key, value in self.headers.items()}, + "body": body, + }) + if self.path not in ("/openai/v1/chat/completions", "/openai/v1/responses"): + self.send_error(404) + return + if body.get("model") == "error-deployment": + status, content_type, payload = 401, "application/json", json.dumps(ERROR).encode() + elif self.path.endswith("/responses"): + status, content_type, payload = 200, "application/json", json.dumps(responses_response(body["model"])).encode() + elif body.get("stream"): + include_usage = body.get("stream_options", {}).get("include_usage", False) + status, content_type, payload = 200, "text/event-stream", chat_stream(body["model"], include_usage) + else: + status, content_type, payload = 200, "application/json", json.dumps(chat_response(body["model"])).encode() + self.send_response(status) + self.send_header("Content-Type", content_type) + self.send_header("Content-Length", str(len(payload))) + self.end_headers() + self.wfile.write(payload) + + def log_message(self, _format, *args): + return + + self._server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + self.base_url = f"http://127.0.0.1:{self._server.server_port}/openai/v1/" + self._thread = threading.Thread(target=self._server.serve_forever, daemon=True) + + def __enter__(self): + self._thread.start() + return self + + def __exit__(self, *_args): + self._server.shutdown() + self._server.server_close() + self._thread.join() + + def requests(self): + with self._lock: + return copy.deepcopy(self._requests) diff --git a/python/examples/azure-ai-foundry/tests/loopback_guard/sitecustomize.py b/python/examples/azure-ai-foundry/tests/loopback_guard/sitecustomize.py new file mode 100644 index 000000000..4de633cb7 --- /dev/null +++ b/python/examples/azure-ai-foundry/tests/loopback_guard/sitecustomize.py @@ -0,0 +1,47 @@ +"""Refuse non-loopback networking before DNS; loaded by child Python processes.""" + +import os +import socket +from pathlib import Path + + +def require_loopback(host): + if isinstance(host, bytes): + host = host.decode("ascii", errors="replace") + if host not in ("127.0.0.1", "localhost"): + message = f"Loopback guard refused host: {host!r}" + log = os.environ.get("LOOPBACK_GUARD_LOG") + if log: + with open(log, "a", encoding="utf-8") as output: + output.write(message + "\n") + raise RuntimeError(message) + + +_getaddrinfo = socket.getaddrinfo +_connect = socket.socket.connect +_connect_ex = socket.socket.connect_ex + + +def guarded_getaddrinfo(host, *args, **kwargs): + require_loopback(host) + return _getaddrinfo(host, *args, **kwargs) + + +def guarded_connect(sock, address): + require_loopback(address[0]) + return _connect(sock, address) + + +def guarded_connect_ex(sock, address): + require_loopback(address[0]) + return _connect_ex(sock, address) + + +socket.getaddrinfo = guarded_getaddrinfo +socket.socket.connect = guarded_connect +socket.socket.connect_ex = guarded_connect_ex + +if ready := os.environ.get("LOOPBACK_GUARD_READY"): + Path(ready).write_text("loopback guard installed\n", encoding="utf-8") +if log := os.environ.get("LOOPBACK_GUARD_LOG"): + Path(log).touch() diff --git a/python/examples/azure-ai-foundry/tests/test_azure_ai_foundry_recipe.py b/python/examples/azure-ai-foundry/tests/test_azure_ai_foundry_recipe.py new file mode 100644 index 000000000..26337beeb --- /dev/null +++ b/python/examples/azure-ai-foundry/tests/test_azure_ai_foundry_recipe.py @@ -0,0 +1,504 @@ +"""Fixture-only Azure v1 recipe contract; syntactic test hosts are not real hosts.""" + +import json +import os +import re +import socket +import subprocess +import sys +import uuid +from contextlib import contextmanager +from pathlib import Path + +import app +import fi_instrumentation +import httpx +import openai +import pytest +import traceai_openai +from harness import Receiver +from traceai_openai import OpenAIInstrumentor + +from _fake_openai import ( + ANSWER, ERROR, FakeOpenAI, chat_response, chat_stream, responses_response, +) + +RECIPE = Path(__file__).resolve().parents[1] +# Syntactic test hosts, not real Azure resources; MockTransport never resolves them. +TEST_BASE_URLS = ( + "https://test-resource-0000.openai.azure.com/openai/v1/", + "https://test-resource-0000.services.ai.azure.com/openai/v1/", +) +MODEL = "my-gpt-deployment" +VENDOR_KEY = "placeholder-azure-key" +FI_KEY = "placeholder-futureagi-key" +FI_SECRET = "placeholder-futureagi-secret" +USAGE = { + "gen_ai.usage.input_tokens": 7, + "gen_ai.usage.output_tokens": 9, + "gen_ai.usage.total_tokens": 16, +} +SOURCE_COMMAND = """env -u PYTHONPATH PYTHONDONTWRITEBYTECODE=1 \\ + PYTHONPATH="python/examples/azure-ai-foundry/src:python:python/frameworks/openai:python/tests" \\ + uv run --no-project --python 3.11 \\ + --with 'openai==3.24.0' --with httpx --with 'wrapt<2' \\ + --with opentelemetry-api --with opentelemetry-sdk --with opentelemetry-instrumentation \\ + --with opentelemetry-exporter-otlp-proto-http --with requests --with protobuf \\ + --with opentelemetry-proto --with pydantic --with jsonschema --with pytest \\ + pytest python/examples/azure-ai-foundry/tests -q -p no:cacheprovider --noconftest -o addopts= -rfEs""" +PUBLISHED_COMMAND = """env -u PYTHONPATH PYTHONDONTWRITEBYTECODE=1 \\ + PYTHONPATH="python/examples/azure-ai-foundry/src:python/tests" \\ + uv run --no-project --python 3.11 \\ + --with 'openai==3.24.0' --with 'traceAI-openai==0.1.10' --with 'fi-instrumentation-otel==1.1.0' \\ + --with httpx --with 'wrapt<2' --with pytest \\ + pytest python/examples/azure-ai-foundry/tests -q -p no:cacheprovider --noconftest -o addopts= -rfEs""" + + +@pytest.fixture(autouse=True) +def isolate_environment_and_network(monkeypatch): + # No inherited credentials, hiding flags, collector settings or proxy routes. + for name in tuple(os.environ): + if name.startswith(("FI_", "OTEL_", "OPENAI_", "AZURE_")) or name.lower().endswith("_proxy"): + monkeypatch.delenv(name) + monkeypatch.setenv("FI_API_KEY", FI_KEY) + monkeypatch.setenv("FI_SECRET_KEY", FI_SECRET) + monkeypatch.setenv(app.API_KEY_ENV, VENDOR_KEY) + monkeypatch.setenv(app.BASE_URL_ENV, TEST_BASE_URLS[0]) + monkeypatch.setenv(app.MODEL_ENV, MODEL) + monkeypatch.setenv("NO_PROXY", "*") + original_dns = socket.getaddrinfo + original_connect = socket.socket.connect + original_connect_ex = socket.socket.connect_ex + + def require_loopback(host): + if host not in ("127.0.0.1", "localhost", b"127.0.0.1", b"localhost"): + raise RuntimeError("Tests refuse non-loopback networking before DNS") + + def guarded_dns(host, *args, **kwargs): + require_loopback(host) + return original_dns(host, *args, **kwargs) + + def guarded_connect(sock, address): + require_loopback(address[0]) + return original_connect(sock, address) + + def guarded_connect_ex(sock, address): + require_loopback(address[0]) + return original_connect_ex(sock, address) + + monkeypatch.setattr(socket, "getaddrinfo", guarded_dns) + monkeypatch.setattr(socket.socket, "connect", guarded_connect) + monkeypatch.setattr(socket.socket, "connect_ex", guarded_connect_ex) + + +@pytest.fixture +def receiver(monkeypatch): + with Receiver() as receiver: + monkeypatch.setenv("FI_BASE_URL", receiver.origin) + yield receiver + + +@pytest.fixture +def tracing(): + @contextmanager + def start(): + project = "azure-recipe-" + uuid.uuid4().hex + provider = app.setup_tracing(project) + try: + yield provider, project + finally: + provider.force_flush() + OpenAIInstrumentor().uninstrument() + provider.shutdown() + return start + + +def attributes(span): + return {item["key"]: next(iter(item["value"].values())) for item in span["attributes"]} + + +def assert_usage(attrs, expected): + actual = {key: int(value) for key, value in attrs.items() if key.startswith("gen_ai.usage.")} + assert actual == expected + + +def assert_export(receiver, project, name="ChatCompletion"): + spans, exports = receiver.spans(), receiver.requests() + assert len(spans) == 1 + assert len(exports) == 1 + span = spans[0] + attrs = attributes(span) + assert span["name"] == name + assert attrs["gen_ai.span.kind"] == "LLM" + assert attrs["gen_ai.provider.name"] == "openai" + export = exports[0] + assert export["path"] == "/tracer/v1/traces" + assert "x-api-key" in export["headers"] and "x-secret-key" in export["headers"] + assert export["headers"]["x-api-key"] == FI_KEY + assert export["headers"]["x-secret-key"] == FI_SECRET + assert export["resource_attributes"][0]["project_name"] == project + assert export["resource_attributes"][0]["project_type"] == "observe" + # The populated span/export and bearer assertion at each call site are controls. + assert VENDOR_KEY not in json.dumps({"spans": spans, "exports": exports}) + return span, attrs + + +def assert_vendor_request(request, base_url, suffix, model=MODEL): + assert str(request.url) == base_url + suffix + assert request.headers["authorization"] == "Bearer " + VENDOR_KEY + assert "x-api-key" not in request.headers and "x-secret-key" not in request.headers + assert FI_KEY not in str(request.headers) and FI_SECRET not in str(request.headers) + assert json.loads(request.content)["model"] == model + + +def mock_client(base_url, payload, requests, status=200): + def handler(request): + requests.append(request) + if isinstance(payload, bytes): + return httpx.Response(status, content=payload, headers={"Content-Type": "text/event-stream"}) + return httpx.Response(status, json=payload) + return app.make_client(base_url=base_url, http_client=httpx.Client(transport=httpx.MockTransport(handler))) + + +def complete(client, prompt="What is a rainbow?", **kwargs): + return client.chat.completions.create(model=MODEL, messages=[{"role": "user", "content": prompt}], **kwargs) + + +def test_documented_forms_require_resource_and_base_url(monkeypatch): + assert app.DEFAULT_BASE_URL is None + assert app.DOCUMENTED_BASE_URL_FORMS == ( + "https://YOUR-RESOURCE-NAME.openai.azure.com/openai/v1/", + "https://YOUR-RESOURCE-NAME.services.ai.azure.com/openai/v1/", + ) + readme = (RECIPE / "README.md").read_text() + for form in app.DOCUMENTED_BASE_URL_FORMS: + assert form in readme + with pytest.raises(ValueError, match="placeholders"): + app.check_base_url(form) + monkeypatch.delenv(app.BASE_URL_ENV) + with pytest.raises(ValueError, match=app.BASE_URL_ENV): + app.make_client() + + +def test_sdk_appends_trailing_slash_without_validator_rewriting(): + supplied = TEST_BASE_URLS[0].rstrip("/") + assert app.check_base_url(supplied) == supplied + with app.make_client(base_url=supplied) as client: + assert str(client.base_url) == TEST_BASE_URLS[0] + + +@pytest.mark.parametrize("base_url", TEST_BASE_URLS) +def test_chat_both_host_forms_and_key_separation(base_url, receiver, tracing, monkeypatch): + monkeypatch.setenv(app.BASE_URL_ENV, base_url) + requests = [] + with tracing() as (provider, project): + with mock_client(base_url, chat_response(MODEL), requests) as client: + assert str(client.base_url) == base_url + assert complete(client).choices[0].message.content == ANSWER + # The environment default is identical to the explicit client base URL. + with app.make_client() as default_client: + assert str(default_client.base_url) == base_url + provider.force_flush() + span, attrs = assert_export(receiver, project) + assert len(requests) == 1 + assert_vendor_request(requests[0], base_url, "chat/completions") + assert attrs["gen_ai.request.model"] == MODEL + assert ANSWER in attrs["output.value"] + assert span["status"]["code"] == "STATUS_CODE_OK" + assert_usage(attrs, USAGE) + + +def test_usage_absent_is_omitted(receiver, tracing): + requests = [] + with tracing() as (provider, project): + with mock_client(TEST_BASE_URLS[0], chat_response(MODEL, usage=False), requests) as client: + assert complete(client).choices[0].message.content == ANSWER + provider.force_flush() + _, attrs = assert_export(receiver, project) + assert_vendor_request(requests[0], TEST_BASE_URLS[0], "chat/completions") + assert attrs["gen_ai.request.model"] == MODEL + assert_usage(attrs, {}) + # The same fixture's normal usage is verified in test_chat_both_host_forms_and_key_separation. + + +@pytest.mark.parametrize("include_usage", [False, True], ids=["default", "include-usage"]) +def test_stream_output_model_gap_and_usage(include_usage, receiver, tracing): + requests = [] + with tracing() as (provider, project): + with mock_client(TEST_BASE_URLS[0], chat_stream(MODEL, include_usage), requests) as client: + options = {"stream_options": {"include_usage": True}} if include_usage else {} + text = "".join(chunk.choices[0].delta.content or "" for chunk in complete(client, stream=True, **options) if chunk.choices) + assert text == ANSWER + provider.force_flush() + span, attrs = assert_export(receiver, project) + assert_vendor_request(requests[0], TEST_BASE_URLS[0], "chat/completions") + params = json.loads(attrs["gen_ai.request.parameters"]) + assert params["model"] == MODEL and params["stream"] is True + assert json.loads(requests[0].content).get("stream_options") == ({"include_usage": True} if include_usage else None) + # traceai-openai records the model only from a non-streamed response; flip this when the instrumentor records the request model + assert "gen_ai.request.model" not in attrs + assert attrs["output.value"] == ANSWER + assert span["status"]["code"] == "STATUS_CODE_OK" + assert_usage(attrs, USAGE if include_usage else {}) + + +def test_authentication_error_records_exception_and_model_gap(receiver, tracing): + requests = [] + with tracing() as (provider, project): + with mock_client(TEST_BASE_URLS[0], ERROR, requests, status=401) as client: + with pytest.raises(openai.AuthenticationError) as error: + complete(client) + provider.force_flush() + span, attrs = assert_export(receiver, project) + assert len(requests) == 1 + assert_vendor_request(requests[0], TEST_BASE_URLS[0], "chat/completions") + assert span["status"]["code"] == "STATUS_CODE_ERROR" + assert "invalid subscription key" in str(error.value) + assert any(event["name"] == "exception" for event in span["events"]) + assert "AuthenticationError" in json.dumps(span["events"]) + assert "invalid subscription key" in json.dumps(span) + # traceai-openai records the model only from a non-streamed response; flip this when the instrumentor records the request model + assert "gen_ai.request.model" not in attrs + assert json.loads(attrs["gen_ai.request.parameters"])["model"] == MODEL + assert_usage(attrs, {}) + assert VENDOR_KEY not in str(error.value) + + +@pytest.mark.parametrize("flag,marker", [("FI_HIDE_INPUTS", "unique-private-prompt-marker"), ("FI_HIDE_OUTPUTS", ANSWER)]) +def test_privacy_hiding_has_visible_control(flag, marker, receiver, tracing, monkeypatch): + for hidden in (False, True): + receiver.clear() + monkeypatch.setenv(flag, str(hidden).lower()) + requests = [] + with tracing() as (provider, project): + with mock_client(TEST_BASE_URLS[0], chat_response(MODEL), requests) as client: + assert complete(client, prompt="unique-private-prompt-marker").choices[0].message.content == ANSWER + provider.force_flush() + assert_export(receiver, project) + assert_vendor_request(requests[0], TEST_BASE_URLS[0], "chat/completions") + assert json.loads(requests[0].content)["messages"][0]["content"] == "unique-private-prompt-marker" + assert (marker in json.dumps(receiver.spans())) is (not hidden) + + +def test_responses_api_span_and_exact_url(receiver, tracing): + requests = [] + with tracing() as (provider, project): + with mock_client(TEST_BASE_URLS[0], responses_response(MODEL), requests) as client: + response = client.responses.create(model=MODEL, input="What is a rainbow?") + assert response.output_text == ANSWER + provider.force_flush() + span, attrs = assert_export(receiver, project, name="Response") + assert len(requests) == 1 + assert_vendor_request(requests[0], TEST_BASE_URLS[0], "responses") + assert requests[0].url.path == "/openai/v1/responses" + assert json.loads(requests[0].content)["input"] == "What is a rainbow?" + assert attrs["gen_ai.request.model"] == MODEL + # Non-streamed Responses records the full parsed response's Python dict text. + assert attrs["output.value"] == str(response.model_dump()) + assert response.model_dump()["output"][0]["content"][0]["text"] == ANSWER + assert span["status"]["code"] == "STATUS_CODE_OK" + assert_usage(attrs, {**USAGE, "gen_ai.usage.output_tokens.reasoning": 0, "gen_ai.usage.input_tokens.cache_read": 0}) + + +WRONG_PATHS = ( + ("/api/projects/my-project", "project endpoint"), + ("/openai/deployments/my-gpt-deployment", "AzureOpenAI"), + ("/openai?api-version=2024-01-01", "AzureOpenAI"), + ("/", "bare resource root"), + ("", "bare resource root"), + ("/models", "azure-ai-inference"), +) +AZURE_HOSTS = ( + "test-resource-0000.openai.azure.com", + "TEST-RESOURCE-0000.SERVICES.AI.AZURE.COM", + "test-resource-0000.cognitiveservices.azure.com.", + "TEST-RESOURCE-0000.OPENAI.AZURE.COM.", +) + + +@pytest.mark.parametrize("host", AZURE_HOSTS) +@pytest.mark.parametrize("path,reason", WRONG_PATHS) +def test_refuse_wrong_azure_surfaces(host, path, reason): + with pytest.raises(ValueError, match=reason) as error: + app.check_base_url("https://" + host + path) + assert app.BASE_URL_ENV in str(error.value) + assert "\n" not in str(error.value) + + +@pytest.mark.parametrize("url", [ + *app.DOCUMENTED_BASE_URL_FORMS, + "https://your-resource-name.OPENAI.AZURE.COM/openai/v1/", + "https://.services.ai.azure.com/openai/v1/", + "https://%3Cresource%3E.openai.azure.com/openai/v1/", + "https://%253Cresource%253E.openai.azure.com/openai/v1/", + "https://%59OUR-RESOURCE-NAME.openai.azure.com/openai/v1/", + "https://%2559OUR-RESOURCE-NAME.openai.azure.com/openai/v1/", + "https://test-resource-0000.services.ai.azure.com/api/projects/", + "https://test-resource-0000.openai.azure.com/openai/deployments/", +]) +def test_refuse_literal_encoded_and_double_encoded_placeholders(url): + with pytest.raises(ValueError, match="Replace placeholders"): + app.check_base_url(url) + + +@pytest.mark.parametrize("url", [ + *TEST_BASE_URLS, + "https://test-resource-0000.openai.azure.com/openai/v1", + "https://TEST-RESOURCE-0000.OPENAI.AZURE.COM./openai/v1/", + "https://test-resource-0000.services.ai.azure.com./openai/v1/", + "https://test-resource-0000.cognitiveservices.azure.com/openai/v1/", + "https://proxy.example/custom/inference/", + "https://proxy.example/models", + "http://127.0.0.1:12345/openai/v1/", + "http://localhost:12345/custom/", +]) +def test_allowed_urls_are_returned_unchanged(url): + assert app.check_base_url(url) == url + + +@pytest.mark.parametrize("missing", [app.BASE_URL_ENV, app.MODEL_ENV, app.API_KEY_ENV]) +def test_missing_configuration_exits_before_tracing(missing, receiver, monkeypatch, capsys): + monkeypatch.delenv(missing) + monkeypatch.setattr(app, "setup_tracing", lambda *args, **kwargs: pytest.fail("Tracing ran before validation")) + assert app.main([]) == 2 + assert missing in capsys.readouterr().err + assert receiver.spans() == [] and receiver.requests() == [] + if missing == app.API_KEY_ENV: + with pytest.raises(KeyError, match=app.API_KEY_ENV): + app.make_client() + + +def test_refused_url_main_exits_without_requests_or_spans(receiver, monkeypatch, capsys): + with FakeOpenAI() as fake: + refused = "https://TEST-RESOURCE-0000.SERVICES.AI.AZURE.COM./api/projects/my-project" + monkeypatch.setenv(app.BASE_URL_ENV, refused) + monkeypatch.setattr(app, "setup_tracing", lambda *args, **kwargs: pytest.fail("Tracing ran for refused URL")) + monkeypatch.setattr(app, "make_client", lambda *args, **kwargs: pytest.fail("Client created for refused URL")) + assert app.main([]) == 2 + assert "project endpoint" in capsys.readouterr().err + assert fake.requests() == [] + assert receiver.spans() == [] and receiver.requests() == [] + + +def child_environment(tmp_path, receiver, base_url, project): + # Resolve actual imported packages so this also works with published wheels. + python_paths = [ + RECIPE / "tests" / "loopback_guard", RECIPE / "src", + Path(fi_instrumentation.__file__).resolve().parent.parent, + Path(traceai_openai.__file__).resolve().parent.parent, + Path(sys.modules["harness"].__file__).resolve().parent.parent, + ] + return { + "PATH": os.defpath, + "PYTHONDONTWRITEBYTECODE": "1", + "PYTHONPATH": os.pathsep.join(dict.fromkeys(str(path) for path in python_paths)), + "NO_PROXY": "*", + "FI_API_KEY": FI_KEY, "FI_SECRET_KEY": FI_SECRET, + "FI_BASE_URL": receiver.origin, "FI_PROJECT_NAME": project, + app.API_KEY_ENV: VENDOR_KEY, app.BASE_URL_ENV: base_url, app.MODEL_ENV: MODEL, + "LOOPBACK_GUARD_READY": str(tmp_path / "guard-ready"), + "LOOPBACK_GUARD_LOG": str(tmp_path / "guard.log"), + } + + +def assert_child_guard(env): + assert Path(env["LOOPBACK_GUARD_READY"]).read_text() == "loopback guard installed\n" + assert Path(env["LOOPBACK_GUARD_LOG"]).read_text() == "" + + +@pytest.mark.parametrize("mode", ["chat", "stream", "model-override", "responses"]) +def test_loopback_subprocess_journeys(mode, tmp_path, receiver): + with FakeOpenAI() as fake: + env = child_environment(tmp_path, receiver, fake.base_url, "azure-subprocess-" + uuid.uuid4().hex) + model = MODEL + if mode == "responses": + project = env["FI_PROJECT_NAME"] + command = [sys.executable, "-c", "import os, app; p = app.setup_tracing(os.environ['FI_PROJECT_NAME']); c = app.make_client(); print(c.responses.create(model=os.environ[app.MODEL_ENV], input='What is a rainbow?').output_text); c.close(); p.force_flush()"] + else: + project = "azure-ai-foundry" + command = [sys.executable, str(RECIPE / "src" / "app.py")] + if mode == "stream": + command.append("--stream") + elif mode == "model-override": + model = "override-deployment" + env.pop(app.MODEL_ENV) + command.extend(["--model", model]) + result = subprocess.run(command, env=env, capture_output=True, text=True, timeout=30) + assert result.returncode == 0, result.stderr + assert result.stdout.strip() == ANSWER + assert VENDOR_KEY not in result.stdout + result.stderr + assert_child_guard(env) + _, attrs = assert_export(receiver, project, name="Response" if mode == "responses" else "ChatCompletion") + requests = fake.requests() + assert len(requests) == 1 + assert requests[0]["path"] == ("/openai/v1/responses" if mode == "responses" else "/openai/v1/chat/completions") + assert requests[0]["headers"]["authorization"] == "Bearer " + VENDOR_KEY + assert "x-api-key" not in requests[0]["headers"] and "x-secret-key" not in requests[0]["headers"] + assert requests[0]["body"]["model"] == model + assert ANSWER in attrs["output.value"] + if mode == "stream": + # traceai-openai records the model only from a non-streamed response; flip this when the instrumentor records the request model + assert "gen_ai.request.model" not in attrs + assert json.loads(attrs["gen_ai.request.parameters"])["model"] == model + assert_usage(attrs, {}) + else: + assert attrs["gen_ai.request.model"] == model + + +def test_guard_refuses_before_dns_with_loopback_control(tmp_path, receiver): + env = child_environment(tmp_path, receiver, TEST_BASE_URLS[0], "guard-control") + host = "test-resource-0000.openai.azure.com" + code = """import socket, sys, sitecustomize +from pathlib import Path +assert Path(sys.argv[1]).is_file() +assert socket.getaddrinfo('localhost', 80) +with socket.create_connection(('127.0.0.1', int(sys.argv[2])), timeout=5): + pass +def must_not_reach_network(*args, **kwargs): + raise AssertionError('Non-loopback operation reached DNS or connect') +sitecustomize._getaddrinfo = must_not_reach_network +sitecustomize._connect = must_not_reach_network +sitecustomize._connect_ex = must_not_reach_network +for operation in (lambda: socket.getaddrinfo(sys.argv[3], 443), + lambda: socket.socket().connect((sys.argv[3], 443)), + lambda: socket.socket().connect_ex((sys.argv[3], 443))): + try: + operation() + except RuntimeError as error: + assert 'Loopback guard refused' in str(error) + else: + raise AssertionError('Guard permitted non-loopback host') +print('guard controls passed') +""" + result = subprocess.run([sys.executable, "-c", code, env["LOOPBACK_GUARD_READY"], receiver.origin.rsplit(":", 1)[1], host], env=env, capture_output=True, text=True, timeout=15) + assert result.returncode == 0, result.stderr + assert result.stdout.strip() == "guard controls passed" + log = Path(env["LOOPBACK_GUARD_LOG"]).read_text().splitlines() + assert len(log) == 3 and all(host in line and "refused" in line for line in log) + assert receiver.requests() == [] + + +def test_readme_and_requirement_pins(): + readme = (RECIPE / "README.md").read_text() + for value in (*app.DOCUMENTED_BASE_URL_FORMS, app.API_KEY_ENV, app.BASE_URL_ENV, app.MODEL_ENV, "FI_API_KEY", "FI_SECRET_KEY", "FI_BASE_URL", "FI_HIDE_INPUTS", "FI_HIDE_OUTPUTS"): + assert value in readme + assert "model is your deployment name" in readme + assert "provider field is `openai`" in readme + assert "Not this page: Foundry Agent Service" in readme + assert "agents are not this page" in readme + assert "AzureAIOpenTelemetryTracer" in readme and "Application Insights" in readme + assert SOURCE_COMMAND in readme and PUBLISHED_COMMAND in readme + assert "https://learn.microsoft.com/en-us/azure/foundry/openai/api-version-lifecycle" in readme + assert "syntactic test hosts" in readme and "not real" in readme + assert "gen_ai.request.model" in readme and "TBD" not in readme + assert "published `traceAI-openai==0.1.10` and `fi-instrumentation-otel==1.1.0`" in readme + exports = dict(re.findall(r'^export (\w+)="([^"]*)"$', readme, re.MULTILINE)) + assert exports["FI_API_KEY"] == FI_KEY + assert exports["FI_SECRET_KEY"] == FI_SECRET + assert exports[app.API_KEY_ENV] == VENDOR_KEY + assert not re.search(r"\b[A-Z]{2,5}-\d{3,}\b", readme), "internal ticket ids do not belong in the README" + for forbidden in ("prepared environment", "sk-", "all models", "every call"): + assert forbidden not in readme + assert (RECIPE / "requirements.txt").read_text().splitlines()[1:] == [ + "openai==3.24.0", "traceAI-openai==0.1.10", "fi-instrumentation-otel==1.1.0", + ] From 93b36ead22431608545bd39e5e2b3e86690eefa9 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Mon, 5 Oct 2026 20:31:22 -0700 Subject: [PATCH 05/10] fix(examples/azure-ai-foundry): review-r1 P3 hardening and README accuracy From pr-reviewer t_cd20cd80 (APPROVED, P3 only), fixed test-first: - R1: a query string or fragment on an Azure v1 URL is refused (the SDK would fold the path into the query and miss /chat/completions). - R2: structural checks run on the URL as given; only the placeholder search decodes. Percent-encoded Azure paths are refused. - R3: Azure hosts must use https (no cleartext key), and credentials in the URL are refused for every host (a userinfo URL that only looks like Azure would otherwise be treated as a proxy). - R4: README scopes the missing-model statement to streamed and failed Chat Completions calls; Responses calls record the model from the request. - R5: README says Azure may return the underlying model name rather than the deployment name, where each name ends up, and the cost-lookup caveat. - R6: removed the vacuous fake-server assertion from the refused-URL main() test (the setup_tracing/make_client traps are the controls). - R7: README states Python 3.10+; app.py adds postponed annotations. - R8 (shared traceai-openai behaviour): README notes that non-streamed Responses output.value can echo request fields such as instructions, which FI_HIDE_INPUTS does not cover. RED before the change: 10 failed (fix-r1-red); GREEN after: 74 passed. approved: Nikhil 2026-10-03 blanket Refs: TH-8317 --- python/examples/azure-ai-foundry/README.md | 20 ++++++-- python/examples/azure-ai-foundry/src/app.py | 13 +++++- .../tests/test_azure_ai_foundry_recipe.py | 46 +++++++++++++++---- 3 files changed, 65 insertions(+), 14 deletions(-) diff --git a/python/examples/azure-ai-foundry/README.md b/python/examples/azure-ai-foundry/README.md index 0fedd3c25..16648d31e 100644 --- a/python/examples/azure-ai-foundry/README.md +++ b/python/examples/azure-ai-foundry/README.md @@ -5,6 +5,8 @@ It uses `traceai-openai`, with no separate provider package. ## Install +Requires Python 3.10 or later. + From this example directory, install the pinned dependencies: ```bash @@ -84,8 +86,10 @@ host forms. The span does not say `azure`. For a successful non-streamed response, `gen_ai.request.model` is the model id the provider returns in its response. The fixtures return the deployment name -that was requested. Azure's `model` request parameter is your deployment name; -the returned name can differ from an underlying catalog model id. +that was requested. Azure's `model` request parameter is your deployment name. +Azure may return the underlying model name rather than your deployment name; +`gen_ai.request.model` then shows the returned name, and your deployment name is +in `gen_ai.request.parameters`. Cost lookup keyed on one name may miss the other. Token usage is recorded only when the response has `usage`. Missing usage attributes are omitted, never replaced with 0. The Chat Completions stream is @@ -93,8 +97,8 @@ accumulated by the same instrumentor. Its default test stream has no usage attributes. With `stream_options={"include_usage": True}` and a final usage chunk, the instrumentor records input, output and total token counts. -In the current `traceai-openai` version, streamed Chat Completions and failed -calls omit `gen_ai.request.model`. The requested deployment remains in the +In the current `traceai-openai` version, streamed and failed Chat Completions calls omit `gen_ai.request.model`; +Responses calls record it from the request. The requested deployment remains in the `gen_ai.request.parameters` JSON unless `FI_HIDE_LLM_INVOCATION_PARAMETERS=true`. Errors set the span status to ERROR and record an exception event. @@ -115,7 +119,9 @@ https://YOUR-RESOURCE-NAME.services.ai.azure.com/openai/v1/ Both end in `/openai/v1/`. The SDK joins `chat/completions` or `responses` to that path. If the slash is omitted after `/openai/v1`, the SDK appends it. The example's validator returns allowed URLs exactly as supplied; it does not -rewrite them. Azure hosts must have the path `/openai/v1` or `/openai/v1/`. +rewrite them. Azure hosts must use `https://` and have the path `/openai/v1` or +`/openai/v1/`, with no query string (the v1 API needs no `api-version`), no +fragment and no percent-encoded path. Credentials in the URL are refused. Proxy hosts and loopback URLs are also allowed. `model` means your model deployment name, for example `my-gpt-deployment`, @@ -164,6 +170,10 @@ text out of exported spans. The tests include visible control runs and verify both flags. You can also pass `TraceConfig(hide_inputs=True)` as the instrumentor's `config`. `FI_HIDE_LLM_INVOCATION_PARAMETERS=true` omits request parameters. +On non-streamed Responses calls, `output.value` holds the whole response object, +which can echo request fields such as `instructions`. `FI_HIDE_INPUTS` does not cover +that field; set `FI_HIDE_OUTPUTS=true` as well if those must stay out of the export. + Azure still receives the prompt. Future AGI masking does not change Azure's own logging or data handling. diff --git a/python/examples/azure-ai-foundry/src/app.py b/python/examples/azure-ai-foundry/src/app.py index 771184c10..f1d3ddd52 100644 --- a/python/examples/azure-ai-foundry/src/app.py +++ b/python/examples/azure-ai-foundry/src/app.py @@ -1,5 +1,7 @@ """Trace Azure OpenAI v1 Chat Completions with the OpenAI client.""" +from __future__ import annotations + import argparse import os import sys @@ -29,13 +31,20 @@ def check_base_url(url: str) -> str: if "your-resource-name" in decoded.lower() or "<" in decoded or ">" in decoded: raise ValueError(f"Replace placeholders in {BASE_URL_ENV} with your resource name.") try: - parsed = urlsplit(decoded) + # Structural checks use the URL as given; only the placeholder check decodes. + parsed = urlsplit(url) host = (parsed.hostname or "").lower().rstrip(".") except ValueError: raise ValueError(f"Set {BASE_URL_ENV} to a valid HTTP(S) base URL.") from None if parsed.scheme not in ("http", "https") or not host: raise ValueError(f"Set {BASE_URL_ENV} to a valid HTTP(S) base URL.") + if parsed.username is not None or parsed.password is not None: + raise ValueError(f"Do not put credentials in {BASE_URL_ENV}.") if host.endswith((".openai.azure.com", ".services.ai.azure.com", ".cognitiveservices.azure.com")): + if parsed.scheme != "https": + raise ValueError(f"Azure hosts must use https; set {BASE_URL_ENV} to your resource's /openai/v1/ URL.") + if "%" in parsed.path: + raise ValueError(f"Percent-encoded paths are not accepted for Azure hosts in {BASE_URL_ENV}.") if parsed.path not in ("/openai/v1", "/openai/v1/"): if parsed.path.startswith("/api/projects"): reason = "The project endpoint is for agents, not model inference." @@ -46,6 +55,8 @@ def check_base_url(url: str) -> str: else: reason = "A bare resource root or other Azure path is not an inference base URL." raise ValueError(f"{reason} Set {BASE_URL_ENV} to your resource's /openai/v1/ URL.") + if parsed.query or parsed.fragment: + raise ValueError(f"Remove the query string or fragment from {BASE_URL_ENV}; the v1 API needs no api-version.") return url diff --git a/python/examples/azure-ai-foundry/tests/test_azure_ai_foundry_recipe.py b/python/examples/azure-ai-foundry/tests/test_azure_ai_foundry_recipe.py index 26337beeb..9cd332ab7 100644 --- a/python/examples/azure-ai-foundry/tests/test_azure_ai_foundry_recipe.py +++ b/python/examples/azure-ai-foundry/tests/test_azure_ai_foundry_recipe.py @@ -316,6 +316,31 @@ def test_responses_api_span_and_exact_url(receiver, tracing): ) +HARDENING_REFUSALS = ( + # The SDK folds the path into a query string, so the request would miss /chat/completions. + ("https://test-resource-0000.openai.azure.com/openai/v1/?api-version=preview", "query string"), + ("https://test-resource-0000.openai.azure.com/openai/v1?api-version=preview", "query string"), + ("https://test-resource-0000.openai.azure.com/openai/v1/#frag", "query string"), + # Encoded paths would pass a decoded check but go on the wire as a different path. + ("https://test-resource-0000.openai.azure.com/openai%2Fv1/", "Percent-encoded paths"), + ("https://test-resource-0000.openai.azure.com/openai/v1%3F/api/projects/x", "Percent-encoded paths"), + # Plain http would send the Azure key in cleartext. + ("http://test-resource-0000.openai.azure.com/openai/v1/", "must use https"), + ("http://test-resource-0000.services.ai.azure.com/openai/v1/", "must use https"), + # Credentials in the URL; this one only looks like Azure and would go to another host. + ("https://test-resource-0000.openai.azure.com@proxy.example/openai/v1/", "credentials"), + ("https://user:pass@test-resource-0000.openai.azure.com/openai/v1/", "credentials"), +) + + +@pytest.mark.parametrize("url,reason", HARDENING_REFUSALS) +def test_refuse_query_encoded_http_and_credentials(url, reason): + with pytest.raises(ValueError, match=reason) as error: + app.check_base_url(url) + assert app.BASE_URL_ENV in str(error.value) + assert "\n" not in str(error.value) + + @pytest.mark.parametrize("host", AZURE_HOSTS) @pytest.mark.parametrize("path,reason", WRONG_PATHS) def test_refuse_wrong_azure_surfaces(host, path, reason): @@ -369,14 +394,13 @@ def test_missing_configuration_exits_before_tracing(missing, receiver, monkeypat def test_refused_url_main_exits_without_requests_or_spans(receiver, monkeypatch, capsys): - with FakeOpenAI() as fake: - refused = "https://TEST-RESOURCE-0000.SERVICES.AI.AZURE.COM./api/projects/my-project" - monkeypatch.setenv(app.BASE_URL_ENV, refused) - monkeypatch.setattr(app, "setup_tracing", lambda *args, **kwargs: pytest.fail("Tracing ran for refused URL")) - monkeypatch.setattr(app, "make_client", lambda *args, **kwargs: pytest.fail("Client created for refused URL")) - assert app.main([]) == 2 - assert "project endpoint" in capsys.readouterr().err - assert fake.requests() == [] + # The setup_tracing and make_client traps are the controls: no tracer, no client, no request. + refused = "https://TEST-RESOURCE-0000.SERVICES.AI.AZURE.COM./api/projects/my-project" + monkeypatch.setenv(app.BASE_URL_ENV, refused) + monkeypatch.setattr(app, "setup_tracing", lambda *args, **kwargs: pytest.fail("Tracing ran for refused URL")) + monkeypatch.setattr(app, "make_client", lambda *args, **kwargs: pytest.fail("Client created for refused URL")) + assert app.main([]) == 2 + assert "project endpoint" in capsys.readouterr().err assert receiver.spans() == [] and receiver.requests() == [] @@ -491,6 +515,12 @@ def test_readme_and_requirement_pins(): assert "https://learn.microsoft.com/en-us/azure/foundry/openai/api-version-lifecycle" in readme assert "syntactic test hosts" in readme and "not real" in readme assert "gen_ai.request.model" in readme and "TBD" not in readme + assert "streamed and failed Chat Completions calls omit `gen_ai.request.model`" in readme + assert "Responses calls record it from the request" in readme + assert "may return the underlying model name rather than your deployment name" in readme + assert "Requires Python 3.10 or later" in readme + assert "Azure hosts must use `https://`" in readme + assert "`FI_HIDE_INPUTS` does not cover" in readme assert "published `traceAI-openai==0.1.10` and `fi-instrumentation-otel==1.1.0`" in readme exports = dict(re.findall(r'^export (\w+)="([^"]*)"$', readme, re.MULTILINE)) assert exports["FI_API_KEY"] == FI_KEY From e90bb511477438d4049ad30e922c824118ec708c Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Mon, 5 Oct 2026 20:35:07 -0700 Subject: [PATCH 06/10] fix(examples/azure-ai-foundry): correct the Responses model statement The Responses span records the requested deployment only on a failed call. On a successful call, traceai-openai (_attributes/_responses_api.py:132) overwrites gen_ai.request.model with the model the response returns. The earlier README sentence ("Responses calls record it from the request") and the Responses test could not tell the two apart, because the fixture echoed the requested deployment. - The Responses test fixture now returns a different model from the deployment requested. The test pins the returned model in gen_ai.request.model and the deployment in gen_ai.request.parameters. - The README states the failed and successful Responses behaviour; pinned by the README test. Found while fixing the same statement in the Perplexity recipe's review. 74 passed. approved: Nikhil 2026-10-03 blanket Refs: TH-8317 --- python/examples/azure-ai-foundry/README.md | 8 +++++--- .../tests/test_azure_ai_foundry_recipe.py | 13 ++++++++++--- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/python/examples/azure-ai-foundry/README.md b/python/examples/azure-ai-foundry/README.md index 16648d31e..9145d1ada 100644 --- a/python/examples/azure-ai-foundry/README.md +++ b/python/examples/azure-ai-foundry/README.md @@ -97,14 +97,16 @@ accumulated by the same instrumentor. Its default test stream has no usage attributes. With `stream_options={"include_usage": True}` and a final usage chunk, the instrumentor records input, output and total token counts. -In the current `traceai-openai` version, streamed and failed Chat Completions calls omit `gen_ai.request.model`; -Responses calls record it from the request. The requested deployment remains in the +In the current `traceai-openai` version, streamed and failed Chat Completions calls omit `gen_ai.request.model`. +On the Responses API, a failed call records the requested deployment, and a successful Responses call records the model the response returns. +The requested deployment remains in the `gen_ai.request.parameters` JSON unless `FI_HIDE_LLM_INVOCATION_PARAMETERS=true`. Errors set the span status to ERROR and record an exception event. The non-streamed Responses fixture records the full parsed response as Python dictionary text in `output.value`, including its output text. It also records -the response's model and usage, including the supplied reasoning and cache +the model the response returns (the fixture returns a different model from the +deployment requested) and usage, including the supplied reasoning and cache token counts. Responses streaming is not tested here. ## Provider specifics diff --git a/python/examples/azure-ai-foundry/tests/test_azure_ai_foundry_recipe.py b/python/examples/azure-ai-foundry/tests/test_azure_ai_foundry_recipe.py index 9cd332ab7..e06de72ee 100644 --- a/python/examples/azure-ai-foundry/tests/test_azure_ai_foundry_recipe.py +++ b/python/examples/azure-ai-foundry/tests/test_azure_ai_foundry_recipe.py @@ -280,10 +280,14 @@ def test_privacy_hiding_has_visible_control(flag, marker, receiver, tracing, mon assert (marker in json.dumps(receiver.spans())) is (not hidden) +RETURNED_MODEL = "gpt-test-model-2026-09-01" + + def test_responses_api_span_and_exact_url(receiver, tracing): requests = [] with tracing() as (provider, project): - with mock_client(TEST_BASE_URLS[0], responses_response(MODEL), requests) as client: + # The response names a different (underlying) model than the deployment requested. + with mock_client(TEST_BASE_URLS[0], responses_response(RETURNED_MODEL), requests) as client: response = client.responses.create(model=MODEL, input="What is a rainbow?") assert response.output_text == ANSWER provider.force_flush() @@ -292,7 +296,10 @@ def test_responses_api_span_and_exact_url(receiver, tracing): assert_vendor_request(requests[0], TEST_BASE_URLS[0], "responses") assert requests[0].url.path == "/openai/v1/responses" assert json.loads(requests[0].content)["input"] == "What is a rainbow?" - assert attrs["gen_ai.request.model"] == MODEL + # On a successful Responses call the instrumentor records the model the response returns; + # the requested deployment stays in the request parameters. + assert attrs["gen_ai.request.model"] == RETURNED_MODEL + assert json.loads(attrs["gen_ai.request.parameters"])["model"] == MODEL # Non-streamed Responses records the full parsed response's Python dict text. assert attrs["output.value"] == str(response.model_dump()) assert response.model_dump()["output"][0]["content"][0]["text"] == ANSWER @@ -516,7 +523,7 @@ def test_readme_and_requirement_pins(): assert "syntactic test hosts" in readme and "not real" in readme assert "gen_ai.request.model" in readme and "TBD" not in readme assert "streamed and failed Chat Completions calls omit `gen_ai.request.model`" in readme - assert "Responses calls record it from the request" in readme + assert "a successful Responses call records the model the response returns" in readme assert "may return the underlying model name rather than your deployment name" in readme assert "Requires Python 3.10 or later" in readme assert "Azure hosts must use `https://`" in readme From 66ecc127d5d5d88ec4154c225e848c4cb629409e Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Sat, 10 Oct 2026 01:41:58 -0700 Subject: [PATCH 07/10] test(examples): pin chat response model apart from the deployment Chat fixtures returned the deployment name, so they did not exercise the README case where Azure returns a different model id. One chat case now returns the existing synthetic response id and asserts the span stores it while the request still carries the deployment. No live Azure call. RED: response.model was my-gpt-deployment. GREEN: the new case passes on the installed traceAI-openai 0.1.10 wheel. approved: Nikhil 2026-10-03 blanket Refs: TH-8317 --- .../tests/test_azure_ai_foundry_recipe.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/python/examples/azure-ai-foundry/tests/test_azure_ai_foundry_recipe.py b/python/examples/azure-ai-foundry/tests/test_azure_ai_foundry_recipe.py index e06de72ee..64e347ef5 100644 --- a/python/examples/azure-ai-foundry/tests/test_azure_ai_foundry_recipe.py +++ b/python/examples/azure-ai-foundry/tests/test_azure_ai_foundry_recipe.py @@ -208,6 +208,24 @@ def test_chat_both_host_forms_and_key_separation(base_url, receiver, tracing, mo assert_usage(attrs, USAGE) +def test_chat_response_model_differs_from_deployment(receiver, tracing): + """Chat span stores the response model, not the deployment name.""" + assert RETURNED_MODEL != MODEL + requests = [] + with tracing() as (provider, project): + with mock_client(TEST_BASE_URLS[0], chat_response(RETURNED_MODEL), requests) as client: + response = complete(client) + provider.force_flush() + _, attrs = assert_export(receiver, project) + assert response.model == RETURNED_MODEL + assert_vendor_request(requests[0], TEST_BASE_URLS[0], "chat/completions", model=MODEL) + assert attrs["gen_ai.request.model"] == RETURNED_MODEL + body = chat_response(RETURNED_MODEL) + assert body["object"] == "chat.completion" + assert body["model"] == RETURNED_MODEL + assert set(body["usage"]) >= {"prompt_tokens", "completion_tokens", "total_tokens"} + + def test_usage_absent_is_omitted(receiver, tracing): requests = [] with tracing() as (provider, project): From f1b1a27fd0f4bec03f003437926be59c2644f3d4 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Sun, 11 Oct 2026 21:32:24 -0700 Subject: [PATCH 08/10] test(examples/azure-ai-foundry): accept traceai-openai before and after the provider/model fix The recipe pinned traceai-openai 0.1.10 behaviour: gen_ai.provider.name is always "openai" and streamed/failed spans carry no gen_ai.request.model. future-agi/traceAI#249 labels known vendor hosts and records the requested model, so these pins would break whichever PR merged second. Accept both behaviours (and the related Responses/embedding output changes where this recipe checks them), and say in the README which version does what. Passes against published traceAI-openai 0.1.10 and against #249. Co-Authored-By: Claude Opus 5.5 --- python/examples/azure-ai-foundry/README.md | 12 +++++---- .../tests/test_azure_ai_foundry_recipe.py | 27 +++++++++++++------ 2 files changed, 26 insertions(+), 13 deletions(-) diff --git a/python/examples/azure-ai-foundry/README.md b/python/examples/azure-ai-foundry/README.md index 9145d1ada..263c65019 100644 --- a/python/examples/azure-ai-foundry/README.md +++ b/python/examples/azure-ai-foundry/README.md @@ -80,9 +80,10 @@ provider.force_flush() ## What you see in Future AGI The fixture tests produce one LLM span per call: `ChatCompletion` for Chat -Completions and `Response` for the Responses API. The provider field is `openai` +Completions and `Response` for the Responses API. Up to `traceai-openai` 0.1.10, the provider field is `openai` (`gen_ai.provider.name`). This is the shared instrumentor's label for both Azure -host forms. The span does not say `azure`. +host forms; the span does not say `azure`. Later versions (future-agi/traceAI#249) +say `azure` for both host forms; proxy and loopback base URLs keep `openai`. For a successful non-streamed response, `gen_ai.request.model` is the model id the provider returns in its response. The fixtures return the deployment name @@ -97,14 +98,15 @@ accumulated by the same instrumentor. Its default test stream has no usage attributes. With `stream_options={"include_usage": True}` and a final usage chunk, the instrumentor records input, output and total token counts. -In the current `traceai-openai` version, streamed and failed Chat Completions calls omit `gen_ai.request.model`. +Up to `traceai-openai` 0.1.10, streamed and failed Chat Completions calls omit `gen_ai.request.model`; later versions record the requested deployment there. On the Responses API, a failed call records the requested deployment, and a successful Responses call records the model the response returns. The requested deployment remains in the `gen_ai.request.parameters` JSON unless `FI_HIDE_LLM_INVOCATION_PARAMETERS=true`. Errors set the span status to ERROR and record an exception event. -The non-streamed Responses fixture records the full parsed response as Python -dictionary text in `output.value`, including its output text. It also records +Up to 0.1.10, the non-streamed Responses fixture records the full parsed response as Python +dictionary text in `output.value`, including its output text; later versions write it as +JSON (`output.mime_type=application/json`) led by `output_text`. It also records the model the response returns (the fixture returns a different model from the deployment requested) and usage, including the supplied reasoning and cache token counts. Responses streaming is not tested here. diff --git a/python/examples/azure-ai-foundry/tests/test_azure_ai_foundry_recipe.py b/python/examples/azure-ai-foundry/tests/test_azure_ai_foundry_recipe.py index 64e347ef5..82ffc935c 100644 --- a/python/examples/azure-ai-foundry/tests/test_azure_ai_foundry_recipe.py +++ b/python/examples/azure-ai-foundry/tests/test_azure_ai_foundry_recipe.py @@ -23,6 +23,12 @@ ANSWER, ERROR, FakeOpenAI, chat_response, chat_stream, responses_response, ) +# traceai-openai 0.1.10 and earlier label every span "openai"; later versions +# (future-agi/traceAI#249) use the vendor's name for its documented API host. +# They also omit gen_ai.request.model on streamed and failed calls, where later +# versions record the requested model. Accept both so this recipe passes on either version. +PROVIDER_NAMES = ("openai", "azure") + RECIPE = Path(__file__).resolve().parents[1] # Syntactic test hosts, not real Azure resources; MockTransport never resolves them. TEST_BASE_URLS = ( @@ -130,7 +136,7 @@ def assert_export(receiver, project, name="ChatCompletion"): attrs = attributes(span) assert span["name"] == name assert attrs["gen_ai.span.kind"] == "LLM" - assert attrs["gen_ai.provider.name"] == "openai" + assert attrs["gen_ai.provider.name"] in PROVIDER_NAMES export = exports[0] assert export["path"] == "/tracer/v1/traces" assert "x-api-key" in export["headers"] and "x-secret-key" in export["headers"] @@ -253,8 +259,8 @@ def test_stream_output_model_gap_and_usage(include_usage, receiver, tracing): params = json.loads(attrs["gen_ai.request.parameters"]) assert params["model"] == MODEL and params["stream"] is True assert json.loads(requests[0].content).get("stream_options") == ({"include_usage": True} if include_usage else None) - # traceai-openai records the model only from a non-streamed response; flip this when the instrumentor records the request model - assert "gen_ai.request.model" not in attrs + # 0.1.10 records the model only from non-streamed responses; later versions record the requested one. + assert attrs.get("gen_ai.request.model") in (None, MODEL) assert attrs["output.value"] == ANSWER assert span["status"]["code"] == "STATUS_CODE_OK" assert_usage(attrs, USAGE if include_usage else {}) @@ -275,8 +281,8 @@ def test_authentication_error_records_exception_and_model_gap(receiver, tracing) assert any(event["name"] == "exception" for event in span["events"]) assert "AuthenticationError" in json.dumps(span["events"]) assert "invalid subscription key" in json.dumps(span) - # traceai-openai records the model only from a non-streamed response; flip this when the instrumentor records the request model - assert "gen_ai.request.model" not in attrs + # 0.1.10 records the model only from non-streamed responses; later versions record the requested one. + assert attrs.get("gen_ai.request.model") in (None, MODEL) assert json.loads(attrs["gen_ai.request.parameters"])["model"] == MODEL assert_usage(attrs, {}) assert VENDOR_KEY not in str(error.value) @@ -319,7 +325,12 @@ def test_responses_api_span_and_exact_url(receiver, tracing): assert attrs["gen_ai.request.model"] == RETURNED_MODEL assert json.loads(attrs["gen_ai.request.parameters"])["model"] == MODEL # Non-streamed Responses records the full parsed response's Python dict text. - assert attrs["output.value"] == str(response.model_dump()) + # traceai-openai 0.1.10: Python repr of the response; later versions: JSON led by output_text. + if attrs["output.value"] != str(response.model_dump()): + body = json.loads(attrs["output.value"]) + assert body["output_text"] == ANSWER + assert body["output"][0]["content"][0]["text"] == ANSWER + assert attrs["output.mime_type"] == "application/json" assert response.model_dump()["output"][0]["content"][0]["text"] == ANSWER assert span["status"]["code"] == "STATUS_CODE_OK" assert_usage(attrs, {**USAGE, "gen_ai.usage.output_tokens.reasoning": 0, "gen_ai.usage.input_tokens.cache_read": 0}) @@ -486,8 +497,8 @@ def test_loopback_subprocess_journeys(mode, tmp_path, receiver): assert requests[0]["body"]["model"] == model assert ANSWER in attrs["output.value"] if mode == "stream": - # traceai-openai records the model only from a non-streamed response; flip this when the instrumentor records the request model - assert "gen_ai.request.model" not in attrs + # 0.1.10 records the model only from non-streamed responses; later versions record the requested one. + assert attrs.get("gen_ai.request.model") in (None, MODEL) assert json.loads(attrs["gen_ai.request.parameters"])["model"] == model assert_usage(attrs, {}) else: From 9de6f3b0a08a150ffb5e898d2be7edc2db34f9ce Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Sun, 18 Oct 2026 22:03:42 -0700 Subject: [PATCH 09/10] fix(examples/azure-ai-foundry): honour FI_PROJECT_NAME setup_tracing() passed project_name or "azure-ai-foundry" to register(), so the default was always truthy and FI_PROJECT_NAME was never read: `python src/app.py` always exported to azure-ai-foundry. The Anannas recipe reads FI_PROJECT_NAME with its name as the default; do the same here. An explicit setup_tracing() argument still wins, and an unset or empty FI_PROJECT_NAME keeps azure-ai-foundry (never register()'s generic default). - README: FI_PROJECT_NAME in the Configure table, the project-name note and the Run section; the Code snippet reads it the same way as src/app.py. - Tests: setup_tracing() default / empty / env / explicit cases. The subprocess journeys already set FI_PROJECT_NAME but asserted that app.py exported to azure-ai-foundry anyway; they now expect the env project, and chat and stream also run with it unset. The README contract checks the new row and snippet. Passes against repository source, published traceAI-openai 0.1.10 and future-agi/traceAI#249. Co-Authored-By: Claude Opus 5.5 --- python/examples/azure-ai-foundry/README.md | 9 ++-- python/examples/azure-ai-foundry/src/app.py | 2 +- .../tests/test_azure_ai_foundry_recipe.py | 46 +++++++++++++++++-- 3 files changed, 48 insertions(+), 9 deletions(-) diff --git a/python/examples/azure-ai-foundry/README.md b/python/examples/azure-ai-foundry/README.md index 263c65019..0bb3f8fa4 100644 --- a/python/examples/azure-ai-foundry/README.md +++ b/python/examples/azure-ai-foundry/README.md @@ -29,11 +29,12 @@ pip install traceAI-openai fi-instrumentation-otel openai | `AZURE_OPENAI_DEPLOYMENT` | Yes, unless `--model` is used | Your deployment name, not an underlying model id. | | `AZURE_OPENAI_BASE_URL` | Yes | Your resource-specific inference base URL. There is no public default. | | `FI_BASE_URL` | No | Future AGI collector origin; defaults to `https://api.futureagi.com`. | +| `FI_PROJECT_NAME` | No | Observe project name; defaults to `azure-ai-foundry`; tracer only. | | `FI_HIDE_INPUTS` | No | Set to `true` to hide prompt text in exported spans. | | `FI_HIDE_OUTPUTS` | No | Set to `true` to hide response text in exported spans. | The client receives the Azure key. The tracer receives the Future AGI keys. -The example uses the Future AGI project name `azure-ai-foundry`. +The example uses the Future AGI project name `azure-ai-foundry` unless `FI_PROJECT_NAME` is set. ## Run @@ -52,6 +53,8 @@ python src/app.py --stream Use `--prompt "What is a rainbow?"` to change the question or `--model your-deployment-name` to override the deployment environment variable. +Set `FI_PROJECT_NAME` to send the traces to another Observe project; unset or +empty, they go to `azure-ai-foundry`. Missing Azure configuration and refused URLs return exit code 2 before tracing. ## Code @@ -67,8 +70,8 @@ from fi_instrumentation.fi_types import ProjectType from traceai_openai import OpenAIInstrumentor from app import check_base_url base_url = check_base_url(os.environ["AZURE_OPENAI_BASE_URL"]) -provider = register(project_name="azure-ai-foundry", project_type=ProjectType.OBSERVE, - set_global_tracer_provider=False, verbose=False) +provider = register(project_name=os.environ.get("FI_PROJECT_NAME") or "azure-ai-foundry", # as in src/app.py + project_type=ProjectType.OBSERVE, set_global_tracer_provider=False, verbose=False) OpenAIInstrumentor().instrument(tracer_provider=provider) client = OpenAI(api_key=os.environ["AZURE_OPENAI_API_KEY"], base_url=base_url) response = client.chat.completions.create(model=os.environ["AZURE_OPENAI_DEPLOYMENT"], diff --git a/python/examples/azure-ai-foundry/src/app.py b/python/examples/azure-ai-foundry/src/app.py index f1d3ddd52..dd758ed49 100644 --- a/python/examples/azure-ai-foundry/src/app.py +++ b/python/examples/azure-ai-foundry/src/app.py @@ -68,7 +68,7 @@ def make_client(base_url: str | None = None, api_key: str | None = None, http_cl def setup_tracing(project_name: str | None = None): provider = register( - project_name=project_name or "azure-ai-foundry", + project_name=project_name or os.environ.get("FI_PROJECT_NAME") or "azure-ai-foundry", project_type=ProjectType.OBSERVE, set_global_tracer_provider=False, verbose=False, diff --git a/python/examples/azure-ai-foundry/tests/test_azure_ai_foundry_recipe.py b/python/examples/azure-ai-foundry/tests/test_azure_ai_foundry_recipe.py index 82ffc935c..91440a60f 100644 --- a/python/examples/azure-ai-foundry/tests/test_azure_ai_foundry_recipe.py +++ b/python/examples/azure-ai-foundry/tests/test_azure_ai_foundry_recipe.py @@ -232,6 +232,31 @@ def test_chat_response_model_differs_from_deployment(receiver, tracing): assert set(body["usage"]) >= {"prompt_tokens", "completion_tokens", "total_tokens"} +@pytest.mark.parametrize("env_project, explicit, expected", [ + (None, None, "azure-ai-foundry"), + ("", None, "azure-ai-foundry"), + ("azure-from-env", None, "azure-from-env"), + ("azure-from-env", "azure-explicit", "azure-explicit"), +], ids=["default", "empty", "env", "explicit"]) +def test_setup_tracing_project_name(env_project, explicit, expected, receiver, monkeypatch): + # FI_PROJECT_NAME overrides the recipe default; an explicit argument wins over both. + # Never fall through to register()'s generic DEFAULT_PROJECT_NAME. + if env_project is not None: + monkeypatch.setenv("FI_PROJECT_NAME", env_project) + requests = [] + provider = app.setup_tracing(explicit) + try: + with mock_client(TEST_BASE_URLS[0], chat_response(MODEL), requests) as client: + complete(client) + provider.force_flush() + assert len(requests) == 1 + assert_export(receiver, expected) + finally: + provider.force_flush() + OpenAIInstrumentor().uninstrument() + provider.shutdown() + + def test_usage_absent_is_omitted(receiver, tracing): requests = [] with tracing() as (provider, project): @@ -466,16 +491,25 @@ def assert_child_guard(env): assert Path(env["LOOPBACK_GUARD_LOG"]).read_text() == "" -@pytest.mark.parametrize("mode", ["chat", "stream", "model-override", "responses"]) -def test_loopback_subprocess_journeys(mode, tmp_path, receiver): +SUBPROCESS_JOURNEYS = [ + ("chat", True), ("chat", False), ("stream", True), ("stream", False), + ("model-override", True), ("responses", True), +] + + +@pytest.mark.parametrize("mode, env_project", SUBPROCESS_JOURNEYS, + ids=[f"{mode}-{'env' if env else 'default'}-project" for mode, env in SUBPROCESS_JOURNEYS]) +def test_loopback_subprocess_journeys(mode, env_project, tmp_path, receiver): with FakeOpenAI() as fake: env = child_environment(tmp_path, receiver, fake.base_url, "azure-subprocess-" + uuid.uuid4().hex) + if not env_project: + # Unset, app.py keeps the recipe default (never register()'s generic one). + env.pop("FI_PROJECT_NAME") + project = env.get("FI_PROJECT_NAME", "azure-ai-foundry") model = MODEL if mode == "responses": - project = env["FI_PROJECT_NAME"] command = [sys.executable, "-c", "import os, app; p = app.setup_tracing(os.environ['FI_PROJECT_NAME']); c = app.make_client(); print(c.responses.create(model=os.environ[app.MODEL_ENV], input='What is a rainbow?').output_text); c.close(); p.force_flush()"] else: - project = "azure-ai-foundry" command = [sys.executable, str(RECIPE / "src" / "app.py")] if mode == "stream": command.append("--stream") @@ -540,8 +574,10 @@ def must_not_reach_network(*args, **kwargs): def test_readme_and_requirement_pins(): readme = (RECIPE / "README.md").read_text() - for value in (*app.DOCUMENTED_BASE_URL_FORMS, app.API_KEY_ENV, app.BASE_URL_ENV, app.MODEL_ENV, "FI_API_KEY", "FI_SECRET_KEY", "FI_BASE_URL", "FI_HIDE_INPUTS", "FI_HIDE_OUTPUTS"): + for value in (*app.DOCUMENTED_BASE_URL_FORMS, app.API_KEY_ENV, app.BASE_URL_ENV, app.MODEL_ENV, "FI_API_KEY", "FI_SECRET_KEY", "FI_BASE_URL", "FI_PROJECT_NAME", "FI_HIDE_INPUTS", "FI_HIDE_OUTPUTS"): assert value in readme + assert "| `FI_PROJECT_NAME` | No | Observe project name; defaults to `azure-ai-foundry`; tracer only. |" in readme + assert 'project_name=os.environ.get("FI_PROJECT_NAME") or "azure-ai-foundry"' in readme assert "model is your deployment name" in readme assert "provider field is `openai`" in readme assert "Not this page: Foundry Agent Service" in readme From e7e0fa0565f396b43734cd889967f4c083273b99 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Mon, 19 Oct 2026 17:13:48 -0700 Subject: [PATCH 10/10] docs(examples/azure-ai-foundry): record live Azure OpenAI verification (2026-10-19) Chat, stream and an unknown-deployment error were run against a real Azure OpenAI v1 endpoint with the pinned packages; results match the page. Co-Authored-By: Claude Opus 5.5 --- python/examples/azure-ai-foundry/README.md | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/python/examples/azure-ai-foundry/README.md b/python/examples/azure-ai-foundry/README.md index 0bb3f8fa4..852b80ba9 100644 --- a/python/examples/azure-ai-foundry/README.md +++ b/python/examples/azure-ai-foundry/README.md @@ -114,6 +114,18 @@ the model the response returns (the fixture returns a different model from the deployment requested) and usage, including the supplied reasoning and cache token counts. Responses streaming is not tested here. +## Live vendor verification (2026-10-19) + +Run against a real Azure OpenAI resource (East US, pay-as-you-go subscription, `gpt-4.1-nano` deployment named `th8103-nano`, Global Standard), using the `.openai.azure.com/openai/v1/` form, `openai` 3.24.0, `traceAI-openai` 0.1.10 and `fi-instrumentation-otel` 1.1.0, exactly as in "Run" above, exported to a self-hosted Future AGI stack: + +| Call | What Azure and the span showed | +|---|---| +| Chat | Succeeded. The `openai` SDK's `Authorization: Bearer ` is accepted by the v1 endpoint. One `llm` span, `gen_ai.request.model` = `gpt-4.1-nano-2025-04-14` (the underlying model Azure returns, **not** the deployment name `th8103-nano`), 12 input and 119 output tokens, provider `openai`. | +| Stream | Succeeded. One `llm` span with output text, no model and no token counts (no `stream_options`), provider `openai`. | +| Error (unknown deployment) | Azure answered `404 DeploymentNotFound`. One `llm` span with status ERROR and the `NotFoundError` text, no model. | + +This confirms the model-name, streaming and error statements in "What you see in Future AGI". Not exercised live: the `services.ai.azure.com` form, the Responses API, Entra ID auth, and `traceai-openai` after future-agi/traceAI#249. + ## Provider specifics Microsoft documents both of these resource-specific base URL forms: @@ -191,7 +203,7 @@ own logging or data handling. - Foundry agents, threads, evaluations and Application Insights export. - A bare Azure resource root or the project endpoint as an inference base URL. - Responses streaming, async calls and other SDK surfaces in this example's tests. -- Live Azure calls: not tested against the live provider (no paid call); the tests use a local fake of the OpenAI API. +- Live Azure calls: chat, stream and one error were run live on 2026-10-19 (see "Live vendor verification"); the automated tests still use a local fake of the OpenAI API. The recipe does not change the shared instrumentor's `openai` provider label.