From 5936e0960f01a23e842d3df90b7c101874ed8149 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Sat, 3 Oct 2026 18:58:31 -0700 Subject: [PATCH 01/15] test(harness): add shared OTLP harness for TH-8103 contract tests --- python/tests/harness/__init__.py | 184 +++++++++++++++++++++++++++ python/tests/harness/test_harness.py | 101 +++++++++++++++ 2 files changed, 285 insertions(+) create mode 100644 python/tests/harness/__init__.py create mode 100644 python/tests/harness/test_harness.py diff --git a/python/tests/harness/__init__.py b/python/tests/harness/__init__.py new file mode 100644 index 000000000..dfd9198e4 --- /dev/null +++ b/python/tests/harness/__init__.py @@ -0,0 +1,184 @@ +"""Small, loopback-only utilities for OTLP contract tests.""" + +from __future__ import annotations + +import copy +import difflib +import json +import subprocess +import threading +from http import HTTPStatus +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from typing import Any, Mapping, Optional, Sequence, Union +from urllib.parse import urlsplit +from urllib.request import Request, urlopen + + +class Receiver: + """An in-memory OTLP/HTTP receiver for tests. + + The receiver only listens on loopback and never forwards received spans. + """ + + def __init__(self) -> None: + self._spans: list[dict[str, Any]] = [] + self._lock = threading.Lock() + owner = self + + class Handler(BaseHTTPRequestHandler): + def do_POST(self) -> None: # noqa: N802 - required by BaseHTTPRequestHandler + if urlsplit(self.path).path != "/v1/traces": + self.send_error(HTTPStatus.NOT_FOUND) + return + + try: + length = int(self.headers.get("Content-Length", "0")) + body = self.rfile.read(length) + content_type = self.headers.get("Content-Type", "").lower() + + if content_type.startswith("application/json"): + request = json.loads(body.decode("utf-8")) + elif content_type.startswith(("application/x-protobuf", "application/protobuf")): + from google.protobuf.json_format import MessageToDict + from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ( + ExportTraceServiceRequest, + ) + + protobuf_request = ExportTraceServiceRequest() + protobuf_request.ParseFromString(body) + request = MessageToDict(protobuf_request) + else: + self.send_error(HTTPStatus.UNSUPPORTED_MEDIA_TYPE) + return + + spans: list[dict[str, Any]] = [] + for resource_spans in request.get( + "resourceSpans", request.get("resource_spans", []) + ): + for scope_spans in resource_spans.get( + "scopeSpans", resource_spans.get("scope_spans", []) + ): + spans.extend(scope_spans.get("spans", [])) + except (ImportError, TypeError, UnicodeDecodeError, ValueError) as error: + self.send_error(HTTPStatus.BAD_REQUEST, str(error)) + return + + with owner._lock: + owner._spans.extend(copy.deepcopy(spans)) + self.send_response(HTTPStatus.OK) + self.send_header("Content-Length", "0") + self.end_headers() + + def log_message(self, _format: str, *args: Any) -> None: + return + + self._server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + self.origin = "http://127.0.0.1:{0}".format(self._server.server_port) + self.endpoint = "{0}/v1/traces".format(self.origin) + self._thread = threading.Thread(target=self._server.serve_forever, daemon=True) + self._thread.start() + + def __enter__(self) -> "Receiver": + return self + + def __exit__(self, _type: Any, _value: Any, _traceback: Any) -> None: + self._server.shutdown() + self._server.server_close() + self._thread.join() + + def spans(self) -> list[dict[str, Any]]: + """Return a copy of the decoded spans received so far.""" + with self._lock: + return copy.deepcopy(self._spans) + + def clear(self) -> None: + """Remove every decoded span received so far.""" + with self._lock: + self._spans.clear() + + +def run( + argv: Sequence[str], + env: Mapping[str, str], + stdin: Optional[bytes], + timeout: float, +) -> subprocess.CompletedProcess[bytes]: + """Run one process and return its captured output, including after a timeout.""" + process = subprocess.Popen( + argv, + env=dict(env), + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + timed_out = False + try: + stdout, stderr = process.communicate(input=stdin, timeout=timeout) + except subprocess.TimeoutExpired: + timed_out = True + process.kill() + stdout, stderr = process.communicate() + + result = subprocess.CompletedProcess(argv, process.returncode, stdout, stderr) + result.timed_out = timed_out + return result + + +def post_otlp(spans: dict[str, Any], endpoint: str) -> int: + """Post one JSON-encoded OTLP request to a loopback receiver.""" + parsed = urlsplit(endpoint) + if parsed.scheme != "http" or parsed.hostname != "127.0.0.1": + raise ValueError("post_otlp only sends to a loopback HTTP endpoint") + + request = Request( + endpoint, + data=json.dumps(spans).encode("utf-8"), + headers={"Content-Type": "application/json"}, + method="POST", + ) + with urlopen(request, timeout=5) as response: + return response.status + + +def compare(actual: Sequence[dict[str, Any]], golden_path: Union[str, Path]) -> None: + """Fail with a diff unless span names, attributes, and statuses match a golden.""" + with Path(golden_path).open(encoding="utf-8") as golden_file: + expected = json.load(golden_file) + + def canonical(spans: Sequence[dict[str, Any]]) -> list[dict[str, Any]]: + normalized = [] + for span in spans: + attributes = span.get("attributes", []) + if isinstance(attributes, dict): + attributes = [ + {"key": key, "value": value} for key, value in attributes.items() + ] + normalized.append( + { + "name": span.get("name", ""), + "attributes": sorted( + attributes, + key=lambda attribute: json.dumps( + attribute, sort_keys=True, separators=(",", ":") + ), + ), + "status": span.get("status", {}), + } + ) + return normalized + + actual_normalized = canonical(actual) + expected_normalized = canonical(expected) + if actual_normalized != expected_normalized: + diff = "\n".join( + difflib.unified_diff( + json.dumps(expected_normalized, indent=2, sort_keys=True).splitlines(), + json.dumps(actual_normalized, indent=2, sort_keys=True).splitlines(), + fromfile=str(golden_path), + tofile="actual", + lineterm="", + ) + ) + print(diff) + raise AssertionError("spans do not match golden") diff --git a/python/tests/harness/test_harness.py b/python/tests/harness/test_harness.py new file mode 100644 index 000000000..4e76ceabe --- /dev/null +++ b/python/tests/harness/test_harness.py @@ -0,0 +1,101 @@ +"""Contract tests for the shared loopback-only harness.""" + +import json +import sys + +import pytest + +from tests.harness import Receiver, compare, post_otlp, run + + +def test_receiver_stores_posted_span_and_can_clear() -> None: + payload = { + "resourceSpans": [ + { + "scopeSpans": [ + { + "spans": [ + { + "traceId": "trace-id", + "spanId": "span-id", + "name": "harness.receiver", + "attributes": [ + {"key": "component", "value": {"stringValue": "test"}} + ], + "status": {"code": 1}, + } + ] + } + ] + } + ] + } + + with Receiver() as receiver: + assert post_otlp(payload, receiver.endpoint) == 200 + assert receiver.spans() == payload["resourceSpans"][0]["scopeSpans"][0]["spans"] + receiver.clear() + assert receiver.spans() == [] + + +def test_run_captures_output_and_kills_timed_out_process() -> None: + completed = run( + [ + sys.executable, + "-c", + "import sys; print('stdout'); sys.stderr.write('stderr\\n')", + ], + env={}, + stdin=b"", + timeout=1, + ) + assert completed.returncode == 0 + assert completed.stdout == b"stdout\n" + assert completed.stderr == b"stderr\n" + assert completed.timed_out is False + + timed_out = run( + [sys.executable, "-c", "import time; time.sleep(30)"], + env={}, + stdin=b"", + timeout=0.1, + ) + assert timed_out.timed_out is True + assert timed_out.returncode != 0 + + +def test_compare_accepts_normalized_match_and_prints_mismatch_diff(tmp_path, capsys) -> None: + golden = [ + { + "traceId": "golden-trace", + "spanId": "golden-span", + "startTimeUnixNano": "1", + "endTimeUnixNano": "2", + "name": "harness.compare", + "attributes": [ + {"key": "alpha", "value": {"stringValue": "a"}}, + {"key": "beta", "value": {"intValue": "2"}}, + ], + "status": {"code": 1}, + } + ] + golden_path = tmp_path / "spans.golden.json" + golden_path.write_text(json.dumps(golden), encoding="utf-8") + + actual = [ + { + "traceId": "other-trace", + "spanId": "other-span", + "startTimeUnixNano": "3", + "endTimeUnixNano": "4", + "name": "harness.compare", + "attributes": list(reversed(golden[0]["attributes"])), + "status": {"code": 1}, + } + ] + compare(actual, golden_path) + + actual[0]["name"] = "harness.mismatch" + with pytest.raises(AssertionError, match="spans do not match golden"): + compare(actual, golden_path) + assert "---" in capsys.readouterr().out From 32f5c9d2b3b80f97a70e00b3f3ca1b408b14dc5c Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Sat, 3 Oct 2026 23:21:29 -0700 Subject: [PATCH 02/15] test(harness): accept the collector /tracer/v1/traces path --- python/tests/harness/__init__.py | 3 ++- python/tests/harness/test_harness.py | 12 ++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/python/tests/harness/__init__.py b/python/tests/harness/__init__.py index dfd9198e4..cb510ad10 100644 --- a/python/tests/harness/__init__.py +++ b/python/tests/harness/__init__.py @@ -28,7 +28,7 @@ def __init__(self) -> None: class Handler(BaseHTTPRequestHandler): def do_POST(self) -> None: # noqa: N802 - required by BaseHTTPRequestHandler - if urlsplit(self.path).path != "/v1/traces": + if urlsplit(self.path).path not in ("/v1/traces", "/tracer/v1/traces"): self.send_error(HTTPStatus.NOT_FOUND) return @@ -76,6 +76,7 @@ def log_message(self, _format: str, *args: Any) -> None: self._server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) self.origin = "http://127.0.0.1:{0}".format(self._server.server_port) self.endpoint = "{0}/v1/traces".format(self.origin) + self.collector_endpoint = "{0}/tracer/v1/traces".format(self.origin) self._thread = threading.Thread(target=self._server.serve_forever, daemon=True) self._thread.start() diff --git a/python/tests/harness/test_harness.py b/python/tests/harness/test_harness.py index 4e76ceabe..6a32c2af8 100644 --- a/python/tests/harness/test_harness.py +++ b/python/tests/harness/test_harness.py @@ -38,6 +38,18 @@ def test_receiver_stores_posted_span_and_can_clear() -> None: assert receiver.spans() == [] +def test_receiver_accepts_the_collector_path() -> None: + payload = { + "resourceSpans": [ + {"scopeSpans": [{"spans": [{"name": "collector.path"}]}]} + ] + } + + with Receiver() as receiver: + assert post_otlp(payload, receiver.collector_endpoint) == 200 + assert receiver.spans() == [{"name": "collector.path"}] + + def test_run_captures_output_and_kills_timed_out_process() -> None: completed = run( [ From 3eaadc8bdbf428a472db6779091bbce3d6d89015 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Sun, 4 Oct 2026 15:59:06 -0700 Subject: [PATCH 03/15] fix(harness): decode chunked OTLP bodies and record request metadata The Node OTLP exporter streams with Transfer-Encoding: chunked and sends no Content-Length. The Receiver read Content-Length only, so every Node export came back 400 with zero spans; two TH-8103 children had to put a de-chunking relay in front of it. Receiver now accepts both framings. It also records one entry per accepted export (path, lower-cased headers, flattened resource attributes) via requests(), so a contract test can assert X-Api-Key/X-Secret-Key and project_name through the shared harness instead of a private recorder. Verified: 6 harness tests pass, and the TanStack example's real Node exporter delivered 4 chunked exports (4 spans, collector path, both auth headers, project_name/project_type) with no relay. Refs: TH-8339, TH-8103 --- python/tests/harness/__init__.py | 61 ++++++++++++++++++++-- python/tests/harness/test_harness.py | 76 ++++++++++++++++++++++++++++ 2 files changed, 133 insertions(+), 4 deletions(-) diff --git a/python/tests/harness/__init__.py b/python/tests/harness/__init__.py index cb510ad10..cf79047f1 100644 --- a/python/tests/harness/__init__.py +++ b/python/tests/harness/__init__.py @@ -23,18 +23,36 @@ class Receiver: def __init__(self) -> None: self._spans: list[dict[str, Any]] = [] + self._requests: list[dict[str, Any]] = [] self._lock = threading.Lock() owner = self class Handler(BaseHTTPRequestHandler): + def _read_body(self) -> bytes: + # The Node OTLP exporter streams with Transfer-Encoding: chunked + # and sends no Content-Length, so both framings are accepted. + if "chunked" in self.headers.get("Transfer-Encoding", "").lower(): + chunks = [] + while True: + size_line = self.rfile.readline().split(b";", 1)[0].strip() + size = int(size_line, 16) + if size == 0: + while self.rfile.readline() not in (b"\r\n", b"\n", b""): + pass + return b"".join(chunks) + chunks.append(self.rfile.read(size)) + self.rfile.readline() + length = int(self.headers.get("Content-Length", "0")) + return self.rfile.read(length) + def do_POST(self) -> None: # noqa: N802 - required by BaseHTTPRequestHandler - if urlsplit(self.path).path not in ("/v1/traces", "/tracer/v1/traces"): + path = urlsplit(self.path).path + if path not in ("/v1/traces", "/tracer/v1/traces"): self.send_error(HTTPStatus.NOT_FOUND) return try: - length = int(self.headers.get("Content-Length", "0")) - body = self.rfile.read(length) + body = self._read_body() content_type = self.headers.get("Content-Type", "").lower() if content_type.startswith("application/json"): @@ -53,9 +71,15 @@ def do_POST(self) -> None: # noqa: N802 - required by BaseHTTPRequestHandler return spans: list[dict[str, Any]] = [] + resource_attributes: list[dict[str, Any]] = [] for resource_spans in request.get( "resourceSpans", request.get("resource_spans", []) ): + resource_attributes.append( + _flatten_attributes( + resource_spans.get("resource", {}).get("attributes", []) + ) + ) for scope_spans in resource_spans.get( "scopeSpans", resource_spans.get("scope_spans", []) ): @@ -66,6 +90,15 @@ def do_POST(self) -> None: # noqa: N802 - required by BaseHTTPRequestHandler with owner._lock: owner._spans.extend(copy.deepcopy(spans)) + owner._requests.append( + { + "path": path, + "headers": { + key.lower(): value for key, value in self.headers.items() + }, + "resource_attributes": resource_attributes, + } + ) self.send_response(HTTPStatus.OK) self.send_header("Content-Length", "0") self.end_headers() @@ -93,10 +126,30 @@ def spans(self) -> list[dict[str, Any]]: with self._lock: return copy.deepcopy(self._spans) + def requests(self) -> list[dict[str, Any]]: + """Return one record per accepted export: path, lower-cased headers, resource attributes.""" + with self._lock: + return copy.deepcopy(self._requests) + def clear(self) -> None: - """Remove every decoded span received so far.""" + """Remove every decoded span and request record received so far.""" with self._lock: self._spans.clear() + self._requests.clear() + + +def _flatten_attributes(attributes: Sequence[Mapping[str, Any]]) -> dict[str, Any]: + """Turn OTLP key/value attributes into a plain dict of scalar values.""" + flat: dict[str, Any] = {} + for attribute in attributes: + value = attribute.get("value", {}) + for kind in ("stringValue", "boolValue", "intValue", "doubleValue"): + if kind in value: + flat[attribute.get("key", "")] = value[kind] + break + else: + flat[attribute.get("key", "")] = value + return flat def run( diff --git a/python/tests/harness/test_harness.py b/python/tests/harness/test_harness.py index 6a32c2af8..bd61d5adc 100644 --- a/python/tests/harness/test_harness.py +++ b/python/tests/harness/test_harness.py @@ -50,6 +50,82 @@ def test_receiver_accepts_the_collector_path() -> None: assert receiver.spans() == [{"name": "collector.path"}] +def _post_chunked(url: str, body: bytes, headers: dict) -> int: + """Send a body with Transfer-Encoding: chunked, as the Node OTLP exporter does.""" + import http.client + from urllib.parse import urlsplit + + parts = urlsplit(url) + connection = http.client.HTTPConnection(parts.hostname, parts.port, timeout=5) + try: + chunks = [body[: len(body) // 2], body[len(body) // 2 :]] + connection.request( + "POST", + parts.path, + body=iter(chunks), + headers={**headers, "Transfer-Encoding": "chunked"}, + encode_chunked=True, + ) + return connection.getresponse().status + finally: + connection.close() + + +def test_receiver_decodes_a_chunked_body() -> None: + payload = { + "resourceSpans": [ + {"scopeSpans": [{"spans": [{"name": "chunked.body"}]}]} + ] + } + + with Receiver() as receiver: + status = _post_chunked( + receiver.collector_endpoint, + json.dumps(payload).encode("utf-8"), + {"Content-Type": "application/json"}, + ) + assert status == 200 + assert receiver.spans() == [{"name": "chunked.body"}] + + +def test_receiver_records_path_headers_and_resource_attributes() -> None: + payload = { + "resourceSpans": [ + { + "resource": { + "attributes": [ + {"key": "project_name", "value": {"stringValue": "harness"}}, + {"key": "project_type", "value": {"stringValue": "observe"}}, + ] + }, + "scopeSpans": [{"spans": [{"name": "with.resource"}]}], + } + ] + } + + with Receiver() as receiver: + status = _post_chunked( + receiver.collector_endpoint, + json.dumps(payload).encode("utf-8"), + { + "Content-Type": "application/json", + "X-Api-Key": "test-key", + "X-Secret-Key": "test-secret", + }, + ) + assert status == 200 + [request] = receiver.requests() + assert request["path"] == "/tracer/v1/traces" + assert request["headers"]["x-api-key"] == "test-key" + assert request["headers"]["x-secret-key"] == "test-secret" + assert "authorization" not in request["headers"] + assert request["resource_attributes"] == [ + {"project_name": "harness", "project_type": "observe"} + ] + receiver.clear() + assert receiver.requests() == [] + + def test_run_captures_output_and_kills_timed_out_process() -> None: completed = run( [ From 3e8bc4b730f81b836291630cab8eefb13fedf198 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Mon, 5 Oct 2026 10:29:34 -0700 Subject: [PATCH 04/15] feat(examples): add MLflow OTLP recipe app pinned to mlflow 3.16.1 No traceAI package: MLflow 3.16.1 already exports its spans over OTLP when OTEL_EXPORTER_OTLP_TRACES_ENDPOINT is set. src/app.py makes two nested mlflow.start_span calls (a CHAIN root and a CHAT_MODEL child with gen_ai.* and mlflow.llm.* model/token keys) around a canned stand-in model client, so it makes no model call. check_environment() exits with a message on the settings that make MLflow export nothing without an error: no full traces URL (MLflow does not append /v1/traces, utils/otlp.py:98-99), a protocol other than http/protobuf (the default is gRPC, otlp.py:124-136), OTEL_EXPORTER_OTLP_ENDPOINT or OTEL_EXPORTER_OTLP_METRICS_ENDPOINT (they turn on gRPC metrics, which drop every span), and OTEL_RESOURCE_ATTRIBUTES without project_name or with an item MLflow cannot parse (provider.py:716-731). --drop-content registers a span processor through mlflow.tracing.configure; MLflow has no content-off switch. requirements.txt pins mlflow==3.16.1 and adds opentelemetry-exporter-otlp-proto-http==1.45.0, which mlflow does not install. Co-Authored-By: Claude Opus 5.5 approved: Nikhil 2026-10-03 blanket Refs: TH-8334 --- python/examples/mlflow/requirements.txt | 5 + python/examples/mlflow/src/app.py | 133 ++++++++++++++++++++++++ 2 files changed, 138 insertions(+) create mode 100644 python/examples/mlflow/requirements.txt create mode 100644 python/examples/mlflow/src/app.py diff --git a/python/examples/mlflow/requirements.txt b/python/examples/mlflow/requirements.txt new file mode 100644 index 000000000..dafc747cd --- /dev/null +++ b/python/examples/mlflow/requirements.txt @@ -0,0 +1,5 @@ +# Not a traceAI package: these are the example's own dependencies. +mlflow==3.16.1 +# mlflow 3.16.1 depends on opentelemetry-sdk but installs no OTLP exporter. +# Without this one it exports nothing over OTLP and logs nothing about it. +opentelemetry-exporter-otlp-proto-http==1.45.0 # the version the contract test ran with diff --git a/python/examples/mlflow/src/app.py b/python/examples/mlflow/src/app.py new file mode 100644 index 000000000..a5fc8703b --- /dev/null +++ b/python/examples/mlflow/src/app.py @@ -0,0 +1,133 @@ +"""Send MLflow traces to Future AGI over OTLP. + +Recipe example, not a package. MLflow already exports its own spans over +OTLP when ``OTEL_EXPORTER_OTLP_TRACES_ENDPOINT`` is set before the first +span; this file only checks that environment and creates two nested spans +with ``mlflow.start_span``. The endpoint, protocol, Future AGI keys and +project come from the environment variables listed in README.md. Nothing +here talks to an MLflow tracking server, Databricks or a model provider. + + python src/app.py "What is the refund window?" + python src/app.py --drop-content "What is the refund window?" +""" + +from __future__ import annotations + +import os +import sys + +import mlflow +from mlflow.entities import LiveSpan, SpanType + +MODEL = "gpt-4o-mini" +PROVIDER = "openai" +REDACTED = "[REDACTED]" + +_POLICY = "Refunds are accepted within 30 days of purchase." + + +def check_environment() -> None: + """Exit with a message on the settings that make MLflow 3.16.1 export nothing. + + Each of these fails without an exception in the app (see README.md). + """ + endpoint = os.environ.get("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", "") + if not endpoint.rstrip("/").endswith("/v1/traces"): + # MLflow uses this URL as is; it does not append /v1/traces. When the + # variable is unset, MLflow writes traces to a local ./mlflow.db. + raise SystemExit( + "Set OTEL_EXPORTER_OTLP_TRACES_ENDPOINT to the full traces URL, " + "for example https:///tracer/v1/traces." + ) + if os.environ.get("OTEL_EXPORTER_OTLP_TRACES_PROTOCOL") != "http/protobuf": + # MLflow's default is gRPC, and it skips export if that exporter is + # not installed. + raise SystemExit("Set OTEL_EXPORTER_OTLP_TRACES_PROTOCOL=http/protobuf.") + for name in ("OTEL_EXPORTER_OTLP_ENDPOINT", "OTEL_EXPORTER_OTLP_METRICS_ENDPOINT"): + if os.environ.get(name): + # Either one also turns on MLflow's OTLP metrics, which fi-collector + # does not accept; with the default gRPC metrics protocol and no + # gRPC exporter installed, MLflow then drops every span. + raise SystemExit("Unset {0}; set only the TRACES variables.".format(name)) + resource = os.environ.get("OTEL_RESOURCE_ATTRIBUTES", "") + items = [item.split("=", 1) for item in resource.split(",")] if resource else [] + if any(len(item) != 2 for item in items): + # MLflow re-parses this variable and, if one item has no "=" (a + # trailing comma is enough), drops every attribute in it. + raise SystemExit("Write OTEL_RESOURCE_ATTRIBUTES as key=value pairs joined by commas.") + if not dict((key.strip(), value.strip()) for key, value in items).get("project_name"): + # fi-collector rejects (HTTP 400) a batch whose resource has no + # project_name, and creates the project on first use. + raise SystemExit("Add project_name= to OTEL_RESOURCE_ATTRIBUTES.") + + +def drop_content(span: LiveSpan) -> None: + """Replace a span's inputs and outputs before it is exported. + + MLflow has no setting that keeps tracing on and content off; a span + processor registered with ``mlflow.tracing.configure`` is its own hook. + """ + if span.inputs is not None: + span.set_inputs(REDACTED) + if span.outputs is not None: + span.set_outputs(REDACTED) + + +def call_model(messages: list[dict[str, str]]) -> tuple[str, dict[str, int]]: + """Stand-in for your model client: a canned answer and its token counts.""" + return "Refunds are accepted for 30 days.", {"input_tokens": 12, "output_tokens": 7} + + +def answer(question: str) -> str: + with mlflow.start_span(name="answer_question", span_type=SpanType.CHAIN) as root: + root.set_inputs({"question": question}) + with mlflow.start_span(name="chat_model", span_type=SpanType.CHAT_MODEL) as llm: + messages = [ + {"role": "system", "content": "Answer from this policy: " + _POLICY}, + {"role": "user", "content": question}, + ] + llm.set_inputs({"model": MODEL, "messages": messages}) + reply, usage = call_model(messages) + llm.set_outputs({"choices": [{"message": {"role": "assistant", "content": reply}}]}) + # The OTel GenAI keys fi-collector reads. MLflow JSON-encodes every + # attribute value, so these arrive as strings (see README.md). + llm.set_attributes( + { + "gen_ai.request.model": MODEL, + "gen_ai.provider.name": PROVIDER, + "gen_ai.usage.input_tokens": usage["input_tokens"], + "gen_ai.usage.output_tokens": usage["output_tokens"], + } + ) + # The keys MLflow's own autolog sets on a model call. With + # MLFLOW_ENABLE_OTEL_GENAI_SEMCONV=true, MLflow exports them as + # plain gen_ai.* values instead. + llm.set_attributes( + { + "mlflow.llm.model": MODEL, + "mlflow.llm.provider": PROVIDER, + "mlflow.chat.tokenUsage": { + **usage, + "total_tokens": usage["input_tokens"] + usage["output_tokens"], + }, + } + ) + root.set_outputs({"answer": reply}) + return reply + + +def main(argv: list[str]) -> int: + args = argv[1:] + check_environment() + if args[:1] == ["--drop-content"]: + mlflow.tracing.configure(span_processors=[drop_content]) + args = args[1:] + question = args[0] if args else "What is the refund window?" + print(answer(question)) + # MLflow's OTLP exporter runs in an OpenTelemetry BatchSpanProcessor, and + # the OpenTelemetry SDK flushes it at exit, so a short script needs no more. + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) From 3265995582ba91e76b81a2fc43b91192af3623a5 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Mon, 5 Oct 2026 10:29:39 -0700 Subject: [PATCH 05/15] test(examples): add loopback guard and fixtures for the MLflow recipe - _guarded_run.py blocks and logs every non-loopback connect, connect_ex and getaddrinfo before the script runs (same guard as the OpenLLMetry recipe); _guard_probe.py is its positive control against TEST-NET-1, the IPv6 documentation prefix and .invalid. - bare_span.py runs the app's spans without check_environment(), so the test can record what MLflow itself does with the settings the app refuses. - set_destination_app.py calls mlflow.tracing.set_destination before the app, for the precedence test. - fixtures/gen_ai_span.otlp.json is a hand-built OTLP request with two gen_ai.* spans for post_otlp(): the shape MLflow sends with MLFLOW_ENABLE_OTEL_GENAI_SEMCONV=true and its default JSON-encoded shape. Co-Authored-By: Claude Opus 5.5 approved: Nikhil 2026-10-03 blanket Refs: TH-8334 --- python/examples/mlflow/tests/_guard_probe.py | 26 ++++++ python/examples/mlflow/tests/_guarded_run.py | 83 +++++++++++++++++++ python/examples/mlflow/tests/bare_span.py | 17 ++++ .../tests/fixtures/gen_ai_span.otlp.json | 54 ++++++++++++ .../mlflow/tests/set_destination_app.py | 23 +++++ 5 files changed, 203 insertions(+) create mode 100644 python/examples/mlflow/tests/_guard_probe.py create mode 100644 python/examples/mlflow/tests/_guarded_run.py create mode 100644 python/examples/mlflow/tests/bare_span.py create mode 100644 python/examples/mlflow/tests/fixtures/gen_ai_span.otlp.json create mode 100644 python/examples/mlflow/tests/set_destination_app.py diff --git a/python/examples/mlflow/tests/_guard_probe.py b/python/examples/mlflow/tests/_guard_probe.py new file mode 100644 index 000000000..5ebeff59d --- /dev/null +++ b/python/examples/mlflow/tests/_guard_probe.py @@ -0,0 +1,26 @@ +"""A test-only probe: try three non-loopback connections under the loopback guard. + +The contract test runs this through ``_guarded_run.py`` to prove the guard +really refuses and logs a non-loopback attempt, so the ``guard_attempts == []`` +assertions elsewhere cannot pass because the guard silently stopped working. +The targets are reserved and never reachable: TEST-NET-1 (RFC 5737), the IPv6 +documentation prefix (RFC 3849) and the ``.invalid`` TLD (RFC 2606). +Not part of the recipe. +""" + +from __future__ import annotations + +import socket + +refused = [] +for label, attempt in ( + ("connect", lambda: socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect(("192.0.2.1", 80))), + ("connect_ex", lambda: socket.socket(socket.AF_INET6, socket.SOCK_STREAM).connect_ex(("2001:db8::1", 80, 0, 0))), + ("getaddrinfo", lambda: socket.getaddrinfo("example.invalid", 443)), +): + try: + attempt() + except OSError as error: + if "blocked by loopback guard" in str(error): + refused.append(label) +print("refused:", ",".join(refused)) diff --git a/python/examples/mlflow/tests/_guarded_run.py b/python/examples/mlflow/tests/_guarded_run.py new file mode 100644 index 000000000..365df4913 --- /dev/null +++ b/python/examples/mlflow/tests/_guarded_run.py @@ -0,0 +1,83 @@ +"""Run a script with every non-loopback network connection blocked and logged. + +Usage: python _guarded_run.py [args...] + +Before the script starts, ``socket.socket.connect``/``connect_ex`` and +``socket.getaddrinfo`` refuse any host other than 127.0.0.1, ::1 or +localhost. Each refused attempt is appended as a JSON line to the file named +by ``LOOPBACK_GUARD_LOG``, so a test can assert that MLflow and the example +tried to reach nothing else (no tracking server, no Databricks, no MLflow +telemetry host, no model provider). Unix sockets are not affected, and +neither is native code that opens sockets itself (the recipe is HTTP only and +installs no gRPC exporter). +""" + +from __future__ import annotations + +import json +import os +import runpy +import socket +import sys + +_ALLOWED_HOSTS = {"127.0.0.1", "::1", "localhost"} +_LOG_PATH = os.environ.get("LOOPBACK_GUARD_LOG", "") + + +def _host(value: object) -> str: + if isinstance(value, bytes): + return value.decode("ascii", "replace") + return str(value) + + +def _refuse(kind: str, target: object) -> None: + if _LOG_PATH: + with open(_LOG_PATH, "a", encoding="utf-8") as log: + log.write(json.dumps({"kind": kind, "target": repr(target)}) + "\n") + + +_original_connect = socket.socket.connect +_original_connect_ex = socket.socket.connect_ex +_original_getaddrinfo = socket.getaddrinfo + + +def _is_blocked(sock: socket.socket, address: object) -> bool: + if sock.family not in (socket.AF_INET, socket.AF_INET6): + return False + host = address[0] if isinstance(address, tuple) else address + return _host(host) not in _ALLOWED_HOSTS + + +def _connect(self: socket.socket, address: object) -> None: + if _is_blocked(self, address): + _refuse("connect", address) + raise OSError("blocked by loopback guard: {0!r}".format(address)) + return _original_connect(self, address) + + +def _connect_ex(self: socket.socket, address: object) -> int: + if _is_blocked(self, address): + _refuse("connect_ex", address) + raise OSError("blocked by loopback guard: {0!r}".format(address)) + return _original_connect_ex(self, address) + + +def _getaddrinfo(host: object, *args: object, **kwargs: object): # type: ignore[no-untyped-def] + if host is not None and _host(host) not in _ALLOWED_HOSTS: + _refuse("getaddrinfo", host) + raise socket.gaierror("blocked by loopback guard: {0!r}".format(host)) + return _original_getaddrinfo(host, *args, **kwargs) + + +def main() -> None: + socket.socket.connect = _connect # type: ignore[method-assign] + socket.socket.connect_ex = _connect_ex # type: ignore[method-assign] + socket.getaddrinfo = _getaddrinfo # type: ignore[assignment] + script = sys.argv[1] + sys.argv = sys.argv[1:] + sys.path.insert(0, os.path.dirname(os.path.abspath(script))) + runpy.run_path(script, run_name="__main__") + + +if __name__ == "__main__": + main() diff --git a/python/examples/mlflow/tests/bare_span.py b/python/examples/mlflow/tests/bare_span.py new file mode 100644 index 000000000..b4ccbf441 --- /dev/null +++ b/python/examples/mlflow/tests/bare_span.py @@ -0,0 +1,17 @@ +"""Test fixture: the recipe's spans without ``check_environment()``. + +The contract test runs this to record what MLflow 3.16.1 itself does with +the settings ``src/app.py`` refuses (a URL without ``/v1/traces``, no +protocol, ``OTEL_EXPORTER_OTLP_ENDPOINT``). Not part of the recipe. +""" + +from __future__ import annotations + +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src")) + +from app import answer # noqa: E402 + +print(answer(sys.argv[1] if len(sys.argv) > 1 else "What is the refund window?")) diff --git a/python/examples/mlflow/tests/fixtures/gen_ai_span.otlp.json b/python/examples/mlflow/tests/fixtures/gen_ai_span.otlp.json new file mode 100644 index 000000000..bd1e043de --- /dev/null +++ b/python/examples/mlflow/tests/fixtures/gen_ai_span.otlp.json @@ -0,0 +1,54 @@ +{ + "resourceSpans": [ + { + "resource": { + "attributes": [ + {"key": "project_name", "value": {"stringValue": "mlflow-recipe-contract"}}, + {"key": "project_type", "value": {"stringValue": "observe"}}, + {"key": "service.name", "value": {"stringValue": "mlflow-recipe"}} + ] + }, + "scopeSpans": [ + { + "scope": {"name": "mlflow.tracing.fluent"}, + "spans": [ + { + "traceId": "5b8efff798038103d269b633813fc60c", + "spanId": "eee19b7ec3c1b174", + "parentSpanId": "eee19b7ec3c1b173", + "name": "chat gpt-4o-mini", + "kind": 3, + "startTimeUnixNano": "1759651200000000000", + "endTimeUnixNano": "1759651200250000000", + "attributes": [ + {"key": "gen_ai.operation.name", "value": {"stringValue": "chat"}}, + {"key": "gen_ai.request.model", "value": {"stringValue": "gpt-4o-mini"}}, + {"key": "gen_ai.provider.name", "value": {"stringValue": "openai"}}, + {"key": "gen_ai.usage.input_tokens", "value": {"intValue": "12"}}, + {"key": "gen_ai.usage.output_tokens", "value": {"intValue": "7"}} + ], + "status": {"code": 1} + }, + { + "traceId": "5b8efff798038103d269b633813fc60c", + "spanId": "eee19b7ec3c1b175", + "parentSpanId": "eee19b7ec3c1b173", + "name": "chat_model", + "kind": 1, + "startTimeUnixNano": "1759651200300000000", + "endTimeUnixNano": "1759651200550000000", + "attributes": [ + {"key": "mlflow.spanType", "value": {"stringValue": "\"CHAT_MODEL\""}}, + {"key": "gen_ai.request.model", "value": {"stringValue": "\"gpt-4o-mini\""}}, + {"key": "gen_ai.provider.name", "value": {"stringValue": "\"openai\""}}, + {"key": "gen_ai.usage.input_tokens", "value": {"stringValue": "12"}}, + {"key": "gen_ai.usage.output_tokens", "value": {"stringValue": "7"}} + ], + "status": {"code": 1} + } + ] + } + ] + } + ] +} diff --git a/python/examples/mlflow/tests/set_destination_app.py b/python/examples/mlflow/tests/set_destination_app.py new file mode 100644 index 000000000..4826e31d7 --- /dev/null +++ b/python/examples/mlflow/tests/set_destination_app.py @@ -0,0 +1,23 @@ +"""Test fixture: ``src/app.py`` after ``mlflow.tracing.set_destination``. + +MLflow documents that a destination set with this call takes precedence over +the OTLP environment variables. The contract test runs this with the +recipe's environment to show what that means for the export. Not part of the +recipe. +""" + +from __future__ import annotations + +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src")) + +import mlflow # noqa: E402 +from mlflow.entities.trace_location import MlflowExperimentLocation # noqa: E402 + +from app import main # noqa: E402 + +# Experiment 0 is the "Default" experiment MLflow creates in a new store. +mlflow.tracing.set_destination(MlflowExperimentLocation(experiment_id="0")) +sys.exit(main(sys.argv)) From b6008ca58fd748f70f22966f7b3fb3b245c55ebd Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Mon, 5 Oct 2026 10:29:51 -0700 Subject: [PATCH 06/15] test(examples): contract-test the MLflow recipe with mlflow 3.16.1 Runs src/app.py as written with the real mlflow 3.16.1 in a subprocess, under the loopback guard, with a fresh temp dir as cwd and HOME so every file MLflow writes is checked. Spans go to the harness Receiver; path and catalog checks use a loopback catch-all. Covered: /tracer/v1/traces and /v1/traces both 200 and MLflow does not append /v1/traces (origin, origin/ and /tracer get a 404); no protocol means silent no-export; a base or metrics endpoint drops every span; percent-encoded x-api-key/x-secret-key with ',' and '=' arrive intact; project_name/project_type on the resource, lost on a trailing comma; MLflow JSON-encodes every attribute value (gen_ai.request.model arrives with its quotes); no experiment id or session on the export, and an experiment id passed in OTEL_RESOURCE_ATTRIBUTES arrives as a resource attribute; content in mlflow.spanInputs/spanOutputs and the drop-content processor; dual export unset/false/true with and without a tracking URI (./mlflow.db checked by trace id) and its model-catalog fetch; set_destination silences OTLP entirely, and with dual export sends both; the telemetry switch; MLFLOW_ENABLE_OTEL_GENAI_SEMCONV=true output; the hand-built gen_ai.* fixture via post_otlp() and its match with live runs. Opt-in (FI_COLLECTOR_SRC): fi-collector's alias tables include the gen_ai.* keys and no mlflow.* key, and the README's collector claims and span-type table match adapter.go/converter.go and MLflow's translator.py. Co-Authored-By: Claude Opus 5.5 approved: Nikhil 2026-10-03 blanket Refs: TH-8334 --- .../mlflow/tests/test_mlflow_recipe.py | 1014 +++++++++++++++++ 1 file changed, 1014 insertions(+) create mode 100644 python/examples/mlflow/tests/test_mlflow_recipe.py diff --git a/python/examples/mlflow/tests/test_mlflow_recipe.py b/python/examples/mlflow/tests/test_mlflow_recipe.py new file mode 100644 index 000000000..50c808e83 --- /dev/null +++ b/python/examples/mlflow/tests/test_mlflow_recipe.py @@ -0,0 +1,1014 @@ +"""Contract test for the MLflow OTLP recipe. + +Runs ``src/app.py`` as written with the real ``mlflow`` 3.16.1, in a +subprocess whose network is limited to 127.0.0.1 (``_guarded_run.py``) and +whose working directory and HOME are a fresh temporary directory, so every +file MLflow writes is visible to the test. Spans go to the shared harness +``Receiver``, which serves ``/v1/traces`` and ``/tracer/v1/traces`` like +fi-collector's HTTP mux but does not authenticate, stamp projects or store +anything. There is no model call: the app's model client is a stand-in. + +All keys are placeholders. Nothing here contacts an MLflow tracking server, +Databricks, a model provider or Future AGI. +""" + +from __future__ import annotations + +import contextlib +import importlib.util +import json +import os +import re +import sqlite3 +import sys +import threading +from http import HTTPStatus +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from typing import Any, Callable, Iterator, Optional +from urllib.parse import quote, urlsplit + +import pytest + +from harness import Receiver, post_otlp, run + +TESTS_DIR = Path(__file__).resolve().parent +RECIPE_DIR = TESTS_DIR.parent +APP = RECIPE_DIR / "src" / "app.py" +README = RECIPE_DIR / "README.md" +BARE_SPAN = TESTS_DIR / "bare_span.py" +SET_DESTINATION = TESTS_DIR / "set_destination_app.py" +GUARD = TESTS_DIR / "_guarded_run.py" +GUARD_PROBE = TESTS_DIR / "_guard_probe.py" +GEN_AI_FIXTURE = TESTS_DIR / "fixtures" / "gen_ai_span.otlp.json" + +# A comma and a space: the project name must be percent-encoded to arrive. +PROJECT = "MLflow recipe, contract" +SERVICE_NAME = "mlflow-recipe" +FI_API_KEY = "fi-api-placeholder-0000" +# A comma and an equals sign: the recipe's percent-encoding must carry them. +FI_SECRET_KEY = "fi-secret-placeholder,part=0000" + +# Content markers: the question (input), the system prompt built from the +# policy (input) and the stand-in model's answer (output). +QUESTION = "QMARK7c1e what is the refund window?" +POLICY = "Refunds are accepted within 30 days of purchase." +ANSWER = "Refunds are accepted for 30 days." # src/app.py call_model() +CONTENT_MARKERS = ("QMARK7c1e", POLICY, ANSWER) + +ROOT = "answer_question" +LLM = "chat_model" +MODEL = "gpt-4o-mini" +# The keys MLflow 3.16.1 puts on each span of src/app.py by default. +DEFAULT_KEYS = { + ROOT: { + "mlflow.traceRequestId", + "mlflow.spanType", + "mlflow.spanInputs", + "mlflow.spanOutputs", + "mlflow.spanLogLevel", + }, + LLM: { + "mlflow.traceRequestId", + "mlflow.spanType", + "mlflow.spanInputs", + "mlflow.spanOutputs", + "mlflow.spanLogLevel", + "mlflow.llm.model", + "mlflow.llm.provider", + "mlflow.chat.tokenUsage", + "gen_ai.request.model", + "gen_ai.provider.name", + "gen_ai.usage.input_tokens", + "gen_ai.usage.output_tokens", + }, +} +# A hang guard, not a speed check: importing mlflow takes a few seconds. +RUN_TIMEOUT_SECONDS = 180 + + +def otlp_headers(api_key: str, secret_key: str) -> str: + """Build OTEL_EXPORTER_OTLP_TRACES_HEADERS exactly as README.md tells users to.""" + return "x-api-key={0},x-secret-key={1}".format( + quote(api_key, safe=""), quote(secret_key, safe="") + ) + + +def resource_attributes(project: str) -> str: + """Build OTEL_RESOURCE_ATTRIBUTES, percent-encoding the project name as README.md says.""" + return "project_name={0},project_type=observe".format(quote(project, safe="")) + + +class _Recorder: + """Loopback catch-all that records every request's method and path. + + It answers POST /v1/traces and /tracer/v1/traces with 200, the two trace + routes fi-collector serves (pkg/server/server.go:233-234), and anything + else with 404. + """ + + def __init__(self) -> None: + self.calls: list[tuple[str, str]] = [] + lock = threading.Lock() + owner = self + + class Handler(BaseHTTPRequestHandler): + def _answer(self) -> None: + path = urlsplit(self.path).path + self.rfile.read(int(self.headers.get("Content-Length", "0"))) + with lock: + owner.calls.append((self.command, path)) + ok = self.command == "POST" and path in ("/v1/traces", "/tracer/v1/traces") + self.send_response(HTTPStatus.OK if ok else HTTPStatus.NOT_FOUND) + self.send_header("Content-Length", "0") + self.end_headers() + + do_GET = do_POST = _answer # noqa: N815 + + def log_message(self, format: str, *args: Any) -> None: # noqa: A002 + return + + self._server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + self.origin = "http://127.0.0.1:{0}".format(self._server.server_port) + self._thread = threading.Thread(target=self._server.serve_forever, daemon=True) + self._thread.start() + + def __enter__(self) -> "_Recorder": + return self + + def __exit__(self, *_exc: Any) -> None: + self._server.shutdown() + self._server.server_close() + self._thread.join() + + +@contextlib.contextmanager +def _working_directory(path: Path) -> Iterator[None]: + # harness.run() has no cwd argument; MLflow resolves its default store + # (./mlflow.db) against the working directory, so the child inherits ours. + previous = os.getcwd() + os.chdir(path) + try: + yield + finally: + os.chdir(previous) + + +def _child_env(work: Path, guard_log: Path, endpoint: str, **overrides: Optional[str]) -> dict[str, str]: + """The recipe's environment, built from scratch (nothing inherited but PATH).""" + env = { + "PATH": os.environ.get("PATH", ""), + "HOME": str(work), + "LOOPBACK_GUARD_LOG": str(guard_log), + "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT": endpoint, + "OTEL_EXPORTER_OTLP_TRACES_PROTOCOL": "http/protobuf", + "OTEL_EXPORTER_OTLP_TRACES_HEADERS": otlp_headers(FI_API_KEY, FI_SECRET_KEY), + "OTEL_RESOURCE_ATTRIBUTES": resource_attributes(PROJECT), + "OTEL_SERVICE_NAME": SERVICE_NAME, + "MLFLOW_DISABLE_TELEMETRY": "true", + } + for key, value in overrides.items(): + if value is None: + env.pop(key, None) + else: + env[key] = value + return env + + +def _execute( + tmp_path: Path, + script: Path, + args: list[str], + endpoint: str, + **overrides: Optional[str], +) -> dict[str, Any]: + """Run one script under the loopback guard in tmp_path/work; return what the tests read.""" + work = tmp_path / "work" + work.mkdir(parents=True, exist_ok=True) + guard_log = tmp_path / "guard.jsonl" + env = _child_env(work, guard_log, endpoint, **overrides) + with _working_directory(work): + result = run( + [sys.executable, str(GUARD), str(script), *args], + env=env, + stdin=None, + timeout=RUN_TIMEOUT_SECONDS, + ) + return { + "result": result, + "stdout": result.stdout.decode("utf-8", "replace"), + "stderr": result.stderr.decode("utf-8", "replace"), + "work": work, + "files": sorted(p.relative_to(work).as_posix() for p in work.rglob("*") if p.is_file()), + "guard_attempts": ( + [json.loads(line) for line in guard_log.read_text().splitlines()] + if guard_log.exists() + else [] + ), + } + + +def _run_script( + tmp_path: Path, + script: Path, + args: list[str], + endpoint_for: Callable[[Receiver], str] = lambda receiver: receiver.collector_endpoint, + **overrides: Optional[str], +) -> dict[str, Any]: + """Run a script against a harness Receiver.""" + with Receiver() as receiver: + record = _execute(tmp_path, script, args, endpoint_for(receiver), **overrides) + record["requests"] = receiver.requests() + record["spans"] = receiver.spans() + return record + + +def _run_recorded( + tmp_path: Path, + script: Path, + args: list[str], + endpoint_for: Callable[[_Recorder], str], + **overrides: Optional[str], +) -> dict[str, Any]: + """Run a script against the catch-all recorder, which sees every request.""" + with _Recorder() as recorder: + record = _execute(tmp_path, script, args, endpoint_for(recorder), **overrides) + record["calls"] = list(recorder.calls) + return record + + +def _value(value: dict[str, Any]) -> Any: + for kind in ("stringValue", "boolValue", "doubleValue"): + if kind in value: + return value[kind] + if "intValue" in value: + return int(value["intValue"]) + return value + + +def _raw(span: dict[str, Any]) -> dict[str, dict[str, Any]]: + """Attribute key -> OTLP value object, e.g. {"stringValue": "..."}.""" + return {item["key"]: item.get("value", {}) for item in span.get("attributes", [])} + + +def _attrs(span: dict[str, Any]) -> dict[str, Any]: + return {key: _value(value) for key, value in _raw(span).items()} + + +def _by_name(spans: list[dict[str, Any]]) -> dict[str, dict[str, Any]]: + named = {span["name"]: span for span in spans} + assert len(named) == len(spans), "span names repeat: {0}".format( + sorted(span["name"] for span in spans) + ) + return named + + +def _export_dump(record: dict[str, Any]) -> str: + """Everything the export carried except HTTP headers: spans and resources.""" + return json.dumps( + { + "spans": record["spans"], + "resources": [r["resource_attributes"] for r in record["requests"]], + }, + sort_keys=True, + ) + + +def _store(path: Path) -> dict[str, Any]: + """The trace ids and span names in an MLflow SQLite store.""" + connection = sqlite3.connect(str(path)) + try: + return { + "traces": [row[0] for row in connection.execute("select request_id from trace_info")], + "spans": sorted(row[0] for row in connection.execute("select name from spans")), + } + finally: + connection.close() + + +def _trace_request_id(record: dict[str, Any]) -> str: + ids = {json.loads(_attrs(span)["mlflow.traceRequestId"]) for span in record["spans"]} + assert len(ids) == 1, ids + return ids.pop() + + +def _assert_exited_cleanly(record: dict[str, Any]) -> None: + result = record["result"] + assert not result.timed_out, record["stderr"] + assert result.returncode == 0, record["stderr"] + assert record["guard_attempts"] == [], record["guard_attempts"] + + +def _assert_ran(record: dict[str, Any]) -> None: + _assert_exited_cleanly(record) + assert ANSWER in record["stdout"] + assert record["requests"], "no export reached the receiver" + assert record["spans"], "the export carried no spans" + + +# -------------------------------------------------------------------------- +# The recipe as documented: full /tracer/v1/traces URL, http/protobuf, +# percent-encoded headers and resource attributes, telemetry off. +# -------------------------------------------------------------------------- + + +@pytest.fixture(scope="module") +def recipe_run(tmp_path_factory: pytest.TempPathFactory) -> dict[str, Any]: + record = _run_script(tmp_path_factory.mktemp("recipe"), APP, [QUESTION]) + _assert_ran(record) + return record + + +def test_recipe_runs_with_loopback_network_only_and_writes_no_file( + recipe_run: dict[str, Any], +) -> None: + assert recipe_run["guard_attempts"] == [] + assert ANSWER in recipe_run["stdout"] + # No ./mlflow.db, no ./mlruns, no ~/.config/mlflow: OTLP only. + assert recipe_run["files"] == [] + + +def test_guard_refuses_and_logs_non_loopback_connections(tmp_path: Path) -> None: + """Positive control for every ``guard_attempts == []`` assertion in this file.""" + guard_log = tmp_path / "guard.jsonl" + result = run( + [sys.executable, str(GUARD), str(GUARD_PROBE)], + env={"PATH": os.environ.get("PATH", ""), "HOME": str(tmp_path), "LOOPBACK_GUARD_LOG": str(guard_log)}, + stdin=None, + timeout=60, + ) + assert not result.timed_out + assert result.returncode == 0, result.stderr.decode("utf-8", "replace") + assert "refused: connect,connect_ex,getaddrinfo" in result.stdout.decode("utf-8", "replace") + logged = [json.loads(line) for line in guard_log.read_text().splitlines()] + assert [entry["kind"] for entry in logged] == ["connect", "connect_ex", "getaddrinfo"] + assert "192.0.2.1" in logged[0]["target"] + assert "2001:db8::1" in logged[1]["target"] + assert "example.invalid" in logged[2]["target"] + + +def test_traces_endpoint_is_posted_to_as_is(recipe_run: dict[str, Any]) -> None: + assert {request["path"] for request in recipe_run["requests"]} == {"/tracer/v1/traces"} + + +def test_headers_carry_the_future_agi_keys(recipe_run: dict[str, Any]) -> None: + for request in recipe_run["requests"]: + headers = request["headers"] + assert headers.get("x-api-key") == FI_API_KEY + assert headers.get("x-secret-key") == FI_SECRET_KEY + assert headers.get("content-type") == "application/x-protobuf" + # No MLflow tracking token or Basic auth on the export. + assert "authorization" not in headers + + +def test_resource_carries_the_project(recipe_run: dict[str, Any]) -> None: + for request in recipe_run["requests"]: + assert request["resource_attributes"], "export without a resource" + for resource in request["resource_attributes"]: + assert resource.get("project_name") == PROJECT + assert resource.get("project_type") == "observe" + assert resource.get("service.name") == SERVICE_NAME + assert resource.get("telemetry.sdk.name") == "mlflow" + assert resource.get("telemetry.sdk.version") == "3.16.1" + + +def test_span_tree(recipe_run: dict[str, Any]) -> None: + spans = _by_name(recipe_run["spans"]) + assert set(spans) == {ROOT, LLM} + assert len({span["traceId"] for span in spans.values()}) == 1 + assert not spans[ROOT].get("parentSpanId") + assert spans[LLM]["parentSpanId"] == spans[ROOT]["spanId"] + for span in spans.values(): + assert span["kind"] == "SPAN_KIND_INTERNAL" + assert span["status"] == {"code": "STATUS_CODE_OK"} + assert not span.get("events") + + +def test_mlflow_json_encodes_every_attribute_value(recipe_run: dict[str, Any]) -> None: + spans = _by_name(recipe_run["spans"]) + assert {name: set(_raw(span)) for name, span in spans.items()} == DEFAULT_KEYS + for span in spans.values(): + for key, value in _raw(span).items(): + assert set(value) == {"stringValue"}, (key, value) + json.loads(value["stringValue"]) + llm = _raw(spans[LLM]) + # MLflow lets the app set the gen_ai.* keys, but encodes the values. + assert llm["gen_ai.request.model"] == {"stringValue": '"gpt-4o-mini"'} + assert llm["gen_ai.provider.name"] == {"stringValue": '"openai"'} + assert llm["gen_ai.usage.input_tokens"] == {"stringValue": "12"} + assert llm["gen_ai.usage.output_tokens"] == {"stringValue": "7"} + assert llm["mlflow.spanType"] == {"stringValue": '"CHAT_MODEL"'} + assert json.loads(llm["mlflow.chat.tokenUsage"]["stringValue"]) == { + "input_tokens": 12, + "output_tokens": 7, + "total_tokens": 19, + } + assert _raw(spans[ROOT])["mlflow.spanType"] == {"stringValue": '"CHAIN"'} + assert _trace_request_id(recipe_run).startswith("tr-") + + +def test_readme_lists_every_exported_key(recipe_run: dict[str, Any]) -> None: + text = README.read_text(encoding="utf-8") + keys = set().union(*DEFAULT_KEYS.values()) + for request in recipe_run["requests"]: + for resource in request["resource_attributes"]: + keys |= set(resource) + missing = sorted(key for key in keys if "`{0}`".format(key) not in text) + assert missing == [] + + +def test_no_experiment_or_session_on_the_export(recipe_run: dict[str, Any]) -> None: + for span in recipe_run["spans"]: + keys = set(_attrs(span)) + assert not [key for key in keys if "experiment" in key.lower()] + assert not keys & {"session.id", "user.id"} + for request in recipe_run["requests"]: + for resource in request["resource_attributes"]: + assert not [key for key in resource if "experiment" in key.lower()] + + +def test_content_is_exported_by_default(recipe_run: dict[str, Any]) -> None: + spans = {name: _attrs(span) for name, span in _by_name(recipe_run["spans"]).items()} + assert "QMARK7c1e" in spans[ROOT]["mlflow.spanInputs"] + assert ANSWER in spans[ROOT]["mlflow.spanOutputs"] + assert POLICY in spans[LLM]["mlflow.spanInputs"] # the system prompt + assert "QMARK7c1e" in spans[LLM]["mlflow.spanInputs"] + assert ANSWER in spans[LLM]["mlflow.spanOutputs"] + # Only those two keys carry content. + for name, attrs in spans.items(): + for key, value in attrs.items(): + if key in ("mlflow.spanInputs", "mlflow.spanOutputs"): + continue + for marker in CONTENT_MARKERS: + assert marker not in str(value), (name, key) + + +def test_no_secret_in_the_export_or_output(recipe_run: dict[str, Any]) -> None: + dump = _export_dump(recipe_run) + output = recipe_run["stdout"] + recipe_run["stderr"] + for secret in (FI_API_KEY, FI_SECRET_KEY, quote(FI_SECRET_KEY, safe="")): + assert secret not in dump + assert secret not in output + + +# -------------------------------------------------------------------------- +# Content: MLflow has no switch; its span-processor hook is the way. +# -------------------------------------------------------------------------- + + +@pytest.mark.parametrize("semconv", [None, "true"]) +def test_drop_content_processor_removes_inputs_and_outputs( + tmp_path: Path, semconv: Optional[str] +) -> None: + record = _run_script( + tmp_path, + APP, + ["--drop-content", QUESTION], + MLFLOW_ENABLE_OTEL_GENAI_SEMCONV=semconv, + ) + _assert_ran(record) + dump = _export_dump(record) + for marker in CONTENT_MARKERS: + assert marker not in dump, marker + assert len(record["spans"]) == 2 + for span in record["spans"]: + attrs = _attrs(span) + if semconv is None: + assert attrs["mlflow.spanInputs"] == '"[REDACTED]"' + assert attrs["mlflow.spanOutputs"] == '"[REDACTED]"' + else: + assert "gen_ai.input.messages" not in attrs + assert "gen_ai.output.messages" not in attrs + assert "gen_ai.system_instructions" not in attrs + + +# -------------------------------------------------------------------------- +# Endpoint, protocol and the settings src/app.py refuses. +# -------------------------------------------------------------------------- + + +def test_v1_traces_is_accepted_too(tmp_path: Path) -> None: + record = _run_script(tmp_path, APP, [QUESTION], lambda receiver: receiver.endpoint) + _assert_ran(record) + assert {request["path"] for request in record["requests"]} == {"/v1/traces"} + assert {span["name"] for span in record["spans"]} == {ROOT, LLM} + + +@pytest.mark.parametrize( + ("suffix", "posted_to"), + [("", "/"), ("/", "/"), ("/tracer", "/tracer")], +) +def test_mlflow_does_not_append_v1_traces(tmp_path: Path, suffix: str, posted_to: str) -> None: + record = _run_recorded( + tmp_path, BARE_SPAN, [QUESTION], lambda recorder: recorder.origin + suffix + ) + _assert_exited_cleanly(record) + assert record["calls"] == [("POST", posted_to)] + assert "Failed to export spans batch code: 404" in record["stderr"] + + +def test_without_an_endpoint_mlflow_writes_a_local_mlflow_db(tmp_path: Path) -> None: + record = _run_recorded( + tmp_path, + BARE_SPAN, + [QUESTION], + lambda recorder: recorder.origin + "/tracer/v1/traces", + OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=None, + MLFLOW_MODEL_CATALOG_URI="", + ) + _assert_exited_cleanly(record) + assert record["calls"] == [] + assert record["files"] == ["mlflow.db"] + assert _store(record["work"] / "mlflow.db")["spans"] == [ROOT, LLM] + + +def test_without_the_protocol_nothing_is_exported_or_logged(tmp_path: Path) -> None: + """MLflow defaults to gRPC; that exporter is not installed, so export is skipped.""" + record = _run_recorded( + tmp_path, + BARE_SPAN, + [QUESTION], + lambda recorder: recorder.origin + "/tracer/v1/traces", + OTEL_EXPORTER_OTLP_TRACES_PROTOCOL=None, + ) + _assert_exited_cleanly(record) + assert ANSWER in record["stdout"] + assert record["calls"] == [] + assert record["stderr"] == "" + assert record["files"] == [] + + +@pytest.mark.parametrize( + ("extra", "expected_calls", "expected_log"), + [ + ( + {"OTEL_EXPORTER_OTLP_ENDPOINT": "{origin}"}, + [], + "No module named 'opentelemetry.exporter.otlp.proto.grpc'", + ), + ( + {"OTEL_EXPORTER_OTLP_METRICS_ENDPOINT": "{origin}/v1/metrics"}, + [], + "No module named 'opentelemetry.exporter.otlp.proto.grpc'", + ), + ( + {"OTEL_EXPORTER_OTLP_ENDPOINT": "{origin}", "OTEL_EXPORTER_OTLP_PROTOCOL": "http/protobuf"}, + [("POST", "/tracer/v1/traces"), ("POST", "/v1/metrics")], + "Failed to export metrics batch code: 404", + ), + ], + ids=["base-endpoint", "metrics-endpoint", "base-endpoint-http"], +) +def test_a_metrics_endpoint_turns_on_metrics_export( + tmp_path: Path, extra: dict[str, str], expected_calls: list[tuple[str, str]], expected_log: str +) -> None: + with _Recorder() as recorder: + overrides = {key: value.format(origin=recorder.origin) for key, value in extra.items()} + record = _execute( + tmp_path, BARE_SPAN, [QUESTION], recorder.origin + "/tracer/v1/traces", **overrides + ) + calls = sorted(recorder.calls) + _assert_exited_cleanly(record) + assert calls == sorted(expected_calls) + assert expected_log in record["stderr"] + + +@pytest.mark.parametrize( + ("overrides", "message"), + [ + ({"OTEL_EXPORTER_OTLP_TRACES_ENDPOINT": None}, "full traces URL"), + ({"OTEL_EXPORTER_OTLP_TRACES_ENDPOINT": "http://127.0.0.1:9"}, "full traces URL"), + ({"OTEL_EXPORTER_OTLP_TRACES_PROTOCOL": None}, "OTEL_EXPORTER_OTLP_TRACES_PROTOCOL=http/protobuf"), + ({"OTEL_EXPORTER_OTLP_TRACES_PROTOCOL": "grpc"}, "OTEL_EXPORTER_OTLP_TRACES_PROTOCOL=http/protobuf"), + ({"OTEL_EXPORTER_OTLP_ENDPOINT": "http://127.0.0.1:9"}, "Unset OTEL_EXPORTER_OTLP_ENDPOINT"), + ( + {"OTEL_EXPORTER_OTLP_METRICS_ENDPOINT": "http://127.0.0.1:9/v1/metrics"}, + "Unset OTEL_EXPORTER_OTLP_METRICS_ENDPOINT", + ), + ({"OTEL_RESOURCE_ATTRIBUTES": None}, "Add project_name="), + ({"OTEL_RESOURCE_ATTRIBUTES": "project_type=observe"}, "Add project_name="), + ({"OTEL_RESOURCE_ATTRIBUTES": "project_name=a,project_type=observe,"}, "key=value pairs"), + ], + ids=[ + "no-endpoint", + "origin-only", + "no-protocol", + "grpc", + "base-endpoint", + "metrics-endpoint", + "no-resource", + "no-project", + "trailing-comma", + ], +) +def test_app_refuses_settings_that_export_nothing( + tmp_path: Path, overrides: dict[str, Optional[str]], message: str +) -> None: + record = _run_script(tmp_path, APP, [QUESTION], **overrides) + assert not record["result"].timed_out + assert record["result"].returncode != 0 + assert message in record["stderr"] + assert record["requests"] == [] + assert record["guard_attempts"] == [] + assert record["files"] == [] + + +def test_unencoded_comma_in_a_header_value_truncates_it(tmp_path: Path) -> None: + """Why the recipe percent-encodes: the exporter splits the variable on commas.""" + raw = "x-api-key={0},x-secret-key={1}".format(FI_API_KEY, FI_SECRET_KEY) + record = _run_script( + tmp_path, APP, [QUESTION], OTEL_EXPORTER_OTLP_TRACES_HEADERS=raw + ) + _assert_ran(record) + for request in record["requests"]: + assert request["headers"].get("x-api-key") == FI_API_KEY + assert request["headers"].get("x-secret-key") == FI_SECRET_KEY.split(",")[0] + + +@pytest.mark.parametrize( + "resource", + [None, "project_name=mlflow-recipe,project_type=observe,"], + ids=["unset", "trailing-comma"], +) +def test_resource_without_project_name(tmp_path: Path, resource: Optional[str]) -> None: + """fi-collector rejects such a batch (pkg/auth/stamp.go); src/app.py refuses both.""" + record = _run_script( + tmp_path, + BARE_SPAN, + [QUESTION], + OTEL_RESOURCE_ATTRIBUTES=resource, + OTEL_SERVICE_NAME=None, + ) + _assert_ran(record) + for request in record["requests"]: + for attributes in request["resource_attributes"]: + assert "project_name" not in attributes + assert "project_type" not in attributes + assert attributes.get("telemetry.sdk.name") == "mlflow" + + +def test_experiment_id_is_exported_only_as_a_resource_attribute(tmp_path: Path) -> None: + record = _run_script( + tmp_path, + APP, + [QUESTION], + MLFLOW_EXPERIMENT_ID="42", + OTEL_RESOURCE_ATTRIBUTES=resource_attributes(PROJECT) + ",mlflow.experiment_id=42", + ) + _assert_ran(record) + assert record["files"] == [] + for request in record["requests"]: + for resource in request["resource_attributes"]: + assert resource.get("mlflow.experiment_id") == "42" + assert resource.get("project_name") == PROJECT + for span in record["spans"]: + keys = set(_attrs(span)) + # MLFLOW_EXPERIMENT_ID itself reaches no span, and no session is set. + assert not [key for key in keys if "experiment" in key.lower()] + assert "session.id" not in keys + + +# -------------------------------------------------------------------------- +# Dual export and set_destination. +# -------------------------------------------------------------------------- + + +def test_dual_export_false_is_otlp_only(tmp_path: Path) -> None: + record = _run_script( + tmp_path, APP, [QUESTION], MLFLOW_TRACE_ENABLE_OTLP_DUAL_EXPORT="false" + ) + _assert_ran(record) + assert {span["name"] for span in record["spans"]} == {ROOT, LLM} + assert record["files"] == [] + + +def test_dual_export_without_a_tracking_uri_writes_a_local_mlflow_db(tmp_path: Path) -> None: + record = _run_script( + tmp_path, + APP, + [QUESTION], + MLFLOW_TRACE_ENABLE_OTLP_DUAL_EXPORT="true", + MLFLOW_MODEL_CATALOG_URI="", + ) + _assert_ran(record) + assert {span["name"] for span in record["spans"]} == {ROOT, LLM} + # MLflow's default store, SQLite in the working directory. + assert record["files"] == ["mlflow.db"] + store = _store(record["work"] / "mlflow.db") + assert store == {"traces": [_trace_request_id(record)], "spans": [ROOT, LLM]} + + +def test_dual_export_with_a_tracking_uri_writes_there(tmp_path: Path) -> None: + database = tmp_path / "work" / "traces.db" + record = _run_script( + tmp_path, + APP, + [QUESTION], + MLFLOW_TRACE_ENABLE_OTLP_DUAL_EXPORT="true", + MLFLOW_TRACKING_URI="sqlite:///" + database.as_posix(), # four slashes: absolute + MLFLOW_MODEL_CATALOG_URI="", + ) + _assert_ran(record) + assert {span["name"] for span in record["spans"]} == {ROOT, LLM} + assert record["files"] == ["traces.db"] + assert _store(database) == {"traces": [_trace_request_id(record)], "spans": [ROOT, LLM]} + + +def test_dual_export_fetches_the_model_catalog_unless_disabled(tmp_path: Path) -> None: + """Writing to an MLflow store fetches MLflow's model catalog (default: github.com). + + The catalog URI is pointed at a loopback recorder, so the fetch is seen + without any lookup of the real host. + """ + with Receiver() as receiver, _Recorder() as catalog: + record = _execute( + tmp_path, + APP, + [QUESTION], + receiver.collector_endpoint, + MLFLOW_TRACE_ENABLE_OTLP_DUAL_EXPORT="true", + MLFLOW_MODEL_CATALOG_URI=catalog.origin + "/catalog", + ) + spans = receiver.spans() + catalog_calls = list(catalog.calls) + _assert_exited_cleanly(record) + assert {span["name"] for span in spans} == {ROOT, LLM} + assert catalog_calls + assert set(catalog_calls) == {("GET", "/catalog/openai.json")} + + +def test_set_destination_silences_otlp(tmp_path: Path) -> None: + record = _run_recorded( + tmp_path, + SET_DESTINATION, + [QUESTION], + lambda recorder: recorder.origin + "/tracer/v1/traces", + MLFLOW_MODEL_CATALOG_URI="", + ) + _assert_exited_cleanly(record) + assert ANSWER in record["stdout"] + # Not one request of any kind reached the OTLP endpoint ... + assert record["calls"] == [] + # ... because the trace went to the destination: the default local store. + assert record["files"] == ["mlflow.db"] + store = _store(record["work"] / "mlflow.db") + assert len(store["traces"]) == 1 + assert store["spans"] == [ROOT, LLM] + + +def test_set_destination_with_dual_export_sends_both(tmp_path: Path) -> None: + record = _run_script( + tmp_path, + SET_DESTINATION, + [QUESTION], + MLFLOW_TRACE_ENABLE_OTLP_DUAL_EXPORT="true", + MLFLOW_MODEL_CATALOG_URI="", + ) + _assert_ran(record) + assert {span["name"] for span in record["spans"]} == {ROOT, LLM} + assert record["files"] == ["mlflow.db"] + assert _store(record["work"] / "mlflow.db") == { + "traces": [_trace_request_id(record)], + "spans": [ROOT, LLM], + } + + +@pytest.mark.parametrize("disabled", [None, "true"]) +def test_telemetry_switch(tmp_path: Path, disabled: Optional[str]) -> None: + record = _run_script(tmp_path, APP, [QUESTION], MLFLOW_DISABLE_TELEMETRY=disabled) + _assert_ran(record) + # With telemetry on, importing mlflow writes an installation id to HOME. + expected = [] if disabled else [".config/mlflow/telemetry.json"] + assert record["files"] == expected + + +# -------------------------------------------------------------------------- +# MLFLOW_ENABLE_OTEL_GENAI_SEMCONV=true: MLflow's own GenAI translation. +# -------------------------------------------------------------------------- + + +@pytest.fixture(scope="module") +def semconv_run(tmp_path_factory: pytest.TempPathFactory) -> dict[str, Any]: + record = _run_script( + tmp_path_factory.mktemp("semconv"), + APP, + [QUESTION], + MLFLOW_ENABLE_OTEL_GENAI_SEMCONV="true", + ) + _assert_ran(record) + return record + + +def test_semconv_exports_plain_gen_ai_values(semconv_run: dict[str, Any]) -> None: + spans = _by_name(semconv_run["spans"]) + chat_name = "chat " + MODEL + # The model span is renamed " "; a span with a type but + # no model is renamed to its type. + assert set(spans) == {chat_name, "CHAIN"} + chat, chain = spans[chat_name], spans["CHAIN"] + assert chat["parentSpanId"] == chain["spanId"] + assert chat["kind"] == "SPAN_KIND_CLIENT" + assert chain["kind"] == "SPAN_KIND_INTERNAL" + raw = _raw(chat) + assert raw["gen_ai.operation.name"] == {"stringValue": "chat"} + assert raw["gen_ai.request.model"] == {"stringValue": MODEL} + assert raw["gen_ai.provider.name"] == {"stringValue": "openai"} + assert raw["gen_ai.usage.input_tokens"] == {"intValue": "12"} + assert raw["gen_ai.usage.output_tokens"] == {"intValue": "7"} + assert set(raw) == { + "gen_ai.operation.name", + "gen_ai.request.model", + "gen_ai.provider.name", + "gen_ai.usage.input_tokens", + "gen_ai.usage.output_tokens", + "gen_ai.system_instructions", + "gen_ai.input.messages", + "gen_ai.output.messages", + } + # Every mlflow.* key is dropped, the CHAIN span's inputs and outputs too. + assert _raw(chain) == {"gen_ai.operation.name": {"stringValue": "CHAIN"}} + + +def test_semconv_moves_chat_content_to_gen_ai_messages(semconv_run: dict[str, Any]) -> None: + attrs = _attrs(_by_name(semconv_run["spans"])["chat " + MODEL]) + assert "QMARK7c1e" in attrs["gen_ai.input.messages"] + assert POLICY in attrs["gen_ai.system_instructions"] # the system message + assert POLICY not in attrs["gen_ai.input.messages"] + assert ANSWER in attrs["gen_ai.output.messages"] + dump = json.dumps(semconv_run["spans"]) + for marker in CONTENT_MARKERS: + assert dump.count(marker) == 1, marker + + +def test_readme_lists_every_semconv_key(semconv_run: dict[str, Any]) -> None: + text = README.read_text(encoding="utf-8") + keys = set().union(*(set(_raw(span)) for span in semconv_run["spans"])) + assert sorted(key for key in keys if "`{0}`".format(key) not in text) == [] + + +# -------------------------------------------------------------------------- +# The hand-built gen_ai.* fixture, posted with the harness post_otlp(). +# -------------------------------------------------------------------------- + + +def _fixture() -> dict[str, Any]: + return json.loads(GEN_AI_FIXTURE.read_text(encoding="utf-8")) + + +def _fixture_spans() -> dict[str, dict[str, Any]]: + payload = _fixture() + return _by_name(payload["resourceSpans"][0]["scopeSpans"][0]["spans"]) + + +def test_hand_built_gen_ai_span_arrives_with_keys_intact() -> None: + payload = _fixture() + with Receiver() as receiver: + assert post_otlp(payload, receiver.collector_endpoint) == 200 + spans = receiver.spans() + requests = receiver.requests() + assert spans == payload["resourceSpans"][0]["scopeSpans"][0]["spans"] + assert [request["path"] for request in requests] == ["/tracer/v1/traces"] + assert requests[0]["resource_attributes"] == [ + {"project_name": "mlflow-recipe-contract", "project_type": "observe", "service.name": "mlflow-recipe"} + ] + arrived = _by_name(spans) + plain = _raw(arrived["chat " + MODEL]) + assert plain["gen_ai.request.model"] == {"stringValue": MODEL} + assert plain["gen_ai.usage.input_tokens"] == {"intValue": "12"} + encoded = _raw(arrived[LLM]) + assert encoded["gen_ai.request.model"] == {"stringValue": '"gpt-4o-mini"'} + assert encoded["gen_ai.usage.input_tokens"] == {"stringValue": "12"} + + +def test_fixture_spans_are_what_mlflow_sends( + recipe_run: dict[str, Any], semconv_run: dict[str, Any] +) -> None: + """The fixture is hand-built, but each span's keys and values are MLflow's own.""" + fixture = _fixture_spans() + live = { + "chat " + MODEL: _raw(_by_name(semconv_run["spans"])["chat " + MODEL]), + LLM: _raw(_by_name(recipe_run["spans"])[LLM]), + } + for name, span in fixture.items(): + for key, value in _raw(span).items(): + assert live[name][key] == value, (name, key) + + +# -------------------------------------------------------------------------- +# Opt-in: read fi-collector's alias tables instead of copying them. +# -------------------------------------------------------------------------- + + +def _collector_source() -> tuple[str, str]: + root = os.environ.get("FI_COLLECTOR_SRC") + if not root: + pytest.skip("set FI_COLLECTOR_SRC to a fi-collector checkout (the directory with pkg/)") + adapter = Path(root) / "pkg" / "adapter" / "adapter.go" + converter = Path(root) / "exporter" / "clickhouse25exporter" / "converter.go" + return adapter.read_text(encoding="utf-8"), converter.read_text(encoding="utf-8") + + +def _go_string_lists(source: str) -> dict[str, list[str]]: + lists = {} + for name, body in re.findall(r"(\w+)\s*=\s*\[\]string\{(.*?)\}", source, flags=re.DOTALL): + lists[name] = re.findall(r'"([^"]+)"', body) + return lists + + +def _collector_keys() -> dict[str, Any]: + adapter, converter = _collector_source() + lists = {**_go_string_lists(adapter), **_go_string_lists(converter)} + synonyms_body = re.search( + r"spanKindSynonyms\s*=\s*map\[string\]string\{(.*?)\n\}", converter, flags=re.DOTALL + ).group(1) + known_body = re.search( + r"knownObservationTypes\s*=\s*map\[string\]struct\{\}\{(.*?)\n\}", converter, flags=re.DOTALL + ).group(1) + span_constants = dict(re.findall(r'(attr\w+)\s*=\s*"([^"]+)"', converter)) + return { + "lists": lists, + "synonyms": dict(re.findall(r'"([^"]+)":\s*"([^"]+)"', synonyms_body)), + "known": set(re.findall(r'"([^"]+)":', known_body)), + "columns": set(re.findall(r'overflowAsString\(overflow, "([^"]+)"\)', converter)), + "span_constants": set(span_constants.values()), + } + + +# The adapter.go / converter.go lists whose keys fi-collector reads off a span. +_ALIAS_LISTS = ( + "modelNameKeys", + "providerKeys", + "inputTokenKeys", + "outputTokenKeys", + "totalTokenKeys", + "costTotalKeys", + "costInputKeys", + "costOutputKeys", + "spanKindAttrKeys", + "operationNameAttrKeys", +) + + +def _read_by_collector(keys: dict[str, Any]) -> set[str]: + read = set().union(*(keys["lists"][name] for name in _ALIAS_LISTS)) + return read | keys["columns"] | keys["span_constants"] + + +def test_collector_reads_the_gen_ai_keys_and_no_mlflow_key() -> None: + keys = _collector_keys() + lists = keys["lists"] + assert "gen_ai.request.model" in lists["modelNameKeys"] + assert "gen_ai.provider.name" in lists["providerKeys"] + assert "gen_ai.usage.input_tokens" in lists["inputTokenKeys"] + assert "gen_ai.usage.output_tokens" in lists["outputTokenKeys"] + assert lists["operationNameAttrKeys"] == ["gen_ai.operation.name"] + assert not [key for key in _read_by_collector(keys) if key.startswith("mlflow.")] + # How MLflow's GenAI operation names would be typed (lower-cased first). + assert keys["synonyms"]["chat"] == "llm" + assert "chain" in keys["known"] + assert "invoke_agent" not in keys["synonyms"] + assert keys["columns"] == {"input.value", "output.value"} + + +def test_readme_inventory_marks_what_the_collector_reads() -> None: + read = _read_by_collector(_collector_keys()) + rows = re.findall( + r"^\| `((?:mlflow|gen_ai)\.[\w.]+)` \|.*\| (.+?) \|$", + README.read_text(encoding="utf-8"), + flags=re.MULTILINE, + ) + assert rows, "no inventory rows found in README.md" + claims: dict[str, set[bool]] = {} + for key, last_cell in rows: + claims.setdefault(key, set()).add(not last_cell.startswith("no")) + assert {key: {key in read} for key in claims} == claims + + +def _mlflow_source(relative: str) -> str: + """Read a file of the installed mlflow package without importing mlflow.""" + spec = importlib.util.find_spec("mlflow") + assert spec is not None and spec.submodule_search_locations + root = Path(next(iter(spec.submodule_search_locations))) + return (root / relative).read_text(encoding="utf-8") + + +def test_readme_span_type_table_matches_mlflow_and_the_collector() -> None: + keys = _collector_keys() + span_module = _mlflow_source("entities/span.py") + span_type_body = re.search(r"^class SpanType\b.*?:\n(.*?)\n\S", span_module, re.M | re.S).group(1) + span_types = re.findall(r'^\s+([A-Z_]+) = "([A-Z_]+)"$', span_type_body, flags=re.MULTILINE) + translator = _mlflow_source("tracing/export/genai_semconv/translator.py") + operations = dict(re.findall(r'SpanType\.(\w+): "(\w+)"', translator)) + expected = {} + for name, value in span_types: + # translator.py passes a type with no GenAI name through as is. + operation = operations.get(name, value) + kind = operation.strip().lower() + kind = keys["synonyms"].get(kind, kind) + expected[value] = (operation, kind if kind in keys["known"] else "unknown") + rows = re.findall( + r"^\| `([A-Z_]+)` \| `(\w+)` \| ([a-z]+) \|$", + README.read_text(encoding="utf-8"), + flags=re.MULTILINE, + ) + assert {span_type: (operation, kind) for span_type, operation, kind in rows} == expected From 2e36174321fad7f3fd1cef15e75a7d510c419e33 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Mon, 5 Oct 2026 10:29:51 -0700 Subject: [PATCH 07/15] docs(examples): add the MLflow recipe README Live OTLP export only: states that historical import is not this integration and that an experiment id is a resource attribute, not a Future AGI session. Publishes one endpoint form, /tracer/v1/traces (PRD O-2: MLflow uses OTEL_EXPORTER_OTLP_TRACES_ENDPOINT as is), with the tested path table. Documents dual export both ways (no tracking URI means a local ./mlflow.db, not OTLP-only), set_destination precedence, the model-catalog fetch, telemetry, MLflow's JSON encoding of attribute values, the key inventory with what fi-collector reads (adapter.go/converter.go at 4af5338, source reading), the optional MLFLOW_ENABLE_OTEL_GENAI_SEMCONV switch and its costs, content capture and the span-processor hook, and the SF-1 gap list. Marks what was tested and what was read from source. Co-Authored-By: Claude Opus 5.5 approved: Nikhil 2026-10-03 blanket Refs: TH-8334 --- python/examples/mlflow/README.md | 445 +++++++++++++++++++++++++++++++ 1 file changed, 445 insertions(+) create mode 100644 python/examples/mlflow/README.md diff --git a/python/examples/mlflow/README.md b/python/examples/mlflow/README.md new file mode 100644 index 000000000..131c5ff0c --- /dev/null +++ b/python/examples/mlflow/README.md @@ -0,0 +1,445 @@ +# MLflow → Future AGI (recipe) + +There is no traceAI package for MLflow. MLflow already exports its own +traces over OTLP; this example points that export at fi-collector with +environment variables and adds the Future AGI project as a resource +attribute. It adds no instrumentor: a wrapper around `mlflow.start_span` +would double every span. + +This is live export only. Historical import is not this integration: there +is no tracking-server client and no import command, and traces that are +already in an MLflow tracking server, Databricks or Unity Catalog are not +sent. Only spans that end while the variables below are set are exported. + +Pinned: `mlflow==3.16.1`, the latest on PyPI on 2026-10-05 +(`Requires-Python: >=3.10`). Its wheel METADATA has no +`License-Expression`; its `License` field holds the Apache License 2.0 text +under a Databricks copyright line, it declares +`Classifier: License :: OSI Approved :: Apache Software License`, and it +ships `LICENSE.txt`. It is a dependency of this example only, not of any +traceAI package. `mlflow` 3.16.1 depends on `opentelemetry-sdk` but installs +no OTLP exporter, so `requirements.txt` adds +`opentelemetry-exporter-otlp-proto-http==1.45.0`. + +The contract test (see [Tests](#tests)) passes on Python 3.10.17, 3.11.12, +3.12.10 and 3.13.15 with `opentelemetry-api`, `opentelemetry-sdk` and +`opentelemetry-proto` 1.45.0 and `protobuf` 6.33.6, the versions uv resolved +on 2026-10-05. Python 3.14 was not tested. Statements marked "source +reading" come from the mlflow 3.16.1 wheel or from fi-collector at +future-agi `main` 4af5338 and are not exercised by the tests. No MLflow +tracking server, Databricks workspace, model provider or Future AGI endpoint +was called. + +## Run + +```bash +cd python/examples/mlflow +pip install -r requirements.txt + +export FI_API_KEY="YOUR_API_KEY" +export FI_SECRET_KEY="YOUR_SECRET_KEY" +# The full URL: MLflow does not append /v1/traces to this variable. +export OTEL_EXPORTER_OTLP_TRACES_ENDPOINT="https://YOUR_FI_COLLECTOR_ORIGIN/tracer/v1/traces" +export OTEL_EXPORTER_OTLP_TRACES_PROTOCOL="http/protobuf" # MLflow's default is gRPC +export OTEL_EXPORTER_OTLP_TRACES_HEADERS="$(python -c 'import os; from urllib.parse import quote; print("x-api-key={0},x-secret-key={1}".format(quote(os.environ["FI_API_KEY"], safe=""), quote(os.environ["FI_SECRET_KEY"], safe="")))')" +export OTEL_RESOURCE_ATTRIBUTES="project_name=my-mlflow-app,project_type=observe" +export OTEL_SERVICE_NAME="my-mlflow-app" # optional: service.name +export MLFLOW_DISABLE_TELEMETRY=true +unset OTEL_EXPORTER_OTLP_ENDPOINT OTEL_EXPORTER_OTLP_METRICS_ENDPOINT + +python src/app.py "What is the refund window?" +``` + +Set the variables before the first span starts: MLflow builds its exporter +once, on first use. `src/app.py` is two nested `mlflow.start_span` calls +around a stand-in model client that returns a canned answer, so it makes no +model call. Its `check_environment()` exits with a message on each setting +below that would make MLflow export nothing. + +## The recipe + +| Variable | Value | Why | +|---|---|---| +| `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT` | `/tracer/v1/traces` | MLflow uses it as is. | +| `OTEL_EXPORTER_OTLP_TRACES_PROTOCOL` | `http/protobuf` | MLflow defaults to gRPC. | +| `OTEL_EXPORTER_OTLP_TRACES_HEADERS` | `x-api-key=,x-secret-key=`, values percent-encoded | The headers fi-collector reads. | +| `OTEL_RESOURCE_ATTRIBUTES` | `project_name=,project_type=observe` | fi-collector needs `project_name`. | +| `OTEL_SERVICE_NAME` | optional | Otherwise `service.name` is OpenTelemetry's `unknown_service` default. | +| `MLFLOW_DISABLE_TELEMETRY` | `true` | See [Telemetry](#telemetry). | + +### Endpoint + +MLflow passes `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT` unchanged to the +OpenTelemetry HTTP exporter (`mlflow/tracing/utils/otlp.py:98-99`, `:81`), +so it must be the full traces URL. It does not append `/v1/traces`. Tested +against a loopback receiver with mlflow 3.16.1: + +| `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT` | Request path | Result | +|---|---|---| +| `http://host:port/tracer/v1/traces` | `/tracer/v1/traces` | 200, the recipe's form | +| `http://host:port/v1/traces` | `/v1/traces` | 200 | +| `http://host:port` | `/` | 404, spans lost | +| `http://host:port/` | `/` | 404, spans lost | +| `http://host:port/tracer` | `/tracer` | 404, spans lost | + +fi-collector serves both `/v1/traces` and `/tracer/v1/traces`. The recipe +uses `/tracer/v1/traces` because that is where traceAI's own exporter posts +(`python/fi_instrumentation/otel.py:719`, origin from `FI_BASE_URL`, default +`https://api.futureagi.com`), so use the same origin you give traceAI. A 404 +shows only as `Failed to export spans batch code: 404` on stderr; the app +does not fail. With the variable unset, MLflow sends nothing and writes the +trace to `./mlflow.db` instead (tested). + +Do not set `OTEL_EXPORTER_OTLP_ENDPOINT` or +`OTEL_EXPORTER_OTLP_METRICS_ENDPOINT`. MLflow derives the traces URL from the +first (appending `/v1/traces`, `otlp.py:101-102`), but either one also turns +on MLflow's OTLP metrics (`otlp.py:45-51`), whose protocol defaults to gRPC +(`mlflow/tracing/processor/otel_metrics_mixin.py:46-56`). Tested: with +either one set and no gRPC exporter installed, every span fails at +`end()` with `No module named 'opentelemetry.exporter.otlp.proto.grpc'` and +nothing is sent. With `OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf` as well, +traces arrive and the metrics go to `/v1/metrics`, which fi-collector does +not serve (404). + +### Protocol + +`OTEL_EXPORTER_OTLP_TRACES_PROTOCOL` must be `http/protobuf`. MLflow's +default is `grpc` (`otlp.py:124-136`). The gRPC exporter is not installed +here, so MLflow cannot build its exporter; `mlflow.start_span` then hands +out no-op spans and logs that only at debug level (`fluent.py:660-662`). +Tested: no request, nothing on stderr, exit code 0. + +### Auth headers + +The OpenTelemetry HTTP exporter (1.45.0) reads +`OTEL_EXPORTER_OTLP_TRACES_HEADERS` itself. It splits the value on commas +and percent-decodes each value (`opentelemetry/util/re.py:60-76` in +`opentelemetry-api`). The test shows that +`x-api-key=,x-secret-key=`, built as in the `export` line +above, arrives as the `x-api-key` and `x-secret-key` request headers that +fi-collector reads (`pkg/auth/middleware.go:45-46`), with a secret that +contains `,` and `=` intact, and no `authorization` header. An unencoded +comma cuts the secret short (tested). Do not copy the `api_key=12345` header +from MLflow's documentation; it is not a Future AGI credential. + +### Resource: project, experiment and session + +`OTEL_RESOURCE_ATTRIBUTES` carries `project_name` and `project_type` +(`observe`) on every export (tested). fi-collector rejects a batch in which +a resource has no `project_name` (`pkg/auth/stamp.go:31-44`). Percent-encode +a project name that contains a comma: write `My bot, EU` as +`project_name=My%20bot%2C%20EU` and it arrives decoded (the test uses a +name with a comma and spaces). + +MLflow re-parses this variable before OpenTelemetry reads it +(`mlflow/tracing/provider.py:693-697`, `:716-731`). If one item has no `=`, +a trailing comma included, it drops every attribute in the variable, so +`project_name` is lost (tested). `src/app.py` refuses that. + +The export also carries `service.name`, `service.instance.id`, +`telemetry.sdk.language`, `telemetry.sdk.name` (`mlflow`) and +`telemetry.sdk.version` (`3.16.1`). + +MLflow puts no experiment id on the OTLP export, not even with +`MLFLOW_EXPERIMENT_ID` set (tested). If you want one in Future AGI, add it +to `OTEL_RESOURCE_ATTRIBUTES`, for example `mlflow.experiment_id=42`: it +arrives as a resource attribute (tested). The experiment id is a resource +attribute, not a Future AGI session. Future AGI sessions come from a +`session.id` span attribute (`exporter/clickhouse25exporter/converter.go:44`, +`:507-515`), and MLflow 3.16.1 sets none on the OTLP path; it copies +`session.id` only for Databricks Unity Catalog tables +(`mlflow/tracing/processor/uc_table.py:99-100`, source reading). + +### Telemetry + +With `MLFLOW_DISABLE_TELEMETRY` unset, importing `mlflow` creates MLflow's +usage-telemetry client and writes an installation id to +`$HOME/.config/mlflow/telemetry.json` (tested). That client sends usage +events to MLflow's telemetry host once an API that records events is used +(`mlflow/telemetry/client.py:177-200`, `mlflow/telemetry/utils.py:144-177`, +source reading); no such request was seen in the tests, which run with every +non-loopback connection blocked and logged. The recipe turns it off. +`DO_NOT_TRACK=true` works too. + +## Dual export and `set_destination` + +`MLFLOW_TRACE_ENABLE_OTLP_DUAL_EXPORT` is optional. Tested: + +| `MLFLOW_TRACE_ENABLE_OTLP_DUAL_EXPORT` | Tracking URI | Result | +|---|---|---| +| unset (default) | unset | OTLP only. No file written. | +| `false` | unset | OTLP only. No file written. | +| `true` | unset | OTLP, and MLflow's default store: it creates `./mlflow.db` (SQLite) in the working directory and writes the trace there. | +| `true` | `MLFLOW_TRACKING_URI=sqlite:////abs/path/traces.db` | OTLP, and that store. | + +So dual export with no tracking URI is not OTLP-only: it writes a local +database next to your app. In 3.16.1 the default store is `./mlflow.db` +unless `./mlruns` already holds experiment data, in which case it is that +file store (`mlflow/tracking/_tracking_service/utils.py:32-63`; only the +SQLite case was tested). + +Writing to an MLflow store also makes MLflow fetch its model catalog to +price LLM spans, from `https://github.com/mlflow/mlflow/releases/download/model-catalog%2Flatest` +by default (`mlflow/environment_variables.py:1758-1766`, +`mlflow/utils/providers.py:252-297`). The test points +`MLFLOW_MODEL_CATALOG_URI` at a loopback server and sees four requests for +`/catalog/openai.json`. Set `MLFLOW_MODEL_CATALOG_URI=` (empty) to turn the +fetch off. OTLP-only export never fetches it. + +`mlflow.tracing.set_destination(...)` takes precedence over the OTLP +variables (`mlflow/tracing/provider.py:825-860`). Tested with +`MlflowExperimentLocation(experiment_id="0")`: not one request of any kind +reaches the OTLP endpoint, and the trace is in `./mlflow.db` instead. With +`MLFLOW_TRACE_ENABLE_OTLP_DUAL_EXPORT=true` as well, the trace goes to both. +Do not call `set_destination` in an app that exports to Future AGI, or turn +dual export on. + +## What is exported + +`src/app.py` exports one trace with two spans: `answer_question` +(`span_type=CHAIN`, the root) and its child `chat_model` +(`span_type=CHAT_MODEL`). Both have OTel kind `INTERNAL` and status `OK`. +MLflow's OTLP processor is an OpenTelemetry `BatchSpanProcessor` +(`mlflow/tracing/processor/otel.py:20`), and the OpenTelemetry SDK flushes it +at exit (`opentelemetry/sdk/trace/__init__.py:1283-1284`), so a short script +exports when it ends. + +MLflow JSON-encodes every attribute value before it sets it on the span +(`mlflow/entities/span.py:1456-1490`, `mlflow/tracing/utils/__init__.py:125-142`). +Every value arrives as an OTLP string (tested): a string arrives with its +quotes, a number as its digits. That applies to keys your code sets too: +MLflow lets the app set `gen_ai.*` keys, but `gen_ai.request.model` arrives +as the 13-character string `"gpt-4o-mini"`, quotes included. + +Keys on the default export (tested), and whether fi-collector reads them +(source reading, `pkg/adapter/adapter.go` and +`exporter/clickhouse25exporter/converter.go` at 4af5338): + +| Key | Spans | Value as exported | fi-collector | +|---|---|---|---| +| `mlflow.traceRequestId` | both | `"tr-<32 hex>"`, JSON-encoded | no | +| `mlflow.spanType` | both | `"CHAIN"`, `"CHAT_MODEL"`, JSON-encoded | no: kind is read only from `fi.span.kind`, `gen_ai.span.kind`, `llm.request.type`, `openinference.span.kind`, then `gen_ai.operation.name` (`converter.go:79-131`) | +| `mlflow.spanLogLevel` | both | `10` (CHAIN), `20` (CHAT_MODEL), as strings | no | +| `mlflow.spanInputs` | both | what the code passed to `set_inputs`, as JSON | no: the input column reads only `input.value` (`converter.go:343`) | +| `mlflow.spanOutputs` | both | what the code passed to `set_outputs`, as JSON | no: the output column reads only `output.value` (`converter.go:344`) | +| `mlflow.llm.model` | `chat_model` | `"gpt-4o-mini"`, JSON-encoded | no | +| `mlflow.llm.provider` | `chat_model` | `"openai"`, JSON-encoded | no | +| `mlflow.chat.tokenUsage` | `chat_model` | `{"input_tokens": 12, "output_tokens": 7, "total_tokens": 19}` | no | +| `gen_ai.request.model` | `chat_model` | `"gpt-4o-mini"`, JSON-encoded | model, copied as is, quotes included (`adapter.go:281`, `:317-323`) | +| `gen_ai.provider.name` | `chat_model` | `"openai"`, JSON-encoded | provider, copied as is, quotes included (`adapter.go:290`) | +| `gen_ai.usage.input_tokens` | `chat_model` | `12`, as a string | prompt tokens, parsed from the string (`adapter.go:298`, `:339-342`) | +| `gen_ai.usage.output_tokens` | `chat_model` | `7`, as a string | completion tokens, parsed from the string (`adapter.go:303`) | + +The `mlflow.llm.*` and `mlflow.chat.tokenUsage` keys are what MLflow's +autolog integrations set on a model call; `src/app.py` sets them by hand, +next to the `gen_ai.*` keys. With these keys, fi-collector would store the +model and provider with their quotes, fill the token columns (total derived +as 19, `adapter.go:232-234`), and store both spans as kind `unknown` +(source reading; no fi-collector was run). + +### `MLFLOW_ENABLE_OTEL_GENAI_SEMCONV=true` + +MLflow can translate its own keys to OpenTelemetry GenAI keys before OTLP +export (`mlflow/environment_variables.py:1092-1097`, +`mlflow/tracing/processor/otel.py:81-82`, +`mlflow/tracing/export/genai_semconv/translator.py`). It is off by default +and optional here. Tested with it on: + +| Key | Spans | Value as exported | fi-collector | +|---|---|---|---| +| `gen_ai.operation.name` | both | `chat`; `CHAIN` on the root (a span type with no GenAI name passes through) | kind when no kind key is set: `chat` becomes llm, `CHAIN` becomes chain (`converter.go:94-131`) | +| `gen_ai.request.model` | chat span | `gpt-4o-mini`, a plain string | model (`adapter.go:281`) | +| `gen_ai.provider.name` | chat span | `openai`, a plain string | provider (`adapter.go:290`) | +| `gen_ai.usage.input_tokens` | chat span | `12`, an integer | prompt tokens (`adapter.go:298`) | +| `gen_ai.usage.output_tokens` | chat span | `7`, an integer | completion tokens (`adapter.go:303`) | +| `gen_ai.system_instructions` | chat span | the system message, as JSON | no | +| `gen_ai.input.messages` | chat span | the other input messages, as JSON | no: the input column reads only `input.value` | +| `gen_ai.output.messages` | chat span | the output messages, as JSON | no: the output column reads only `output.value` | + +What else changes (tested): + +- Every `mlflow.*` key is dropped, including `mlflow.spanInputs` and + `mlflow.spanOutputs`. Chat-shaped inputs and outputs become the three + message keys above (`genai_semconv/converter.py:62-80`); the root span's + `{"question": ...}` and `{"answer": ...}` are dropped. +- Spans are renamed. The model span becomes `chat gpt-4o-mini`; a span with + a type but no model is renamed to its type, so `answer_question` becomes + `CHAIN` (`translator.py:164-175`). +- The model span's OTel kind becomes `CLIENT`. +- The `gen_ai.*` keys the app set itself are overwritten by the translated + values, which come from `mlflow.llm.model`, `mlflow.llm.provider` and + `mlflow.chat.tokenUsage`. A manual span that sets only `gen_ai.*` keys + keeps their JSON-encoded values. + +How MLflow's span types would be typed in Future AGI with the switch on +(source reading: the operation name from `translator.py:26-32`, then +fi-collector lower-cases it and applies `spanKindSynonyms` and +`knownObservationTypes`, `converter.go:62-131`). With the switch off every +MLflow span is `unknown`. + +| MLflow span type | `gen_ai.operation.name` | Future AGI kind | +|---|---|---| +| `CHAT_MODEL` | `chat` | llm | +| `LLM` | `generate_content` | llm | +| `EMBEDDING` | `embeddings` | embedding | +| `TOOL` | `execute_tool` | tool | +| `AGENT` | `invoke_agent` | unknown | +| `CHAIN` | `CHAIN` | chain | +| `RETRIEVER` | `RETRIEVER` | retriever | +| `RERANKER` | `RERANKER` | reranker | +| `GUARDRAIL` | `GUARDRAIL` | guardrail | +| `EVALUATOR` | `EVALUATOR` | evaluator | +| `PARSER` | `PARSER` | unknown | +| `MEMORY` | `MEMORY` | unknown | +| `WORKFLOW` | `WORKFLOW` | unknown | +| `TASK` | `TASK` | unknown | +| `UNKNOWN` | `UNKNOWN` | unknown | + +The switch is the only way found to get a model name without quotes and +typed spans out of MLflow 3.16.1 without code. Its cost is the renamed +spans and the dropped non-chat inputs and outputs. That trade-off is yours; +the recipe leaves it off. + +### Gaps for the shared processor (SF-1) + +Nothing in this example changes fi-collector or adds an alias. These are the +MLflow keys fi-collector does not map, read from the source above, for the +shared processor work: + +- `mlflow.spanType`: raw. Proposed SF-1 alias for the span kind, after JSON + decoding (types as in the table above; `AGENT` to agent). +- `mlflow.llm.model`, `mlflow.llm.provider`: raw. Proposed SF-1 model and + provider aliases, after JSON decoding. +- `mlflow.chat.tokenUsage`: raw. Proposed SF-1 token source: its + `input_tokens`, `output_tokens` and `total_tokens` fields. +- `mlflow.spanInputs`, `mlflow.spanOutputs`: raw. Proposed SF-1 input and + output source. +- JSON-encoded string values on `gen_ai.*` keys from MLflow spans (model and + provider with quotes): proposed SF-1 decoding for spans whose resource has + `telemetry.sdk.name=mlflow`. +- `gen_ai.operation.name=invoke_agent`: proposed SF-1 synonym for agent. +- `gen_ai.system_instructions`, `gen_ai.input.messages`, + `gen_ai.output.messages`: raw; not lifted into the input and output + columns. +- `mlflow.traceRequestId`, `mlflow.spanLogLevel`: raw; no Future AGI column. + +Whether fi-collector promotes any of these is not asserted here: the +harness receiver decodes and records, it does not run fi-collector. + +## Content + +MLflow has no setting that keeps tracing on and content off. For +`mlflow.start_span`, MLflow records what your code passes to `set_inputs`, +`set_outputs` and `set_attribute(s)`. In the test, the question, the system +prompt (which holds the policy text) and the answer are in +`mlflow.spanInputs` and `mlflow.spanOutputs`, and in no other key. With +`MLFLOW_ENABLE_OTEL_GENAI_SEMCONV=true` they are in +`gen_ai.system_instructions`, `gen_ai.input.messages` and +`gen_ai.output.messages` instead (tested). `@mlflow.trace` and MLflow's +autolog integrations record function arguments, return values and model +messages without being asked (source reading; not tested here). Whatever +MLflow captures lands in Future AGI. + +MLflow's own hook for removing it is a span processor: + +```python +def drop_content(span): + if span.inputs is not None: + span.set_inputs("[REDACTED]") + if span.outputs is not None: + span.set_outputs("[REDACTED]") + +mlflow.tracing.configure(span_processors=[drop_content]) +``` + +MLflow runs it when a span ends, before export (`mlflow/entities/span.py:1176-1177`). +`python src/app.py --drop-content ...` registers it. Tested with the GenAI +switch off and on: none of the question, the policy text or the answer is +anywhere in the export; with the switch off the two keys remain, holding +`"[REDACTED]"`, and with it on no message key is exported. + +Not covered by that processor (source reading, not tested): when code +inside a span raises, MLflow sets the span status to `ERROR` and adds an +`exception` event with the message and stack trace +(`mlflow/tracing/provider.py:309-311`, `mlflow/entities/span.py:1062-1073`), +and attributes your code sets with other keys are exported as they are. + +No Future AGI key or secret appears in any exported span or resource, or in +the app's output (tested); they travel only as request headers. + +## Troubleshooting + +- **No request reaches the collector, no error.** Check + `OTEL_EXPORTER_OTLP_TRACES_PROTOCOL=http/protobuf` and that + `opentelemetry-exporter-otlp-proto-http` is installed. Then check for a + `mlflow.tracing.set_destination(...)` call: it wins over the OTLP + variables. +- **`Failed to end span ...: No module named 'opentelemetry.exporter.otlp.proto.grpc'`.** + `OTEL_EXPORTER_OTLP_ENDPOINT` or `OTEL_EXPORTER_OTLP_METRICS_ENDPOINT` is + set. Unset both. +- **`Failed to export spans batch code: 404`.** The traces URL has no + `/tracer/v1/traces` (or `/v1/traces`) path. MLflow does not add it. +- **HTTP 400 from the collector, or no project.** `project_name` is missing + from `OTEL_RESOURCE_ATTRIBUTES`, or one item in it has no `=` (a trailing + comma), which makes MLflow drop them all. +- **An `mlflow.db` file appeared.** `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT` is + unset, or dual export or `set_destination` is writing to MLflow's default + local store. +- **Model name shown with quotes.** That is MLflow's JSON encoding; see + [What is exported](#what-is-exported). +- **Old traces never arrived.** Expected: this is live export, not a + backfill. + +## Tests + +`tests/test_mlflow_recipe.py` runs `src/app.py` as written, in a subprocess, +with the real `mlflow` 3.16.1. Each run gets a fresh temporary directory as +its working directory and `HOME`, so every file MLflow writes is checked. +Spans go to the shared harness `Receiver` (`python/tests/harness`), which +serves `/v1/traces` and `/tracer/v1/traces` on 127.0.0.1 like fi-collector +but does not authenticate, stamp projects or store anything. Tests that need +every request path, including 404s and the model catalog, use a loopback +catch-all instead. `tests/_guarded_run.py` blocks and logs every +non-loopback connection and every test asserts its log is empty; a positive +control (`tests/_guard_probe.py`) proves it refuses and logs IPv4, IPv6 and +DNS attempts. All keys are placeholders. + +`tests/bare_span.py` (the app's spans without `check_environment()`) and +`tests/set_destination_app.py` (the app after a `set_destination` call) are +fixtures, not part of the recipe. `tests/fixtures/gen_ai_span.otlp.json` is +a hand-built OTLP request with two `gen_ai.*` spans, one in the shape MLflow +sends with the GenAI switch on and one in the default JSON-encoded shape. +The test posts it with the harness `post_otlp()` and checks that it arrives +with `gen_ai.request.model` and `gen_ai.usage.input_tokens` intact, and +checks each fixture value against a live MLflow run. + +From the repository root (Python 3.12 shown): + +```bash +env -u PYTHONPATH PYTHONPATH="python:python/tests" \ + uv run --no-project --python 3.12 \ + --with pytest --with protobuf --with opentelemetry-proto \ + --with wrapt --with jsonschema \ + --with 'mlflow==3.16.1' --with 'opentelemetry-exporter-otlp-proto-http==1.45.0' \ + pytest python/examples/mlflow/tests -q -p no:cacheprovider \ + --noconftest -o addopts= -rfEs +``` + +For Python 3.10, 3.11 or 3.13, replace `--python 3.12`. `wrapt` and +`jsonschema` are not used by the recipe: pytest imports the repository's +`python/__init__.py` (traceAI) while collecting, and that import needs them. + +Three tests are opt-in. They read fi-collector's alias tables from its +source, check that the `gen_ai.*` keys above are in them and no `mlflow.*` +key is, and check the "fi-collector" column of the key tables, and the span +type table (against MLflow's installed `translator.py` too), against them. +Point `FI_COLLECTOR_SRC` at a fi-collector checkout (the directory that +holds `pkg/` and `exporter/`): + +```bash +export FI_COLLECTOR_SRC=/fi-collector # then run the command above +``` + +Without it they are skipped. No CI job runs this example, so run them +whenever fi-collector's aliases or this recipe change. The tests start +MLflow in a new process about 30 times, so a full run takes 1 to 2 minutes. From 997ac28d11e9628b819e9b3cce269c520a5b69e8 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Mon, 5 Oct 2026 10:39:02 -0700 Subject: [PATCH 08/15] docs(examples): state only what the MLflow catalog test asserts The model-catalog test asserts GET /catalog/openai.json requests, not a count; the count of four came from exploration runs. Say only what the test checks. Co-Authored-By: Claude Opus 5.5 approved: Nikhil 2026-10-03 blanket Refs: TH-8334 --- python/examples/mlflow/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/examples/mlflow/README.md b/python/examples/mlflow/README.md index 131c5ff0c..ea0392a8d 100644 --- a/python/examples/mlflow/README.md +++ b/python/examples/mlflow/README.md @@ -182,7 +182,7 @@ Writing to an MLflow store also makes MLflow fetch its model catalog to price LLM spans, from `https://github.com/mlflow/mlflow/releases/download/model-catalog%2Flatest` by default (`mlflow/environment_variables.py:1758-1766`, `mlflow/utils/providers.py:252-297`). The test points -`MLFLOW_MODEL_CATALOG_URI` at a loopback server and sees four requests for +`MLFLOW_MODEL_CATALOG_URI` at a loopback server and sees `GET` requests for `/catalog/openai.json`. Set `MLFLOW_MODEL_CATALOG_URI=` (empty) to turn the fetch off. OTLP-only export never fetches it. From c66da1e2958b30c5046753b1f1662c7f11ec287b Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Mon, 5 Oct 2026 15:04:05 -0700 Subject: [PATCH 09/15] test(examples): pin MLflow recipe refusals for Databricks, destinations and headers Tests first for review-r1/verify-r1 (R1, R3, R4, V1, V2). On the unchanged app and README at 997ac28 these fail with assertion failures (15 on Python 3.12): - check_environment() must refuse, before the first span and with zero requests to a loopback catch-all that also stands in for the model catalog and DATABRICKS_HOST: MLFLOW_TRACKING_URI=databricks and databricks://profile, MLFLOW_TRACING_DESTINATION=0 (dual export unset or false), a trailing-slash traces endpoint, a general OTEL_EXPORTER_OTLP_HEADERS, unset or empty TRACES headers, and TRACES headers without x-secret-key. No header value may appear in the output. - OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf with the TRACES protocol unset must export normally. - The README Run block's own export/unset lines, run by bash over a stale environment (MLFLOW_TRACING_DESTINATION=0, a general other-vendor header, a catalog URI on a loopback recorder) and with no recipe variable pre-set, must deliver both spans with the FI keys to the intended receiver, leave no stale value, write no file and fetch no catalog. All those lines must come before the app line. - The README header line must stop, in a script and when pasted into an interactive bash, when FI_API_KEY is unset or FI_SECRET_KEY is empty. Vendor-behaviour tests that pass already: MLflow posts a trailing-slash endpoint verbatim (404); MLFLOW_TRACING_DESTINATION silences OTLP like a set_destination call and only logs INFO lines; with dual export it sends both; with MLFLOW_TRACKING_URI=databricks and dual export off MLflow still fetches /catalog/openai.json and, with an experiment id, GETs /api/2.0/mlflow/experiments/get, all on loopback with the guard log empty; upper-case header names arrive; TRACES grpc over a general http/protobuf and a general grpc are refused. Co-Authored-By: Claude Opus 5.5 approved: Nikhil 2026-10-03 blanket Refs: TH-8334 --- .../mlflow/tests/test_mlflow_recipe.py | 329 +++++++++++++++++- 1 file changed, 327 insertions(+), 2 deletions(-) diff --git a/python/examples/mlflow/tests/test_mlflow_recipe.py b/python/examples/mlflow/tests/test_mlflow_recipe.py index 50c808e83..d91a49eff 100644 --- a/python/examples/mlflow/tests/test_mlflow_recipe.py +++ b/python/examples/mlflow/tests/test_mlflow_recipe.py @@ -9,7 +9,8 @@ anything. There is no model call: the app's model client is a stand-in. All keys are placeholders. Nothing here contacts an MLflow tracking server, -Databricks, a model provider or Future AGI. +Databricks, a model provider or Future AGI: the Databricks tests point the +workspace and MLflow's model catalog at a loopback recorder. """ from __future__ import annotations @@ -19,6 +20,7 @@ import json import os import re +import shlex import sqlite3 import sys import threading @@ -496,9 +498,11 @@ def test_v1_traces_is_accepted_too(tmp_path: Path) -> None: @pytest.mark.parametrize( ("suffix", "posted_to"), - [("", "/"), ("/", "/"), ("/tracer", "/tracer")], + [("", "/"), ("/", "/"), ("/tracer", "/tracer"), ("/tracer/v1/traces/", "/tracer/v1/traces/")], ) def test_mlflow_does_not_append_v1_traces(tmp_path: Path, suffix: str, posted_to: str) -> None: + # Nor does it strip a trailing slash: the recorder, like fi-collector's + # exact "/tracer/v1/traces" pattern (server.go:233-234), answers 404. record = _run_recorded( tmp_path, BARE_SPAN, [QUESTION], lambda recorder: recorder.origin + suffix ) @@ -580,6 +584,15 @@ def test_a_metrics_endpoint_turns_on_metrics_export( ({"OTEL_EXPORTER_OTLP_TRACES_ENDPOINT": "http://127.0.0.1:9"}, "full traces URL"), ({"OTEL_EXPORTER_OTLP_TRACES_PROTOCOL": None}, "OTEL_EXPORTER_OTLP_TRACES_PROTOCOL=http/protobuf"), ({"OTEL_EXPORTER_OTLP_TRACES_PROTOCOL": "grpc"}, "OTEL_EXPORTER_OTLP_TRACES_PROTOCOL=http/protobuf"), + ( + {"OTEL_EXPORTER_OTLP_TRACES_PROTOCOL": None, "OTEL_EXPORTER_OTLP_PROTOCOL": "grpc"}, + "OTEL_EXPORTER_OTLP_TRACES_PROTOCOL=http/protobuf", + ), + ( + # MLflow reads the TRACES variable first (otlp.py:133-136). + {"OTEL_EXPORTER_OTLP_TRACES_PROTOCOL": "grpc", "OTEL_EXPORTER_OTLP_PROTOCOL": "http/protobuf"}, + "OTEL_EXPORTER_OTLP_TRACES_PROTOCOL=http/protobuf", + ), ({"OTEL_EXPORTER_OTLP_ENDPOINT": "http://127.0.0.1:9"}, "Unset OTEL_EXPORTER_OTLP_ENDPOINT"), ( {"OTEL_EXPORTER_OTLP_METRICS_ENDPOINT": "http://127.0.0.1:9/v1/metrics"}, @@ -594,6 +607,8 @@ def test_a_metrics_endpoint_turns_on_metrics_export( "origin-only", "no-protocol", "grpc", + "general-grpc", + "traces-grpc-over-general-http", "base-endpoint", "metrics-endpoint", "no-resource", @@ -613,6 +628,138 @@ def test_app_refuses_settings_that_export_nothing( assert record["files"] == [] +@pytest.mark.parametrize( + ("suffix", "overrides", "message"), + [ + ("/tracer/v1/traces", {"MLFLOW_TRACKING_URI": "databricks"}, "Unset MLFLOW_TRACKING_URI"), + ("/tracer/v1/traces", {"MLFLOW_TRACKING_URI": "databricks://profile"}, "Unset MLFLOW_TRACKING_URI"), + ("/tracer/v1/traces", {"MLFLOW_TRACING_DESTINATION": "0"}, "Unset MLFLOW_TRACING_DESTINATION"), + ( + "/tracer/v1/traces", + {"MLFLOW_TRACING_DESTINATION": "0", "MLFLOW_TRACE_ENABLE_OTLP_DUAL_EXPORT": "false"}, + "Unset MLFLOW_TRACING_DESTINATION", + ), + ("/tracer/v1/traces/", {}, "trailing slash"), + ( + "/tracer/v1/traces", + {"OTEL_EXPORTER_OTLP_HEADERS": "x-api-key=other-vendor-placeholder"}, + "Unset OTEL_EXPORTER_OTLP_HEADERS", + ), + ("/tracer/v1/traces", {"OTEL_EXPORTER_OTLP_TRACES_HEADERS": None}, "missing: x-api-key, x-secret-key"), + ("/tracer/v1/traces", {"OTEL_EXPORTER_OTLP_TRACES_HEADERS": ""}, "missing: x-api-key, x-secret-key"), + ( + "/tracer/v1/traces", + {"OTEL_EXPORTER_OTLP_TRACES_HEADERS": "x-api-key=" + quote(FI_API_KEY, safe="")}, + "missing: x-secret-key", + ), + ], + ids=[ + "databricks", + "databricks-profile", + "tracing-destination", + "tracing-destination-dual-false", + "trailing-slash", + "general-headers", + "no-traces-headers", + "empty-traces-headers", + "traces-headers-without-secret", + ], +) +def test_app_refuses_settings_that_send_traces_or_keys_elsewhere( + tmp_path: Path, suffix: str, overrides: dict[str, Optional[str]], message: str +) -> None: + """Refused before the first span, so not one request of any kind is made. + + The endpoint, the model catalog and DATABRICKS_HOST are all the loopback + catch-all, so a span export, a catalog fetch or a workspace call would be + recorded here, not only blocked by the guard. + """ + with _Recorder() as recorder: + loopback = { + "MLFLOW_MODEL_CATALOG_URI": recorder.origin + "/catalog", + "DATABRICKS_HOST": recorder.origin, + } + record = _execute( + tmp_path, APP, [QUESTION], recorder.origin + suffix, **{**loopback, **overrides} + ) + calls = list(recorder.calls) + assert not record["result"].timed_out + assert record["result"].returncode != 0 + assert message in record["stderr"] + assert ANSWER not in record["stdout"] + assert calls == [] + assert record["files"] == [] + assert record["guard_attempts"] == [] + # The message names variables and headers, never a value. + output = record["stdout"] + record["stderr"] + for value in (FI_API_KEY, FI_SECRET_KEY, quote(FI_SECRET_KEY, safe=""), "other-vendor-placeholder"): + assert value not in output + + +def test_general_otlp_protocol_is_accepted(tmp_path: Path) -> None: + """MLflow falls back to OTEL_EXPORTER_OTLP_PROTOCOL when the TRACES one is unset.""" + record = _run_script( + tmp_path, + APP, + [QUESTION], + OTEL_EXPORTER_OTLP_TRACES_PROTOCOL=None, + OTEL_EXPORTER_OTLP_PROTOCOL="http/protobuf", + ) + _assert_ran(record) + assert {request["path"] for request in record["requests"]} == {"/tracer/v1/traces"} + assert {span["name"] for span in record["spans"]} == {ROOT, LLM} + assert record["files"] == [] + + +def test_header_names_are_case_insensitive(tmp_path: Path) -> None: + headers = otlp_headers(FI_API_KEY, FI_SECRET_KEY) + headers = headers.replace("x-api-key", "X-Api-Key").replace("x-secret-key", "X-Secret-Key") + record = _run_script(tmp_path, APP, [QUESTION], OTEL_EXPORTER_OTLP_TRACES_HEADERS=headers) + _assert_ran(record) + for request in record["requests"]: + # The exporter lower-cases names; fi-collector reads them case-insensitively. + assert request["headers"].get("x-api-key") == FI_API_KEY + assert request["headers"].get("x-secret-key") == FI_SECRET_KEY + + +@pytest.mark.parametrize("experiment_id", [None, "123"], ids=["no-experiment", "experiment"]) +def test_databricks_tracking_uri_fetches_the_catalog_and_calls_the_workspace( + tmp_path: Path, experiment_id: Optional[str] +) -> None: + """Why src/app.py refuses a Databricks tracking URI when dual export is off. + + tests/bare_span.py skips check_environment(). The OTLP endpoint, the model + catalog and the workspace (DATABRICKS_HOST, placeholder token) are all the + loopback catch-all, so nothing reaches github.com or Databricks. + """ + with _Recorder() as recorder: + record = _execute( + tmp_path, + BARE_SPAN, + [QUESTION], + recorder.origin + "/tracer/v1/traces", + MLFLOW_TRACKING_URI="databricks", + MLFLOW_MODEL_CATALOG_URI=recorder.origin + "/catalog", + DATABRICKS_HOST=recorder.origin, + DATABRICKS_TOKEN="databricks-token-placeholder", + MLFLOW_EXPERIMENT_ID=experiment_id, + ) + calls = list(recorder.calls) + _assert_exited_cleanly(record) + assert ("POST", "/tracer/v1/traces") in calls + # MLflow prices the model span on the client, so it fetches the catalog, + # although dual export is off. + assert ("GET", "/catalog/openai.json") in calls + workspace = [call for call in calls if call[1].startswith("/api/")] + if experiment_id is None: + assert workspace == [] + else: + # On the first span MLflow reads the experiment, to see whether it is + # linked to a Unity Catalog trace location. + assert workspace == [("GET", "/api/2.0/mlflow/experiments/get")] + assert record["files"] == [] + + def test_unencoded_comma_in_a_header_value_truncates_it(tmp_path: Path) -> None: """Why the recipe percent-encodes: the exporter splits the variable on commas.""" raw = "x-api-key={0},x-secret-key={1}".format(FI_API_KEY, FI_SECRET_KEY) @@ -773,6 +920,43 @@ def test_set_destination_with_dual_export_sends_both(tmp_path: Path) -> None: } +def test_tracing_destination_variable_silences_otlp(tmp_path: Path) -> None: + """MLFLOW_TRACING_DESTINATION does what a set_destination call does, with no code.""" + record = _run_recorded( + tmp_path, + BARE_SPAN, + [QUESTION], + lambda recorder: recorder.origin + "/tracer/v1/traces", + MLFLOW_TRACING_DESTINATION="0", + MLFLOW_MODEL_CATALOG_URI="", + ) + _assert_exited_cleanly(record) + assert ANSWER in record["stdout"] + assert record["calls"] == [] + # Only INFO lines about the local database; nothing says OTLP was skipped. + assert [line for line in record["stderr"].splitlines() if " INFO " not in line] == [] + assert record["files"] == ["mlflow.db"] + assert _store(record["work"] / "mlflow.db")["spans"] == [ROOT, LLM] + + +def test_tracing_destination_with_dual_export_is_accepted_and_sends_both(tmp_path: Path) -> None: + record = _run_script( + tmp_path, + APP, + [QUESTION], + MLFLOW_TRACING_DESTINATION="0", + MLFLOW_TRACE_ENABLE_OTLP_DUAL_EXPORT="true", + MLFLOW_MODEL_CATALOG_URI="", + ) + _assert_ran(record) + assert {span["name"] for span in record["spans"]} == {ROOT, LLM} + assert record["files"] == ["mlflow.db"] + assert _store(record["work"] / "mlflow.db") == { + "traces": [_trace_request_id(record)], + "spans": [ROOT, LLM], + } + + @pytest.mark.parametrize("disabled", [None, "true"]) def test_telemetry_switch(tmp_path: Path, disabled: Optional[str]) -> None: record = _run_script(tmp_path, APP, [QUESTION], MLFLOW_DISABLE_TELEMETRY=disabled) @@ -782,6 +966,147 @@ def test_telemetry_switch(tmp_path: Path, disabled: Optional[str]) -> None: assert record["files"] == expected +# -------------------------------------------------------------------------- +# The README Run block itself, run by bash. +# -------------------------------------------------------------------------- + +# The Run block lines the tests execute: export and unset, optionally behind a +# ': "${VAR:?}" ... &&' guard. Never cd, pip or the line that runs the app. +_SETUP_LINE = re.compile(r'^(?:: (?:"\$\{\w+:\?\}" )+&& )?(?:export|unset) ') + + +def _readme_setup_lines(origin: str) -> list[str]: + """README.md's Run block export/unset lines, with a loopback origin and the test's keys. + + The test runs these lines on their own, so they must all come before the + line that runs the app. + """ + text = README.read_text(encoding="utf-8") + block = text.split("## Run\n", 1)[1].split("```bash\n", 1)[1].split("```", 1)[0].splitlines() + run_line = next(n for n, line in enumerate(block) if line.startswith("python src/app.py")) + picked = [n for n, line in enumerate(block) if _SETUP_LINE.match(line)] + assert picked, "no export or unset line in the Run block" + assert max(picked) < run_line, "an export or unset line comes after the app runs" + setup = "\n".join(block[n] for n in picked) + for placeholder, value in ( + ("https://YOUR_FI_COLLECTOR_ORIGIN", origin), + ("YOUR_API_KEY", FI_API_KEY), + ("YOUR_SECRET_KEY", FI_SECRET_KEY), + ): + assert setup.count(placeholder) == 1, placeholder + setup = setup.replace(placeholder, value) + return setup.replace("python -c", shlex.quote(sys.executable) + " -c").splitlines() + + +def _emit_line(path: Path, names: tuple[str, ...]) -> str: + """A shell line that writes the named variables to path as JSON (test values only).""" + code = "import json,os,sys; open(sys.argv[1], 'w').write(json.dumps({{k: os.environ.get(k) for k in {0!r}}}))".format( + names + ) + return " ".join(shlex.quote(part) for part in (sys.executable, "-c", code, str(path))) + + +def test_readme_run_block_replaces_a_stale_environment(tmp_path: Path) -> None: + """The Run block's own lines, run by bash over settings left from another setup. + + The shell inherits PATH, HOME, the guard log and three stale values, and + no recipe variable, so deleting a line from the README fails this test. + The stale model catalog URI is a loopback recorder, so a fetch would be + recorded, not only blocked. + """ + work = tmp_path / "work" + work.mkdir() + guard_log = tmp_path / "guard.jsonl" + emitted = tmp_path / "emitted.json" + stale_names = ("MLFLOW_MODEL_CATALOG_URI", "MLFLOW_TRACING_DESTINATION", "OTEL_EXPORTER_OTLP_HEADERS") + with Receiver() as receiver, _Recorder() as catalog: + env = { + "PATH": os.environ.get("PATH", ""), + "HOME": str(work), + "LOOPBACK_GUARD_LOG": str(guard_log), + "MLFLOW_TRACING_DESTINATION": "0", + "OTEL_EXPORTER_OTLP_HEADERS": "x-api-key=other-vendor-placeholder", + "MLFLOW_MODEL_CATALOG_URI": catalog.origin + "/catalog", + } + app = " ".join(shlex.quote(part) for part in (sys.executable, str(GUARD), str(APP), QUESTION)) + script = "\n".join( + ["set -eu", *_readme_setup_lines(receiver.origin), _emit_line(emitted, stale_names), "exec " + app] + ) + with _working_directory(work): + result = run( + ["/bin/bash", "--noprofile", "--norc", "-c", script], + env=env, + stdin=None, + timeout=RUN_TIMEOUT_SECONDS, + ) + requests = receiver.requests() + spans = receiver.spans() + catalog_calls = list(catalog.calls) + stderr = result.stderr.decode("utf-8", "replace") + assert not result.timed_out + assert result.returncode == 0, stderr + assert ANSWER in result.stdout.decode("utf-8", "replace") + assert requests, "no export reached the intended receiver" + assert {span["name"] for span in spans} == {ROOT, LLM} + for request in requests: + assert request["path"] == "/tracer/v1/traces" + assert request["headers"].get("x-api-key") == FI_API_KEY + assert request["headers"].get("x-secret-key") == FI_SECRET_KEY + assert "other-vendor-placeholder" not in json.dumps(request["headers"]) + for resource in request["resource_attributes"]: + assert resource.get("project_name") == "my-mlflow-app" + assert catalog_calls == [] + assert json.loads(emitted.read_text()) == { + "MLFLOW_MODEL_CATALOG_URI": "", + "MLFLOW_TRACING_DESTINATION": None, + "OTEL_EXPORTER_OTLP_HEADERS": None, + } + assert not guard_log.exists(), guard_log.read_text() + # No ./mlflow.db (destination) and no telemetry file under HOME. + assert sorted(p.name for p in work.rglob("*")) == [] + + +@pytest.mark.parametrize("pasted", [False, True], ids=["script", "pasted"]) +@pytest.mark.parametrize("missing", ["FI_API_KEY", "FI_SECRET_KEY"]) +def test_readme_header_line_stops_when_a_key_is_missing( + tmp_path: Path, missing: str, pasted: bool +) -> None: + """README.md's header line fails loudly instead of exporting an empty value. + + With OTEL_EXPORTER_OTLP_TRACES_HEADERS empty, the exporter would send + OTEL_EXPORTER_OTLP_HEADERS instead. FI_API_KEY is left unset and + FI_SECRET_KEY empty. "pasted" feeds the lines to an interactive bash, as + when they are pasted into a terminal: the shell goes on after an error. + """ + emitted = tmp_path / "emitted.json" + lines = [] + for line in _readme_setup_lines("http://127.0.0.1:9"): + if line.startswith("export {0}=".format(missing)): + if missing == "FI_API_KEY": + continue + line = 'export {0}=""'.format(missing) + lines.append(line) + lines.append(_emit_line(emitted, ("OTEL_EXPORTER_OTLP_TRACES_HEADERS",))) + script = "\n".join(lines) + "\n" + env = { + "PATH": os.environ.get("PATH", ""), + "HOME": str(tmp_path), + "HISTFILE": str(tmp_path / "history"), + "BASH_SILENCE_DEPRECATION_WARNING": "1", + } + argv = ["/bin/bash", "--noprofile", "--norc"] + (["-i"] if pasted else ["-c", script]) + result = run(argv, env=env, stdin=script.encode("utf-8") if pasted else None, timeout=60) + stderr = result.stderr.decode("utf-8", "replace") + assert not result.timed_out + assert missing in stderr + if pasted: + # The line fails and the variable is not exported; the next lines run. + assert json.loads(emitted.read_text()) == {"OTEL_EXPORTER_OTLP_TRACES_HEADERS": None} + else: + assert result.returncode != 0 + assert not emitted.exists() + + # -------------------------------------------------------------------------- # MLFLOW_ENABLE_OTEL_GENAI_SEMCONV=true: MLflow's own GenAI translation. # -------------------------------------------------------------------------- From e7a641108f2ea583e5c13ac8bd4cb31894812e25 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Mon, 5 Oct 2026 15:08:47 -0700 Subject: [PATCH 10/15] fix(examples): refuse MLflow settings that send traces or keys elsewhere check_environment() now also exits, before the first span and with one message each (decision D-8334a): - MLFLOW_TRACKING_URI=databricks or databricks://... with dual export off (R1, V2): MLflow then reads the experiment from the workspace on the first span, fetches its model catalog (github.com by default) to price the model span, and replaces OTLP with Unity Catalog for a linked experiment. - MLFLOW_TRACING_DESTINATION set with dual export off (V1): it replaces the OTLP export with no message. - A trailing slash after /v1/traces (R3): MLflow posts it verbatim and fi-collector's exact patterns answer 404. The rstrip("/") is gone. - OTEL_EXPORTER_OTLP_HEADERS set, or OTEL_EXPORTER_OTLP_TRACES_HEADERS without both x-api-key and x-secret-key (R4): the exporter falls back to the general headers when the TRACES ones are empty. Names are matched case-insensitively as the exporter parses them; values are never printed. The protocol is now read as MLflow reads it (TRACES, then OTEL_EXPORTER_OTLP_PROTOCOL, default grpc), so a general http/protobuf is accepted (R3). Dual export is parsed as MLflow parses booleans (true/1). The README Run block exports MLFLOW_MODEL_CATALOG_URI="" and unsets OTEL_EXPORTER_OTLP_HEADERS and MLFLOW_TRACING_DESTINATION, and its header line stops behind ': "${FI_API_KEY:?}" "${FI_SECRET_KEY:?}" &&' when a key is empty, instead of exporting an empty value. The app docstring states the tested condition, and the comment on the gen_ai.* keys says the quoted model is not priced (R2). Co-Authored-By: Claude Opus 5.5 approved: Nikhil 2026-10-03 blanket Refs: TH-8334 --- python/examples/mlflow/README.md | 7 +++- python/examples/mlflow/src/app.py | 68 +++++++++++++++++++++++++++---- 2 files changed, 65 insertions(+), 10 deletions(-) diff --git a/python/examples/mlflow/README.md b/python/examples/mlflow/README.md index ea0392a8d..bdc1ffeae 100644 --- a/python/examples/mlflow/README.md +++ b/python/examples/mlflow/README.md @@ -41,11 +41,16 @@ export FI_SECRET_KEY="YOUR_SECRET_KEY" # The full URL: MLflow does not append /v1/traces to this variable. export OTEL_EXPORTER_OTLP_TRACES_ENDPOINT="https://YOUR_FI_COLLECTOR_ORIGIN/tracer/v1/traces" export OTEL_EXPORTER_OTLP_TRACES_PROTOCOL="http/protobuf" # MLflow's default is gRPC -export OTEL_EXPORTER_OTLP_TRACES_HEADERS="$(python -c 'import os; from urllib.parse import quote; print("x-api-key={0},x-secret-key={1}".format(quote(os.environ["FI_API_KEY"], safe=""), quote(os.environ["FI_SECRET_KEY"], safe="")))')" +# Stops with an error, and exports nothing, if FI_API_KEY or FI_SECRET_KEY is empty. +: "${FI_API_KEY:?}" "${FI_SECRET_KEY:?}" && export OTEL_EXPORTER_OTLP_TRACES_HEADERS="$(python -c 'import os; from urllib.parse import quote; print("x-api-key={0},x-secret-key={1}".format(quote(os.environ["FI_API_KEY"], safe=""), quote(os.environ["FI_SECRET_KEY"], safe="")))')" export OTEL_RESOURCE_ATTRIBUTES="project_name=my-mlflow-app,project_type=observe" export OTEL_SERVICE_NAME="my-mlflow-app" # optional: service.name export MLFLOW_DISABLE_TELEMETRY=true +export MLFLOW_MODEL_CATALOG_URI="" # no model-catalog fetch from github.com unset OTEL_EXPORTER_OTLP_ENDPOINT OTEL_EXPORTER_OTLP_METRICS_ENDPOINT +# The exporter falls back to these when the TRACES headers are empty, which could send another vendor's key to Future AGI. +unset OTEL_EXPORTER_OTLP_HEADERS +unset MLFLOW_TRACING_DESTINATION # it replaces the OTLP export python src/app.py "What is the refund window?" ``` diff --git a/python/examples/mlflow/src/app.py b/python/examples/mlflow/src/app.py index a5fc8703b..7e4b50624 100644 --- a/python/examples/mlflow/src/app.py +++ b/python/examples/mlflow/src/app.py @@ -4,8 +4,12 @@ OTLP when ``OTEL_EXPORTER_OTLP_TRACES_ENDPOINT`` is set before the first span; this file only checks that environment and creates two nested spans with ``mlflow.start_span``. The endpoint, protocol, Future AGI keys and -project come from the environment variables listed in README.md. Nothing -here talks to an MLflow tracking server, Databricks or a model provider. +project come from the environment variables listed in README.md. With those +variables (OTLP only, no Databricks tracking URI, ``MLFLOW_MODEL_CATALOG_URI`` +empty) MLflow sends the spans to the OTLP endpoint and contacts nothing else +(tested); ``check_environment()`` refuses the settings that would make it +call Databricks or send the traces elsewhere. The model client is a +stand-in, so there is no model call. python src/app.py "What is the refund window?" python src/app.py --drop-content "What is the refund window?" @@ -15,6 +19,7 @@ import os import sys +from urllib.parse import unquote, urlparse import mlflow from mlflow.entities import LiveSpan, SpanType @@ -26,22 +31,35 @@ _POLICY = "Refunds are accepted within 30 days of purchase." +def _dual_export() -> bool: + # Parsed as MLflow parses its boolean variables. + return os.environ.get("MLFLOW_TRACE_ENABLE_OTLP_DUAL_EXPORT", "").lower() in ("true", "1") + + def check_environment() -> None: - """Exit with a message on the settings that make MLflow 3.16.1 export nothing. + """Exit with a message on the settings that make MLflow 3.16.1 send nothing + to Future AGI, send traces or keys elsewhere, or call Databricks. - Each of these fails without an exception in the app (see README.md). + None of these raises an exception in the app (see README.md). """ endpoint = os.environ.get("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", "") - if not endpoint.rstrip("/").endswith("/v1/traces"): + if endpoint.endswith("/") and endpoint.rstrip("/").endswith("/v1/traces"): + # MLflow posts to this URL as is, and fi-collector serves only the + # exact paths, so the slash gets a 404. + raise SystemExit("Remove the trailing slash from OTEL_EXPORTER_OTLP_TRACES_ENDPOINT.") + if not endpoint.endswith("/v1/traces"): # MLflow uses this URL as is; it does not append /v1/traces. When the # variable is unset, MLflow writes traces to a local ./mlflow.db. raise SystemExit( "Set OTEL_EXPORTER_OTLP_TRACES_ENDPOINT to the full traces URL, " "for example https:///tracer/v1/traces." ) - if os.environ.get("OTEL_EXPORTER_OTLP_TRACES_PROTOCOL") != "http/protobuf": - # MLflow's default is gRPC, and it skips export if that exporter is - # not installed. + # MLflow reads the TRACES protocol, then the general one. Its default is + # gRPC, and it skips export if that exporter is not installed. + protocol = os.environ.get("OTEL_EXPORTER_OTLP_TRACES_PROTOCOL") or os.environ.get( + "OTEL_EXPORTER_OTLP_PROTOCOL", "grpc" + ) + if protocol != "http/protobuf": raise SystemExit("Set OTEL_EXPORTER_OTLP_TRACES_PROTOCOL=http/protobuf.") for name in ("OTEL_EXPORTER_OTLP_ENDPOINT", "OTEL_EXPORTER_OTLP_METRICS_ENDPOINT"): if os.environ.get(name): @@ -49,6 +67,36 @@ def check_environment() -> None: # does not accept; with the default gRPC metrics protocol and no # gRPC exporter installed, MLflow then drops every span. raise SystemExit("Unset {0}; set only the TRACES variables.".format(name)) + if os.environ.get("OTEL_EXPORTER_OTLP_HEADERS"): + # The exporter sends these instead when the TRACES headers are empty, + # so a key meant for another destination could reach Future AGI. + raise SystemExit( + "Unset OTEL_EXPORTER_OTLP_HEADERS; put the Future AGI keys in " + "OTEL_EXPORTER_OTLP_TRACES_HEADERS." + ) + # Header names as the exporter parses them. Values are never printed. + headers = os.environ.get("OTEL_EXPORTER_OTLP_TRACES_HEADERS", "") + names = {unquote(item.split("=", 1)[0]).strip().lower() for item in headers.split(",") if "=" in item} + missing = [name for name in ("x-api-key", "x-secret-key") if name not in names] + if missing: + raise SystemExit( + "Set OTEL_EXPORTER_OTLP_TRACES_HEADERS to x-api-key=,x-secret-key= " + "(missing: {0}).".format(", ".join(missing)) + ) + if os.environ.get("MLFLOW_TRACING_DESTINATION") and not _dual_export(): + # MLflow sends the traces to that destination (an experiment's store, + # or Databricks) instead of the OTLP endpoint, and says nothing. + raise SystemExit("Unset MLFLOW_TRACING_DESTINATION; it replaces the OTLP export.") + tracking_uri = os.environ.get("MLFLOW_TRACKING_URI", "") + if (tracking_uri == "databricks" or urlparse(tracking_uri).scheme == "databricks") and not _dual_export(): + # MLflow then reads the experiment from the workspace on the first + # span, fetches its model catalog to price model spans, and sends the + # traces to Unity Catalog instead of OTLP if the experiment is linked + # to it. + raise SystemExit( + "Unset MLFLOW_TRACKING_URI for this recipe: with a Databricks tracking URI " + "MLflow calls Databricks and can replace the OTLP export." + ) resource = os.environ.get("OTEL_RESOURCE_ATTRIBUTES", "") items = [item.split("=", 1) for item in resource.split(",")] if resource else [] if any(len(item) != 2 for item in items): @@ -90,7 +138,9 @@ def answer(question: str) -> str: reply, usage = call_model(messages) llm.set_outputs({"choices": [{"message": {"role": "assistant", "content": reply}}]}) # The OTel GenAI keys fi-collector reads. MLflow JSON-encodes every - # attribute value, so these arrive as strings (see README.md). + # attribute value: the token counts still parse, but the model + # arrives with its quotes, so fi-collector prices nothing (cost 0; + # see README.md). llm.set_attributes( { "gen_ai.request.model": MODEL, From 7b4751480f10d407ad4afe25eb2cbc69f13e6bae Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Mon, 5 Oct 2026 15:13:29 -0700 Subject: [PATCH 11/15] docs(examples): document the MLflow recipe's Databricks, destination and header refusals README prose for review-r1/verify-r1, following D-8334a: - R1: the catalog paragraph now states the true condition. With dual export off and a non-Databricks tracking URI MLflow does not fetch the catalog (tested: no connection attempt with the default URI); with a Databricks tracking URI it does (tested on loopback). The recipe table lists MLFLOW_MODEL_CATALOG_URI empty and the variables that must be unset, and says which of them src/app.py checks. - V2: a "Databricks tracking URI" section: the client-side pricing and catalog fetch, the experiment read on the first span (both tested with a loopback workspace and catalog), the Unity Catalog replacement of OTLP (source reading), the refusal, and dual export with Databricks as out of scope and untested, with the calls it makes from source. The intro now states the tested condition for "contacts only the OTLP endpoint". - V1: MLFLOW_TRACING_DESTINATION next to set_destination (tested silence and dual export), and that a set_destination call in code cannot be seen by check_environment(). - R3: the trailing-slash row in the endpoint table, and the OTEL_EXPORTER_OTLP_PROTOCOL fallback. - R4: the general-headers fallback, the refusal, and why the header line starts with ': "${FI_API_KEY:?}" "${FI_SECRET_KEY:?}" &&'. - R2: the quoted model gets no token price (cost 0); "cost not priced" in the SF-1 JSON-decoding bullet. - R5: mlflow.traceTag. in the SF-1 list and in what the drop-content processor does not cover. - Troubleshooting and the Tests section follow. Co-Authored-By: Claude Opus 5.5 approved: Nikhil 2026-10-03 blanket Refs: TH-8334 --- python/examples/mlflow/README.md | 159 +++++++++++++++++++++++++++---- 1 file changed, 140 insertions(+), 19 deletions(-) diff --git a/python/examples/mlflow/README.md b/python/examples/mlflow/README.md index bdc1ffeae..69274ac4f 100644 --- a/python/examples/mlflow/README.md +++ b/python/examples/mlflow/README.md @@ -28,7 +28,11 @@ on 2026-10-05. Python 3.14 was not tested. Statements marked "source reading" come from the mlflow 3.16.1 wheel or from fi-collector at future-agi `main` 4af5338 and are not exercised by the tests. No MLflow tracking server, Databricks workspace, model provider or Future AGI endpoint -was called. +was called: the tests block every non-loopback connection, and the +Databricks tests point the workspace and MLflow's model catalog at a +loopback recorder. With the Run block below (OTLP only, no Databricks +tracking URI, `MLFLOW_MODEL_CATALOG_URI` empty) the recipe contacts only the +OTLP endpoint (tested). ## Run @@ -59,7 +63,8 @@ Set the variables before the first span starts: MLflow builds its exporter once, on first use. `src/app.py` is two nested `mlflow.start_span` calls around a stand-in model client that returns a canned answer, so it makes no model call. Its `check_environment()` exits with a message on each setting -below that would make MLflow export nothing. +below that would make MLflow send nothing to Future AGI, send the traces or +keys elsewhere, or call Databricks. ## The recipe @@ -71,6 +76,19 @@ below that would make MLflow export nothing. | `OTEL_RESOURCE_ATTRIBUTES` | `project_name=,project_type=observe` | fi-collector needs `project_name`. | | `OTEL_SERVICE_NAME` | optional | Otherwise `service.name` is OpenTelemetry's `unknown_service` default. | | `MLFLOW_DISABLE_TELEMETRY` | `true` | See [Telemetry](#telemetry). | +| `MLFLOW_MODEL_CATALOG_URI` | empty | No model-catalog fetch from github.com; see [Dual export](#dual-export-and-set_destination). | +| `OTEL_EXPORTER_OTLP_ENDPOINT`, `OTEL_EXPORTER_OTLP_METRICS_ENDPOINT` | unset | They turn on metrics export; see [Endpoint](#endpoint). | +| `OTEL_EXPORTER_OTLP_HEADERS` | unset | Sent instead of empty TRACES headers; see [Auth headers](#auth-headers). | +| `MLFLOW_TRACING_DESTINATION` | unset | It replaces the OTLP export; see [Dual export](#dual-export-and-set_destination). | +| `MLFLOW_TRACKING_URI` | unset, or a non-Databricks URI | See [Databricks tracking URI](#databricks-tracking-uri). | + +`src/app.py` checks each of these except `OTEL_SERVICE_NAME`, +`MLFLOW_DISABLE_TELEMETRY` and `MLFLOW_MODEL_CATALOG_URI`, and exits with a +message on a value that would lose or misroute the export or make MLflow +call Databricks. For the headers +it checks the two names, not the values. It refuses +`MLFLOW_TRACING_DESTINATION` and a Databricks `MLFLOW_TRACKING_URI` only +while dual export is off. ### Endpoint @@ -86,6 +104,12 @@ against a loopback receiver with mlflow 3.16.1: | `http://host:port` | `/` | 404, spans lost | | `http://host:port/` | `/` | 404, spans lost | | `http://host:port/tracer` | `/tracer` | 404, spans lost | +| `http://host:port/tracer/v1/traces/` | `/tracer/v1/traces/` | 404, spans lost | + +MLflow does not strip a trailing slash either. fi-collector registers the +exact paths `/v1/traces` and `/tracer/v1/traces` (`pkg/server/server.go:233-234`, +source reading), so it would answer a trailing slash with 404 as the test's +receiver does. `src/app.py` refuses a trailing slash with its own message. fi-collector serves both `/v1/traces` and `/tracer/v1/traces`. The recipe uses `/tracer/v1/traces` because that is where traceAI's own exporter posts @@ -114,6 +138,12 @@ here, so MLflow cannot build its exporter; `mlflow.start_span` then hands out no-op spans and logs that only at debug level (`fluent.py:660-662`). Tested: no request, nothing on stderr, exit code 0. +MLflow reads `OTEL_EXPORTER_OTLP_TRACES_PROTOCOL` first and falls back to +`OTEL_EXPORTER_OTLP_PROTOCOL` (`otlp.py:133-136`). `src/app.py` reads them +the same way, so `OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf` alone exports +(tested), and a TRACES value of `grpc` is refused even when the general one +is `http/protobuf` (tested). + ### Auth headers The OpenTelemetry HTTP exporter (1.45.0) reads @@ -127,6 +157,25 @@ contains `,` and `=` intact, and no `authorization` header. An unencoded comma cuts the secret short (tested). Do not copy the `api_key=12345` header from MLflow's documentation; it is not a Future AGI credential. +When `OTEL_EXPORTER_OTLP_TRACES_HEADERS` is unset or empty, the exporter +sends the general `OTEL_EXPORTER_OTLP_HEADERS` instead +(`opentelemetry/exporter/otlp/proto/http/_common/__init__.py:146-147` in +`opentelemetry-exporter-otlp-proto-http`), so a key you set for another +vendor would reach Future AGI. The Run block unsets the general variable. +`src/app.py` refuses it when set, and refuses TRACES headers that lack the +`x-api-key` or `x-secret-key` name (names compared case-insensitively; +the message names what is missing and never shows a value). Tested, as is +a stale environment under the Run block's own lines: the spans reach the +intended receiver with the Future AGI keys only. + +`export VAR="$(...)"` hides a failed command and exports an empty value, so +the header line in the Run block starts with +`: "${FI_API_KEY:?}" "${FI_SECRET_KEY:?}" &&`. With either key unset or +empty, a script stops there, and a shell you paste into reports +`parameter null or not set` and exports nothing (tested with bash, as a +script and as an interactive shell; zsh 5.9 behaves the same in a manual +check). + ### Resource: project, experiment and session `OTEL_RESOURCE_ATTRIBUTES` carries `project_name` and `project_type` @@ -189,7 +238,11 @@ by default (`mlflow/environment_variables.py:1758-1766`, `mlflow/utils/providers.py:252-297`). The test points `MLFLOW_MODEL_CATALOG_URI` at a loopback server and sees `GET` requests for `/catalog/openai.json`. Set `MLFLOW_MODEL_CATALOG_URI=` (empty) to turn the -fetch off. OTLP-only export never fetches it. +fetch off; the Run block does. With dual export off and a non-Databricks +tracking URI, MLflow does not fetch it (tested: the recipe run with the +default catalog URI attempts no connection). With a Databricks tracking URI +it fetches it even with dual export off (tested); see +[Databricks tracking URI](#databricks-tracking-uri). `mlflow.tracing.set_destination(...)` takes precedence over the OTLP variables (`mlflow/tracing/provider.py:825-860`). Tested with @@ -197,7 +250,51 @@ variables (`mlflow/tracing/provider.py:825-860`). Tested with reaches the OTLP endpoint, and the trace is in `./mlflow.db` instead. With `MLFLOW_TRACE_ENABLE_OTLP_DUAL_EXPORT=true` as well, the trace goes to both. Do not call `set_destination` in an app that exports to Future AGI, or turn -dual export on. +dual export on. `check_environment()` reads only the environment, so it +cannot see a `set_destination` call in your code. + +`MLFLOW_TRACING_DESTINATION` does the same without code +(`mlflow/tracing/destination.py:29-74`): an experiment id sends the traces +to that experiment in the tracking store, and `.` sends +them to Databricks Unity Catalog and sets the tracking URI to `databricks` +(source reading). Tested with `MLFLOW_TRACING_DESTINATION=0`: no request +reaches the OTLP endpoint, the trace is in `./mlflow.db`, and MLflow logs +nothing above INFO. With dual export on as well, +the trace goes to both (tested). The Run block unsets it, and `src/app.py` +refuses it unless dual export is on. + +### Databricks tracking URI + +With `MLFLOW_TRACKING_URI=databricks` or `databricks://` and dual +export off, MLflow still exports over OTLP, but also: + +- prices each span that has a model, a provider and token usage on the + client (`mlflow/entities/span.py:1161-1162`, + `mlflow/tracing/utils/__init__.py:934-944`), which fetches the model + catalog, from github.com by default (`mlflow/utils/providers.py:252-337`). + Tested with `tests/bare_span.py` (no `check_environment()`) and the + catalog on a loopback recorder: `GET /catalog/openai.json`, and the spans + still reach the OTLP endpoint. +- reads the active experiment from the workspace on the first span, to see + whether it is linked to a Unity Catalog trace location + (`mlflow/tracing/provider.py:758-794`). Tested the same way, with + `MLFLOW_EXPERIMENT_ID` set and `DATABRICKS_HOST` on the loopback recorder: + `GET /api/2.0/mlflow/experiments/get`. With no experiment id there was no + workspace request. +- sends the traces to Unity Catalog instead of the OTLP endpoint if that + experiment is linked to it, and logs a failed lookup only at debug level + (`provider.py:789-793`, `:816-859`; source reading, not tested). + +`src/app.py` refuses a Databricks tracking URI unless dual export is on: +unset `MLFLOW_TRACKING_URI` for this recipe. The OTLP export does not use +it. `check_environment()` reads only the variable, not a tracking URI set in +code with `mlflow.set_tracking_uri(...)`. + +Dual export with a Databricks tracking URI is out of scope and not tested. +From the source, it makes the same catalog fetch and experiment read, and +also writes every trace to the workspace (`provider.py:884-907`), or to +Unity Catalog for a linked experiment (`provider.py:827-857`), next to the +OTLP export. ## What is exported @@ -240,7 +337,13 @@ autolog integrations set on a model call; `src/app.py` sets them by hand, next to the `gen_ai.*` keys. With these keys, fi-collector would store the model and provider with their quotes, fill the token columns (total derived as 19, `adapter.go:232-234`), and store both spans as kind `unknown` -(source reading; no fi-collector was run). +(source reading; no fi-collector was run). It would not price the tokens: +it looks the price up by exact model name (`converter.go:288-292`, +`pkg/pricing/table.go:99-102`), and `"gpt-4o-mini"` with its quotes matches +no entry, so the cost is 0. The quoted name also keeps these spans apart +from other integrations' `gpt-4o-mini` in model filters. Without a model the +cost would be 0 as well, so the keys are still worth setting for the token +columns. ### `MLFLOW_ENABLE_OTEL_GENAI_SEMCONV=true` @@ -320,13 +423,16 @@ shared processor work: - `mlflow.spanInputs`, `mlflow.spanOutputs`: raw. Proposed SF-1 input and output source. - JSON-encoded string values on `gen_ai.*` keys from MLflow spans (model and - provider with quotes): proposed SF-1 decoding for spans whose resource has - `telemetry.sdk.name=mlflow`. + provider with quotes, so the cost is not priced): proposed SF-1 decoding + for spans whose resource has `telemetry.sdk.name=mlflow`. - `gen_ai.operation.name=invoke_agent`: proposed SF-1 synonym for agent. - `gen_ai.system_instructions`, `gen_ai.input.messages`, `gen_ai.output.messages`: raw; not lifted into the input and output columns. - `mlflow.traceRequestId`, `mlflow.spanLogLevel`: raw; no Future AGI column. +- `mlflow.traceTag.` (user trace tags, which MLflow writes on the root + span at export, `mlflow/tracing/processor/otel.py:68-79`): raw; not + exercised here (source reading). Whether fi-collector promotes any of these is not asserted here: the harness receiver decodes and records, it does not run fi-collector. @@ -366,8 +472,10 @@ anywhere in the export; with the switch off the two keys remain, holding Not covered by that processor (source reading, not tested): when code inside a span raises, MLflow sets the span status to `ERROR` and adds an `exception` event with the message and stack trace -(`mlflow/tracing/provider.py:309-311`, `mlflow/entities/span.py:1062-1073`), -and attributes your code sets with other keys are exported as they are. +(`mlflow/tracing/provider.py:309-311`, `mlflow/entities/span.py:1062-1073`); +attributes your code sets with other keys are exported as they are; and +trace tags are written on the root span as `mlflow.traceTag.` after the +span processors have run (`mlflow/tracing/processor/otel.py:68-79`). No Future AGI key or secret appears in any exported span or resource, or in the app's output (tested); they travel only as request headers. @@ -378,18 +486,20 @@ the app's output (tested); they travel only as request headers. `OTEL_EXPORTER_OTLP_TRACES_PROTOCOL=http/protobuf` and that `opentelemetry-exporter-otlp-proto-http` is installed. Then check for a `mlflow.tracing.set_destination(...)` call: it wins over the OTLP - variables. + variables, and `src/app.py` cannot see it (it does refuse + `MLFLOW_TRACING_DESTINATION`). - **`Failed to end span ...: No module named 'opentelemetry.exporter.otlp.proto.grpc'`.** `OTEL_EXPORTER_OTLP_ENDPOINT` or `OTEL_EXPORTER_OTLP_METRICS_ENDPOINT` is set. Unset both. - **`Failed to export spans batch code: 404`.** The traces URL has no - `/tracer/v1/traces` (or `/v1/traces`) path. MLflow does not add it. + `/tracer/v1/traces` (or `/v1/traces`) path, or ends with a slash. MLflow + does not add or remove either. - **HTTP 400 from the collector, or no project.** `project_name` is missing from `OTEL_RESOURCE_ATTRIBUTES`, or one item in it has no `=` (a trailing comma), which makes MLflow drop them all. - **An `mlflow.db` file appeared.** `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT` is - unset, or dual export or `set_destination` is writing to MLflow's default - local store. + unset, or dual export, `set_destination` or `MLFLOW_TRACING_DESTINATION` is + writing to MLflow's default local store. - **Model name shown with quotes.** That is MLflow's JSON encoding; see [What is exported](#what-is-exported). - **Old traces never arrived.** Expected: this is live export, not a @@ -403,11 +513,22 @@ its working directory and `HOME`, so every file MLflow writes is checked. Spans go to the shared harness `Receiver` (`python/tests/harness`), which serves `/v1/traces` and `/tracer/v1/traces` on 127.0.0.1 like fi-collector but does not authenticate, stamp projects or store anything. Tests that need -every request path, including 404s and the model catalog, use a loopback -catch-all instead. `tests/_guarded_run.py` blocks and logs every -non-loopback connection and every test asserts its log is empty; a positive -control (`tests/_guard_probe.py`) proves it refuses and logs IPv4, IPv6 and -DNS attempts. All keys are placeholders. +every request path, including 404s, the model catalog and the Databricks +workspace (`DATABRICKS_HOST`), use a loopback catch-all instead. +`tests/_guarded_run.py` blocks and logs every non-loopback connection and +every test asserts its log is empty; a positive control +(`tests/_guard_probe.py`) proves it refuses and logs IPv4, IPv6 and DNS +attempts. All keys are placeholders. + +Two tests run the Run block above with `bash --noprofile --norc`: only its +`export` and `unset` lines, with a loopback origin and placeholder keys +substituted, and they check that all of those lines come before the line +that runs the app. One runs them over a stale environment +(`MLFLOW_TRACING_DESTINATION=0`, another vendor's +`OTEL_EXPORTER_OTLP_HEADERS`, a catalog URI on a loopback recorder) with no +recipe variable set beforehand, then the app. The other leaves a key unset +or empty and checks that the header line stops, as a script and in an +interactive shell. `tests/bare_span.py` (the app's spans without `check_environment()`) and `tests/set_destination_app.py` (the app after a `set_destination` call) are @@ -447,4 +568,4 @@ export FI_COLLECTOR_SRC=/fi-collector # then run the comm Without it they are skipped. No CI job runs this example, so run them whenever fi-collector's aliases or this recipe change. The tests start -MLflow in a new process about 30 times, so a full run takes 1 to 2 minutes. +MLflow in a new process about 55 times, so a full run takes 2 to 3 minutes. From b93c37c3df3bb31c3aa0be2ba8a9cbcb6145165a Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Sun, 11 Oct 2026 20:04:10 -0700 Subject: [PATCH 12/15] docs(examples): say what MLflow autolog traces look like in Future AGI A live run with mlflow.openai.autolog() against fi-collector (future-agi dev ea12cc3) showed that, under the Run block (GenAI semconv switch off), autolog spans arrive as a tree of unknown spans with no model, tokens or cost: autolog writes only mlflow.llm.model and mlflow.chat.tokenUsage, which fi-collector does not read. The gen_ai.* keys the recipe app sets by hand hide this. With MLFLOW_ENABLE_OTEL_GENAI_SEMCONV=true the model calls are llm spans with model and tokens, tools are tool spans and chains chain spans. Add that result and tell autolog users to turn the switch on, and mark the recipe app's collector behaviour as seen live, not only read. Co-Authored-By: Claude Opus 5.5 --- python/examples/mlflow/README.md | 42 ++++++++++++++++++++++++++------ 1 file changed, 35 insertions(+), 7 deletions(-) diff --git a/python/examples/mlflow/README.md b/python/examples/mlflow/README.md index 69274ac4f..fd8e03ef4 100644 --- a/python/examples/mlflow/README.md +++ b/python/examples/mlflow/README.md @@ -334,10 +334,11 @@ Keys on the default export (tested), and whether fi-collector reads them The `mlflow.llm.*` and `mlflow.chat.tokenUsage` keys are what MLflow's autolog integrations set on a model call; `src/app.py` sets them by hand, -next to the `gen_ai.*` keys. With these keys, fi-collector would store the -model and provider with their quotes, fill the token columns (total derived -as 19, `adapter.go:232-234`), and store both spans as kind `unknown` -(source reading; no fi-collector was run). It would not price the tokens: +next to the `gen_ai.*` keys. With these keys, fi-collector stores the +model and provider with their quotes, fills the token columns (total derived +as 19, `adapter.go:232-234`), and stores both spans as kind `unknown` +(source reading, and seen once against a local fi-collector; see +[Autolog in Future AGI](#autolog-in-future-agi)). It does not price the tokens: it looks the price up by exact model name (`converter.go:288-292`, `pkg/pricing/table.go:99-102`), and `"gpt-4o-mini"` with its quotes matches no entry, so the cost is 0. The quoted name also keeps these spans apart @@ -406,7 +407,34 @@ MLflow span is `unknown`. The switch is the only way found to get a model name without quotes and typed spans out of MLflow 3.16.1 without code. Its cost is the renamed spans and the dropped non-chat inputs and outputs. That trade-off is yours; -the recipe leaves it off. +the recipe leaves it off. If you use autolog, read the next section first. + +### Autolog in Future AGI + +Autolog does not set the `gen_ai.*` keys that `src/app.py` sets by hand. It +writes the model to `mlflow.llm.model` (JSON-encoded) and the token counts to +`mlflow.chat.tokenUsage`, which fi-collector does not read. One run, not part +of the tests: `mlflow.openai.autolog()` on the real `openai` 2.54.0 client, +under the Run block's variables, against a local fi-collector (future-agi +`dev` ea12cc3, 2026-10-11). An `AGENT` function called a chat model with a +tool, a `TOOL` function ran the tool, and two `CHAIN` functions made a +streamed call and a failed call. Stored: + +| Span | Switch off (the Run block) | `MLFLOW_ENABLE_OTEL_GENAI_SEMCONV=true` | +|---|---|---| +| Model call | named `Completions`; kind `unknown`; no model, no tokens, cost 0 | named `chat `; kind `llm`; model without quotes, prompt and completion tokens (streamed call too); priced when fi-collector's price table has the model | +| Tool function | kind `unknown` | renamed `execute_tool`; kind `tool` | +| Agent function | kind `unknown` | renamed `invoke_agent`; kind `unknown` | +| Chain function | kind `unknown` | renamed `CHAIN`; kind `chain` | +| Failed call | status `ERROR` on the model span and its parent | the same | +| `input` / `output` columns | empty | empty | + +So with the switch off, an autolog trace reaches Future AGI as a tree of +`unknown` spans with no model, tokens or cost, and the content only in +`mlflow.spanInputs` and `mlflow.spanOutputs`. If you use autolog, set +`MLFLOW_ENABLE_OTEL_GENAI_SEMCONV=true`. Autolog set no +`mlflow.llm.provider` in that run, so the provider column stayed empty with +the switch on as well. ### Gaps for the shared processor (SF-1) @@ -448,8 +476,8 @@ prompt (which holds the policy text) and the answer are in `gen_ai.system_instructions`, `gen_ai.input.messages` and `gen_ai.output.messages` instead (tested). `@mlflow.trace` and MLflow's autolog integrations record function arguments, return values and model -messages without being asked (source reading; not tested here). Whatever -MLflow captures lands in Future AGI. +messages without being asked (seen in the autolog run above, not in the +tests). Whatever MLflow captures lands in Future AGI. MLflow's own hook for removing it is a span processor: From 5ed53a62531d41bf8a95a6bfc93415b124a0aec9 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Sun, 11 Oct 2026 20:19:23 -0700 Subject: [PATCH 13/15] docs(examples): note that the MLflow GenAI switch drops function names With MLFLOW_ENABLE_OTEL_GENAI_SEMCONV=true, the live autolog run's tool and agent spans were renamed execute_tool and invoke_agent and carried no gen_ai.tool.name or gen_ai.agent.name, so the function names reached Future AGI nowhere. Say so next to the advice to turn the switch on. Co-Authored-By: Claude Opus 5.5 --- python/examples/mlflow/README.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/python/examples/mlflow/README.md b/python/examples/mlflow/README.md index fd8e03ef4..6a14d7f0c 100644 --- a/python/examples/mlflow/README.md +++ b/python/examples/mlflow/README.md @@ -432,9 +432,12 @@ streamed call and a failed call. Stored: So with the switch off, an autolog trace reaches Future AGI as a tree of `unknown` spans with no model, tokens or cost, and the content only in `mlflow.spanInputs` and `mlflow.spanOutputs`. If you use autolog, set -`MLFLOW_ENABLE_OTEL_GENAI_SEMCONV=true`. Autolog set no -`mlflow.llm.provider` in that run, so the provider column stayed empty with -the switch on as well. +`MLFLOW_ENABLE_OTEL_GENAI_SEMCONV=true`. Know what it costs: the renamed +spans keep no trace of the function names. The `execute_tool` span had no +`gen_ai.tool.name` and the `invoke_agent` span no `gen_ai.agent.name`, so +`days_left` and `refund_agent` appeared nowhere in Future AGI. Autolog set +no `mlflow.llm.provider` in that run, so the provider column stayed empty +with the switch on as well. ### Gaps for the shared processor (SF-1) From 59483b8229e1f1c74a8cbd674f7de9600c126873 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Sun, 11 Oct 2026 20:28:39 -0700 Subject: [PATCH 14/15] feat(examples): turn on MLflow's OTel GenAI keys in the MLflow recipe Decision (Nikhil, 2026-10-11): the Run block now exports MLFLOW_ENABLE_OTEL_GENAI_SEMCONV=true. With it, fi-collector types MLflow's spans (chat -> llm, execute_tool -> tool, CHAIN -> chain) and reads a plain model name, token counts and cost; without it every span is unknown and an autolog model call carries no model or tokens. The cost, renamed spans and lost function names, is documented with how to turn the switch off. README: "What is exported" now leads with the switch-on export, moves the JSON-encoded inventory under "Turning the switch off", and records the live run of the Run block (recipe app and real autolog) against fi-collector. Tests: the recipe environment includes the switch; recipe_run is the translated export (CHAIN > chat gpt-4o-mini) and a switch_off_run fixture keeps the JSON-encoded key checks. The Run block test starts from a stale MLFLOW_ENABLE_OTEL_GENAI_SEMCONV=false and requires the block to set it. MLflow's trace id is now derived from the OTLP trace id, since the switch drops mlflow.traceRequestId; the switch-off run checks the two agree. Co-Authored-By: Claude Opus 5.5 --- python/examples/mlflow/README.md | 228 ++++++++++-------- python/examples/mlflow/src/app.py | 18 +- .../mlflow/tests/test_mlflow_recipe.py | 151 +++++++----- 3 files changed, 231 insertions(+), 166 deletions(-) diff --git a/python/examples/mlflow/README.md b/python/examples/mlflow/README.md index 6a14d7f0c..550a911c6 100644 --- a/python/examples/mlflow/README.md +++ b/python/examples/mlflow/README.md @@ -2,9 +2,10 @@ There is no traceAI package for MLflow. MLflow already exports its own traces over OTLP; this example points that export at fi-collector with -environment variables and adds the Future AGI project as a resource -attribute. It adds no instrumentor: a wrapper around `mlflow.start_span` -would double every span. +environment variables, adds the Future AGI project as a resource +attribute, and turns on MLflow's OpenTelemetry GenAI keys so Future AGI can +type the spans and read the model and token counts. It adds no +instrumentor: a wrapper around `mlflow.start_span` would double every span. This is live export only. Historical import is not this integration: there is no tracking-server client and no import command, and traces that are @@ -51,6 +52,8 @@ export OTEL_RESOURCE_ATTRIBUTES="project_name=my-mlflow-app,project_type=observe export OTEL_SERVICE_NAME="my-mlflow-app" # optional: service.name export MLFLOW_DISABLE_TELEMETRY=true export MLFLOW_MODEL_CATALOG_URI="" # no model-catalog fetch from github.com +# OTel GenAI keys: typed spans, plain model name, tokens and cost in Future AGI (see "What is exported"). +export MLFLOW_ENABLE_OTEL_GENAI_SEMCONV=true unset OTEL_EXPORTER_OTLP_ENDPOINT OTEL_EXPORTER_OTLP_METRICS_ENDPOINT # The exporter falls back to these when the TRACES headers are empty, which could send another vendor's key to Future AGI. unset OTEL_EXPORTER_OTLP_HEADERS @@ -77,13 +80,15 @@ keys elsewhere, or call Databricks. | `OTEL_SERVICE_NAME` | optional | Otherwise `service.name` is OpenTelemetry's `unknown_service` default. | | `MLFLOW_DISABLE_TELEMETRY` | `true` | See [Telemetry](#telemetry). | | `MLFLOW_MODEL_CATALOG_URI` | empty | No model-catalog fetch from github.com; see [Dual export](#dual-export-and-set_destination). | +| `MLFLOW_ENABLE_OTEL_GENAI_SEMCONV` | `true` | MLflow exports OpenTelemetry GenAI keys that fi-collector reads; see [What is exported](#what-is-exported). | | `OTEL_EXPORTER_OTLP_ENDPOINT`, `OTEL_EXPORTER_OTLP_METRICS_ENDPOINT` | unset | They turn on metrics export; see [Endpoint](#endpoint). | | `OTEL_EXPORTER_OTLP_HEADERS` | unset | Sent instead of empty TRACES headers; see [Auth headers](#auth-headers). | | `MLFLOW_TRACING_DESTINATION` | unset | It replaces the OTLP export; see [Dual export](#dual-export-and-set_destination). | | `MLFLOW_TRACKING_URI` | unset, or a non-Databricks URI | See [Databricks tracking URI](#databricks-tracking-uri). | `src/app.py` checks each of these except `OTEL_SERVICE_NAME`, -`MLFLOW_DISABLE_TELEMETRY` and `MLFLOW_MODEL_CATALOG_URI`, and exits with a +`MLFLOW_DISABLE_TELEMETRY`, `MLFLOW_MODEL_CATALOG_URI` and +`MLFLOW_ENABLE_OTEL_GENAI_SEMCONV`, and exits with a message on a value that would lose or misroute the export or make MLflow call Databricks. For the headers it checks the two names, not the values. It refuses @@ -298,61 +303,24 @@ OTLP export. ## What is exported -`src/app.py` exports one trace with two spans: `answer_question` +`src/app.py` makes one trace with two spans: `answer_question` (`span_type=CHAIN`, the root) and its child `chat_model` -(`span_type=CHAT_MODEL`). Both have OTel kind `INTERNAL` and status `OK`. -MLflow's OTLP processor is an OpenTelemetry `BatchSpanProcessor` -(`mlflow/tracing/processor/otel.py:20`), and the OpenTelemetry SDK flushes it -at exit (`opentelemetry/sdk/trace/__init__.py:1283-1284`), so a short script -exports when it ends. - -MLflow JSON-encodes every attribute value before it sets it on the span -(`mlflow/entities/span.py:1456-1490`, `mlflow/tracing/utils/__init__.py:125-142`). -Every value arrives as an OTLP string (tested): a string arrives with its -quotes, a number as its digits. That applies to keys your code sets too: -MLflow lets the app set `gen_ai.*` keys, but `gen_ai.request.model` arrives -as the 13-character string `"gpt-4o-mini"`, quotes included. - -Keys on the default export (tested), and whether fi-collector reads them -(source reading, `pkg/adapter/adapter.go` and -`exporter/clickhouse25exporter/converter.go` at 4af5338): - -| Key | Spans | Value as exported | fi-collector | -|---|---|---|---| -| `mlflow.traceRequestId` | both | `"tr-<32 hex>"`, JSON-encoded | no | -| `mlflow.spanType` | both | `"CHAIN"`, `"CHAT_MODEL"`, JSON-encoded | no: kind is read only from `fi.span.kind`, `gen_ai.span.kind`, `llm.request.type`, `openinference.span.kind`, then `gen_ai.operation.name` (`converter.go:79-131`) | -| `mlflow.spanLogLevel` | both | `10` (CHAIN), `20` (CHAT_MODEL), as strings | no | -| `mlflow.spanInputs` | both | what the code passed to `set_inputs`, as JSON | no: the input column reads only `input.value` (`converter.go:343`) | -| `mlflow.spanOutputs` | both | what the code passed to `set_outputs`, as JSON | no: the output column reads only `output.value` (`converter.go:344`) | -| `mlflow.llm.model` | `chat_model` | `"gpt-4o-mini"`, JSON-encoded | no | -| `mlflow.llm.provider` | `chat_model` | `"openai"`, JSON-encoded | no | -| `mlflow.chat.tokenUsage` | `chat_model` | `{"input_tokens": 12, "output_tokens": 7, "total_tokens": 19}` | no | -| `gen_ai.request.model` | `chat_model` | `"gpt-4o-mini"`, JSON-encoded | model, copied as is, quotes included (`adapter.go:281`, `:317-323`) | -| `gen_ai.provider.name` | `chat_model` | `"openai"`, JSON-encoded | provider, copied as is, quotes included (`adapter.go:290`) | -| `gen_ai.usage.input_tokens` | `chat_model` | `12`, as a string | prompt tokens, parsed from the string (`adapter.go:298`, `:339-342`) | -| `gen_ai.usage.output_tokens` | `chat_model` | `7`, as a string | completion tokens, parsed from the string (`adapter.go:303`) | - -The `mlflow.llm.*` and `mlflow.chat.tokenUsage` keys are what MLflow's -autolog integrations set on a model call; `src/app.py` sets them by hand, -next to the `gen_ai.*` keys. With these keys, fi-collector stores the -model and provider with their quotes, fills the token columns (total derived -as 19, `adapter.go:232-234`), and stores both spans as kind `unknown` -(source reading, and seen once against a local fi-collector; see -[Autolog in Future AGI](#autolog-in-future-agi)). It does not price the tokens: -it looks the price up by exact model name (`converter.go:288-292`, -`pkg/pricing/table.go:99-102`), and `"gpt-4o-mini"` with its quotes matches -no entry, so the cost is 0. The quoted name also keeps these spans apart -from other integrations' `gpt-4o-mini` in model filters. Without a model the -cost would be 0 as well, so the keys are still worth setting for the token -columns. +(`span_type=CHAT_MODEL`), both with status `OK`. MLflow's OTLP processor is +an OpenTelemetry `BatchSpanProcessor` (`mlflow/tracing/processor/otel.py:20`), +and the OpenTelemetry SDK flushes it at exit +(`opentelemetry/sdk/trace/__init__.py:1283-1284`), so a short script exports +when it ends. -### `MLFLOW_ENABLE_OTEL_GENAI_SEMCONV=true` +### `MLFLOW_ENABLE_OTEL_GENAI_SEMCONV=true` (the Run block) -MLflow can translate its own keys to OpenTelemetry GenAI keys before OTLP -export (`mlflow/environment_variables.py:1092-1097`, +The Run block sets `MLFLOW_ENABLE_OTEL_GENAI_SEMCONV=true`. MLflow then +translates its own keys to OpenTelemetry GenAI keys before OTLP export +(`mlflow/environment_variables.py:1092-1097`, `mlflow/tracing/processor/otel.py:81-82`, -`mlflow/tracing/export/genai_semconv/translator.py`). It is off by default -and optional here. Tested with it on: +`mlflow/tracing/export/genai_semconv/translator.py`). MLflow's default is +off. Keys on the export (tested), and whether fi-collector reads them +(source reading, `pkg/adapter/adapter.go` and +`exporter/clickhouse25exporter/converter.go` at 4af5338): | Key | Spans | Value as exported | fi-collector | |---|---|---|---| @@ -365,26 +333,24 @@ and optional here. Tested with it on: | `gen_ai.input.messages` | chat span | the other input messages, as JSON | no: the input column reads only `input.value` | | `gen_ai.output.messages` | chat span | the output messages, as JSON | no: the output column reads only `output.value` | -What else changes (tested): +What else the switch changes (tested): +- Spans are renamed. The model span becomes `chat gpt-4o-mini`, with OTel + kind `CLIENT`; a span with a type but no model is renamed to its type, so + `answer_question` becomes `CHAIN` (`translator.py:164-175`). - Every `mlflow.*` key is dropped, including `mlflow.spanInputs` and `mlflow.spanOutputs`. Chat-shaped inputs and outputs become the three message keys above (`genai_semconv/converter.py:62-80`); the root span's `{"question": ...}` and `{"answer": ...}` are dropped. -- Spans are renamed. The model span becomes `chat gpt-4o-mini`; a span with - a type but no model is renamed to its type, so `answer_question` becomes - `CHAIN` (`translator.py:164-175`). -- The model span's OTel kind becomes `CLIENT`. - The `gen_ai.*` keys the app set itself are overwritten by the translated values, which come from `mlflow.llm.model`, `mlflow.llm.provider` and `mlflow.chat.tokenUsage`. A manual span that sets only `gen_ai.*` keys keeps their JSON-encoded values. -How MLflow's span types would be typed in Future AGI with the switch on -(source reading: the operation name from `translator.py:26-32`, then -fi-collector lower-cases it and applies `spanKindSynonyms` and -`knownObservationTypes`, `converter.go:62-131`). With the switch off every -MLflow span is `unknown`. +How MLflow's span types are typed in Future AGI with the switch on (source +reading: the operation name from `translator.py:26-32`, then fi-collector +lower-cases it and applies `spanKindSynonyms` and `knownObservationTypes`, +`converter.go:62-131`): | MLflow span type | `gen_ai.operation.name` | Future AGI kind | |---|---|---| @@ -404,40 +370,89 @@ MLflow span is `unknown`. | `TASK` | `TASK` | unknown | | `UNKNOWN` | `UNKNOWN` | unknown | -The switch is the only way found to get a model name without quotes and -typed spans out of MLflow 3.16.1 without code. Its cost is the renamed -spans and the dropped non-chat inputs and outputs. That trade-off is yours; -the recipe leaves it off. If you use autolog, read the next section first. +Seen against a local fi-collector (future-agi `dev` ea12cc3, 2026-10-11), +with the Run block's lines run as written (only the origin, keys and +project name substituted): `src/app.py` was stored as `CHAIN` (chain) over +`chat gpt-4o-mini` (llm; model `gpt-4o-mini`, provider `openai`, tokens +12/7/19, priced). For `mlflow.openai.autolog()` under the same variables, +see [Autolog in Future AGI](#autolog-in-future-agi). Neither run filled the +`input` and `output` columns: fi-collector does not lift the +`gen_ai.*.messages` keys (see the SF-1 list below). + +### Turning the switch off + +`unset MLFLOW_ENABLE_OTEL_GENAI_SEMCONV` (or set it to `false`) exports +MLflow's own keys instead. You get back the function names +(`answer_question`, `chat_model`, your `@mlflow.trace` names) and every +span's `mlflow.spanInputs` and `mlflow.spanOutputs`, chain and tool spans +included. You lose what Future AGI shows: fi-collector reads none of the +`mlflow.*` keys, so every span is kind `unknown`, and a model call has a +model, tokens and cost only if your code sets the `gen_ai.*` keys itself, as +`src/app.py` does. Autolog does not, so its model calls then have none of +them. + +With the switch off, MLflow JSON-encodes every attribute value before it +sets it on the span (`mlflow/entities/span.py:1456-1490`, +`mlflow/tracing/utils/__init__.py:125-142`). Every value arrives as an OTLP +string (tested): a string arrives with its quotes, a number as its digits. +That applies to keys your code sets too: MLflow lets the app set `gen_ai.*` +keys, but `gen_ai.request.model` arrives as the 13-character string +`"gpt-4o-mini"`, quotes included. Both spans have OTel kind `INTERNAL`. + +Keys on the export with the switch off (tested), and whether fi-collector +reads them (source reading, as above): + +| Key | Spans | Value as exported | fi-collector | +|---|---|---|---| +| `mlflow.traceRequestId` | both | `"tr-<32 hex>"`, JSON-encoded | no | +| `mlflow.spanType` | both | `"CHAIN"`, `"CHAT_MODEL"`, JSON-encoded | no: kind is read only from `fi.span.kind`, `gen_ai.span.kind`, `llm.request.type`, `openinference.span.kind`, then `gen_ai.operation.name` (`converter.go:79-131`) | +| `mlflow.spanLogLevel` | both | `10` (CHAIN), `20` (CHAT_MODEL), as strings | no | +| `mlflow.spanInputs` | both | what the code passed to `set_inputs`, as JSON | no: the input column reads only `input.value` (`converter.go:343`) | +| `mlflow.spanOutputs` | both | what the code passed to `set_outputs`, as JSON | no: the output column reads only `output.value` (`converter.go:344`) | +| `mlflow.llm.model` | `chat_model` | `"gpt-4o-mini"`, JSON-encoded | no | +| `mlflow.llm.provider` | `chat_model` | `"openai"`, JSON-encoded | no | +| `mlflow.chat.tokenUsage` | `chat_model` | `{"input_tokens": 12, "output_tokens": 7, "total_tokens": 19}` | no | +| `gen_ai.request.model` | `chat_model` | `"gpt-4o-mini"`, JSON-encoded | model, copied as is, quotes included (`adapter.go:281`, `:317-323`) | +| `gen_ai.provider.name` | `chat_model` | `"openai"`, JSON-encoded | provider, copied as is, quotes included (`adapter.go:290`) | +| `gen_ai.usage.input_tokens` | `chat_model` | `12`, as a string | prompt tokens, parsed from the string (`adapter.go:298`, `:339-342`) | +| `gen_ai.usage.output_tokens` | `chat_model` | `7`, as a string | completion tokens, parsed from the string (`adapter.go:303`) | + +The `mlflow.llm.*` and `mlflow.chat.tokenUsage` keys are what MLflow's +autolog integrations set on a model call; `src/app.py` sets them by hand, +next to the `gen_ai.*` keys. With the switch off, fi-collector stores the +model and provider with their quotes, fills the token columns (total derived +as 19, `adapter.go:232-234`), and stores both spans as kind `unknown` +(source reading, and seen against a local fi-collector). It does not price +the tokens: it looks the price up by exact model name (`converter.go:288-292`, +`pkg/pricing/table.go:99-102`), and `"gpt-4o-mini"` with its quotes matches +no entry, so the cost is 0. The quoted name also keeps these spans apart +from other integrations' `gpt-4o-mini` in model filters. ### Autolog in Future AGI -Autolog does not set the `gen_ai.*` keys that `src/app.py` sets by hand. It -writes the model to `mlflow.llm.model` (JSON-encoded) and the token counts to -`mlflow.chat.tokenUsage`, which fi-collector does not read. One run, not part -of the tests: `mlflow.openai.autolog()` on the real `openai` 2.54.0 client, -under the Run block's variables, against a local fi-collector (future-agi -`dev` ea12cc3, 2026-10-11). An `AGENT` function called a chat model with a -tool, a `TOOL` function ran the tool, and two `CHAIN` functions made a -streamed call and a failed call. Stored: +Autolog writes the model to `mlflow.llm.model` and the token counts to +`mlflow.chat.tokenUsage`, and sets no `gen_ai.*` key itself; the switch is +what turns those into keys fi-collector reads. Two runs, not part of the +tests: `mlflow.openai.autolog()` on the real `openai` 2.54.0 client against +a local fi-collector (future-agi `dev` ea12cc3, 2026-10-11). An `AGENT` +function called a chat model with a tool, a `TOOL` function ran the tool, +and two `CHAIN` functions made a streamed call and a failed call. Stored: -| Span | Switch off (the Run block) | `MLFLOW_ENABLE_OTEL_GENAI_SEMCONV=true` | +| Span | Run block (switch on) | Switch off | |---|---|---| -| Model call | named `Completions`; kind `unknown`; no model, no tokens, cost 0 | named `chat `; kind `llm`; model without quotes, prompt and completion tokens (streamed call too); priced when fi-collector's price table has the model | -| Tool function | kind `unknown` | renamed `execute_tool`; kind `tool` | -| Agent function | kind `unknown` | renamed `invoke_agent`; kind `unknown` | -| Chain function | kind `unknown` | renamed `CHAIN`; kind `chain` | +| Model call | named `chat `; kind `llm`; model without quotes, prompt and completion tokens (streamed call too); priced when fi-collector's price table has the model | named `Completions`; kind `unknown`; no model, no tokens, cost 0 | +| Tool function | renamed `execute_tool`; kind `tool` | kind `unknown` | +| Agent function | renamed `invoke_agent`; kind `unknown` | kind `unknown` | +| Chain function | renamed `CHAIN`; kind `chain` | kind `unknown` | | Failed call | status `ERROR` on the model span and its parent | the same | | `input` / `output` columns | empty | empty | -So with the switch off, an autolog trace reaches Future AGI as a tree of -`unknown` spans with no model, tokens or cost, and the content only in -`mlflow.spanInputs` and `mlflow.spanOutputs`. If you use autolog, set -`MLFLOW_ENABLE_OTEL_GENAI_SEMCONV=true`. Know what it costs: the renamed -spans keep no trace of the function names. The `execute_tool` span had no -`gen_ai.tool.name` and the `invoke_agent` span no `gen_ai.agent.name`, so -`days_left` and `refund_agent` appeared nowhere in Future AGI. Autolog set -no `mlflow.llm.provider` in that run, so the provider column stayed empty -with the switch on as well. +What the switch costs with autolog: the renamed spans keep no trace of the +function names. The `execute_tool` span had no `gen_ai.tool.name` and the +`invoke_agent` span no `gen_ai.agent.name`, so `days_left` and +`refund_agent` appeared nowhere in Future AGI, and every chain-rooted trace +is listed as `CHAIN`. Autolog set no `mlflow.llm.provider` in these runs, +so the provider column stayed empty. ### Gaps for the shared processor (SF-1) @@ -472,12 +487,12 @@ harness receiver decodes and records, it does not run fi-collector. MLflow has no setting that keeps tracing on and content off. For `mlflow.start_span`, MLflow records what your code passes to `set_inputs`, -`set_outputs` and `set_attribute(s)`. In the test, the question, the system -prompt (which holds the policy text) and the answer are in -`mlflow.spanInputs` and `mlflow.spanOutputs`, and in no other key. With -`MLFLOW_ENABLE_OTEL_GENAI_SEMCONV=true` they are in +`set_outputs` and `set_attribute(s)`. In the test, with the Run block's +`MLFLOW_ENABLE_OTEL_GENAI_SEMCONV=true`, the system prompt (which holds the +policy text), the question and the answer are in `gen_ai.system_instructions`, `gen_ai.input.messages` and -`gen_ai.output.messages` instead (tested). `@mlflow.trace` and MLflow's +`gen_ai.output.messages`, and in no other key. With the switch off they are +in `mlflow.spanInputs` and `mlflow.spanOutputs` instead (tested). `@mlflow.trace` and MLflow's autolog integrations record function arguments, return values and model messages without being asked (seen in the autolog run above, not in the tests). Whatever MLflow captures lands in Future AGI. @@ -531,16 +546,20 @@ the app's output (tested); they travel only as request headers. - **An `mlflow.db` file appeared.** `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT` is unset, or dual export, `set_destination` or `MLFLOW_TRACING_DESTINATION` is writing to MLflow's default local store. -- **Model name shown with quotes.** That is MLflow's JSON encoding; see - [What is exported](#what-is-exported). +- **Every span `unknown`, no model or tokens, or a model name with + quotes.** `MLFLOW_ENABLE_OTEL_GENAI_SEMCONV=true` is missing, so MLflow + exports its own JSON-encoded keys; see + [Turning the switch off](#turning-the-switch-off). - **Old traces never arrived.** Expected: this is live export, not a backfill. ## Tests `tests/test_mlflow_recipe.py` runs `src/app.py` as written, in a subprocess, -with the real `mlflow` 3.16.1. Each run gets a fresh temporary directory as -its working directory and `HOME`, so every file MLflow writes is checked. +with the real `mlflow` 3.16.1 and the Run block's variables, the GenAI +switch included; separate runs check the export with the switch off. Each +run gets a fresh temporary directory as its working directory and `HOME`, +so every file MLflow writes is checked. Spans go to the shared harness `Receiver` (`python/tests/harness`), which serves `/v1/traces` and `/tracer/v1/traces` on 127.0.0.1 like fi-collector but does not authenticate, stamp projects or store anything. Tests that need @@ -556,8 +575,10 @@ Two tests run the Run block above with `bash --noprofile --norc`: only its substituted, and they check that all of those lines come before the line that runs the app. One runs them over a stale environment (`MLFLOW_TRACING_DESTINATION=0`, another vendor's -`OTEL_EXPORTER_OTLP_HEADERS`, a catalog URI on a loopback recorder) with no -recipe variable set beforehand, then the app. The other leaves a key unset +`OTEL_EXPORTER_OTLP_HEADERS`, a catalog URI on a loopback recorder, +`MLFLOW_ENABLE_OTEL_GENAI_SEMCONV=false`) with no recipe variable set +beforehand, then the app, and checks that the spans arrive translated. The +other leaves a key unset or empty and checks that the header line stops, as a script and in an interactive shell. @@ -565,7 +586,8 @@ interactive shell. `tests/set_destination_app.py` (the app after a `set_destination` call) are fixtures, not part of the recipe. `tests/fixtures/gen_ai_span.otlp.json` is a hand-built OTLP request with two `gen_ai.*` spans, one in the shape MLflow -sends with the GenAI switch on and one in the default JSON-encoded shape. +sends with the GenAI switch on (the Run block) and one in the JSON-encoded +shape it sends with the switch off. The test posts it with the harness `post_otlp()` and checks that it arrives with `gen_ai.request.model` and `gen_ai.usage.input_tokens` intact, and checks each fixture value against a live MLflow run. diff --git a/python/examples/mlflow/src/app.py b/python/examples/mlflow/src/app.py index 7e4b50624..c2190af47 100644 --- a/python/examples/mlflow/src/app.py +++ b/python/examples/mlflow/src/app.py @@ -4,7 +4,9 @@ OTLP when ``OTEL_EXPORTER_OTLP_TRACES_ENDPOINT`` is set before the first span; this file only checks that environment and creates two nested spans with ``mlflow.start_span``. The endpoint, protocol, Future AGI keys and -project come from the environment variables listed in README.md. With those +project come from the environment variables listed in README.md, as does +``MLFLOW_ENABLE_OTEL_GENAI_SEMCONV=true``, which makes MLflow export +OpenTelemetry GenAI keys that Future AGI reads. With those variables (OTLP only, no Databricks tracking URI, ``MLFLOW_MODEL_CATALOG_URI`` empty) MLflow sends the spans to the OTLP endpoint and contacts nothing else (tested); ``check_environment()`` refuses the settings that would make it @@ -137,10 +139,12 @@ def answer(question: str) -> str: llm.set_inputs({"model": MODEL, "messages": messages}) reply, usage = call_model(messages) llm.set_outputs({"choices": [{"message": {"role": "assistant", "content": reply}}]}) - # The OTel GenAI keys fi-collector reads. MLflow JSON-encodes every - # attribute value: the token counts still parse, but the model - # arrives with its quotes, so fi-collector prices nothing (cost 0; - # see README.md). + # The OTel GenAI keys fi-collector reads, for an export with + # MLFLOW_ENABLE_OTEL_GENAI_SEMCONV off. MLflow then JSON-encodes + # every attribute value: the token counts still parse, but the + # model arrives with its quotes, so fi-collector prices nothing + # (cost 0; see README.md). With the switch on, as in the Run + # block, MLflow replaces them with the translated keys below. llm.set_attributes( { "gen_ai.request.model": MODEL, @@ -150,8 +154,8 @@ def answer(question: str) -> str: } ) # The keys MLflow's own autolog sets on a model call. With - # MLFLOW_ENABLE_OTEL_GENAI_SEMCONV=true, MLflow exports them as - # plain gen_ai.* values instead. + # MLFLOW_ENABLE_OTEL_GENAI_SEMCONV=true (the Run block), MLflow + # exports them as plain gen_ai.* model, provider and token keys. llm.set_attributes( { "mlflow.llm.model": MODEL, diff --git a/python/examples/mlflow/tests/test_mlflow_recipe.py b/python/examples/mlflow/tests/test_mlflow_recipe.py index d91a49eff..9944e0a3f 100644 --- a/python/examples/mlflow/tests/test_mlflow_recipe.py +++ b/python/examples/mlflow/tests/test_mlflow_recipe.py @@ -1,6 +1,8 @@ """Contract test for the MLflow OTLP recipe. -Runs ``src/app.py`` as written with the real ``mlflow`` 3.16.1, in a +Runs ``src/app.py`` as written with the real ``mlflow`` 3.16.1 and the Run +block's environment (``MLFLOW_ENABLE_OTEL_GENAI_SEMCONV=true`` included; the +switch-off export is checked too), in a subprocess whose network is limited to 127.0.0.1 (``_guarded_run.py``) and whose working directory and HOME are a fresh temporary directory, so every file MLflow writes is visible to the test. Spans go to the shared harness @@ -15,6 +17,7 @@ from __future__ import annotations +import base64 import contextlib import importlib.util import json @@ -61,8 +64,15 @@ ROOT = "answer_question" LLM = "chat_model" MODEL = "gpt-4o-mini" -# The keys MLflow 3.16.1 puts on each span of src/app.py by default. -DEFAULT_KEYS = { +# Span names on the export with MLFLOW_ENABLE_OTEL_GENAI_SEMCONV=true, as the +# Run block sets it: the model span is " ", and a span with +# a type but no model is named after its type. ROOT and LLM are the names +# MLflow keeps in its own store and exports with the switch off. +CHAT = "chat " + MODEL +CHAIN = "CHAIN" +EXPORTED = {CHAIN, CHAT} +# The keys MLflow 3.16.1 puts on each span of src/app.py with the switch off. +SWITCH_OFF_KEYS = { ROOT: { "mlflow.traceRequestId", "mlflow.spanType", @@ -168,6 +178,7 @@ def _child_env(work: Path, guard_log: Path, endpoint: str, **overrides: Optional "OTEL_RESOURCE_ATTRIBUTES": resource_attributes(PROJECT), "OTEL_SERVICE_NAME": SERVICE_NAME, "MLFLOW_DISABLE_TELEMETRY": "true", + "MLFLOW_ENABLE_OTEL_GENAI_SEMCONV": "true", } for key, value in overrides.items(): if value is None: @@ -289,7 +300,12 @@ def _store(path: Path) -> dict[str, Any]: def _trace_request_id(record: dict[str, Any]) -> str: - ids = {json.loads(_attrs(span)["mlflow.traceRequestId"]) for span in record["spans"]} + """MLflow's id for the exported trace: "tr-" and the OTel trace id in hex. + + Read from the OTLP trace id (base64 in the Receiver's decoded protobuf), + because the GenAI switch drops mlflow.traceRequestId from the export. + """ + ids = {"tr-" + base64.b64decode(span["traceId"]).hex() for span in record["spans"]} assert len(ids) == 1, ids return ids.pop() @@ -376,19 +392,45 @@ def test_resource_carries_the_project(recipe_run: dict[str, Any]) -> None: def test_span_tree(recipe_run: dict[str, Any]) -> None: spans = _by_name(recipe_run["spans"]) - assert set(spans) == {ROOT, LLM} + assert set(spans) == EXPORTED assert len({span["traceId"] for span in spans.values()}) == 1 - assert not spans[ROOT].get("parentSpanId") - assert spans[LLM]["parentSpanId"] == spans[ROOT]["spanId"] + assert not spans[CHAIN].get("parentSpanId") + assert spans[CHAT]["parentSpanId"] == spans[CHAIN]["spanId"] + assert spans[CHAIN]["kind"] == "SPAN_KIND_INTERNAL" + assert spans[CHAT]["kind"] == "SPAN_KIND_CLIENT" for span in spans.values(): - assert span["kind"] == "SPAN_KIND_INTERNAL" assert span["status"] == {"code": "STATUS_CODE_OK"} assert not span.get("events") -def test_mlflow_json_encodes_every_attribute_value(recipe_run: dict[str, Any]) -> None: - spans = _by_name(recipe_run["spans"]) - assert {name: set(_raw(span)) for name, span in spans.items()} == DEFAULT_KEYS +# -------------------------------------------------------------------------- +# MLFLOW_ENABLE_OTEL_GENAI_SEMCONV unset: MLflow's own keys, JSON-encoded. +# -------------------------------------------------------------------------- + + +@pytest.fixture(scope="module") +def switch_off_run(tmp_path_factory: pytest.TempPathFactory) -> dict[str, Any]: + record = _run_script( + tmp_path_factory.mktemp("switch-off"), + APP, + [QUESTION], + MLFLOW_ENABLE_OTEL_GENAI_SEMCONV=None, + ) + _assert_ran(record) + return record + + +def test_switch_off_keeps_the_function_names(switch_off_run: dict[str, Any]) -> None: + spans = _by_name(switch_off_run["spans"]) + assert set(spans) == {ROOT, LLM} + assert spans[LLM]["parentSpanId"] == spans[ROOT]["spanId"] + for span in spans.values(): + assert span["kind"] == "SPAN_KIND_INTERNAL" + + +def test_mlflow_json_encodes_every_attribute_value(switch_off_run: dict[str, Any]) -> None: + spans = _by_name(switch_off_run["spans"]) + assert {name: set(_raw(span)) for name, span in spans.items()} == SWITCH_OFF_KEYS for span in spans.values(): for key, value in _raw(span).items(): assert set(value) == {"stringValue"}, (key, value) @@ -406,13 +448,14 @@ def test_mlflow_json_encodes_every_attribute_value(recipe_run: dict[str, Any]) - "total_tokens": 19, } assert _raw(spans[ROOT])["mlflow.spanType"] == {"stringValue": '"CHAIN"'} - assert _trace_request_id(recipe_run).startswith("tr-") + request_ids = {json.loads(_attrs(span)["mlflow.traceRequestId"]) for span in spans.values()} + assert request_ids == {_trace_request_id(switch_off_run)} -def test_readme_lists_every_exported_key(recipe_run: dict[str, Any]) -> None: +def test_readme_lists_every_switch_off_key(switch_off_run: dict[str, Any]) -> None: text = README.read_text(encoding="utf-8") - keys = set().union(*DEFAULT_KEYS.values()) - for request in recipe_run["requests"]: + keys = set().union(*SWITCH_OFF_KEYS.values()) + for request in switch_off_run["requests"]: for resource in request["resource_attributes"]: keys |= set(resource) missing = sorted(key for key in keys if "`{0}`".format(key) not in text) @@ -429,8 +472,8 @@ def test_no_experiment_or_session_on_the_export(recipe_run: dict[str, Any]) -> N assert not [key for key in resource if "experiment" in key.lower()] -def test_content_is_exported_by_default(recipe_run: dict[str, Any]) -> None: - spans = {name: _attrs(span) for name, span in _by_name(recipe_run["spans"]).items()} +def test_switch_off_puts_content_in_span_inputs_and_outputs(switch_off_run: dict[str, Any]) -> None: + spans = {name: _attrs(span) for name, span in _by_name(switch_off_run["spans"]).items()} assert "QMARK7c1e" in spans[ROOT]["mlflow.spanInputs"] assert ANSWER in spans[ROOT]["mlflow.spanOutputs"] assert POLICY in spans[LLM]["mlflow.spanInputs"] # the system prompt @@ -493,7 +536,7 @@ def test_v1_traces_is_accepted_too(tmp_path: Path) -> None: record = _run_script(tmp_path, APP, [QUESTION], lambda receiver: receiver.endpoint) _assert_ran(record) assert {request["path"] for request in record["requests"]} == {"/v1/traces"} - assert {span["name"] for span in record["spans"]} == {ROOT, LLM} + assert {span["name"] for span in record["spans"]} == EXPORTED @pytest.mark.parametrize( @@ -707,7 +750,7 @@ def test_general_otlp_protocol_is_accepted(tmp_path: Path) -> None: ) _assert_ran(record) assert {request["path"] for request in record["requests"]} == {"/tracer/v1/traces"} - assert {span["name"] for span in record["spans"]} == {ROOT, LLM} + assert {span["name"] for span in record["spans"]} == EXPORTED assert record["files"] == [] @@ -825,7 +868,7 @@ def test_dual_export_false_is_otlp_only(tmp_path: Path) -> None: tmp_path, APP, [QUESTION], MLFLOW_TRACE_ENABLE_OTLP_DUAL_EXPORT="false" ) _assert_ran(record) - assert {span["name"] for span in record["spans"]} == {ROOT, LLM} + assert {span["name"] for span in record["spans"]} == EXPORTED assert record["files"] == [] @@ -838,7 +881,7 @@ def test_dual_export_without_a_tracking_uri_writes_a_local_mlflow_db(tmp_path: P MLFLOW_MODEL_CATALOG_URI="", ) _assert_ran(record) - assert {span["name"] for span in record["spans"]} == {ROOT, LLM} + assert {span["name"] for span in record["spans"]} == EXPORTED # MLflow's default store, SQLite in the working directory. assert record["files"] == ["mlflow.db"] store = _store(record["work"] / "mlflow.db") @@ -856,7 +899,7 @@ def test_dual_export_with_a_tracking_uri_writes_there(tmp_path: Path) -> None: MLFLOW_MODEL_CATALOG_URI="", ) _assert_ran(record) - assert {span["name"] for span in record["spans"]} == {ROOT, LLM} + assert {span["name"] for span in record["spans"]} == EXPORTED assert record["files"] == ["traces.db"] assert _store(database) == {"traces": [_trace_request_id(record)], "spans": [ROOT, LLM]} @@ -879,7 +922,7 @@ def test_dual_export_fetches_the_model_catalog_unless_disabled(tmp_path: Path) - spans = receiver.spans() catalog_calls = list(catalog.calls) _assert_exited_cleanly(record) - assert {span["name"] for span in spans} == {ROOT, LLM} + assert {span["name"] for span in spans} == EXPORTED assert catalog_calls assert set(catalog_calls) == {("GET", "/catalog/openai.json")} @@ -912,7 +955,7 @@ def test_set_destination_with_dual_export_sends_both(tmp_path: Path) -> None: MLFLOW_MODEL_CATALOG_URI="", ) _assert_ran(record) - assert {span["name"] for span in record["spans"]} == {ROOT, LLM} + assert {span["name"] for span in record["spans"]} == EXPORTED assert record["files"] == ["mlflow.db"] assert _store(record["work"] / "mlflow.db") == { "traces": [_trace_request_id(record)], @@ -949,7 +992,7 @@ def test_tracing_destination_with_dual_export_is_accepted_and_sends_both(tmp_pat MLFLOW_MODEL_CATALOG_URI="", ) _assert_ran(record) - assert {span["name"] for span in record["spans"]} == {ROOT, LLM} + assert {span["name"] for span in record["spans"]} == EXPORTED assert record["files"] == ["mlflow.db"] assert _store(record["work"] / "mlflow.db") == { "traces": [_trace_request_id(record)], @@ -1009,7 +1052,7 @@ def _emit_line(path: Path, names: tuple[str, ...]) -> str: def test_readme_run_block_replaces_a_stale_environment(tmp_path: Path) -> None: """The Run block's own lines, run by bash over settings left from another setup. - The shell inherits PATH, HOME, the guard log and three stale values, and + The shell inherits PATH, HOME, the guard log and four stale values, and no recipe variable, so deleting a line from the README fails this test. The stale model catalog URI is a loopback recorder, so a fetch would be recorded, not only blocked. @@ -1018,7 +1061,12 @@ def test_readme_run_block_replaces_a_stale_environment(tmp_path: Path) -> None: work.mkdir() guard_log = tmp_path / "guard.jsonl" emitted = tmp_path / "emitted.json" - stale_names = ("MLFLOW_MODEL_CATALOG_URI", "MLFLOW_TRACING_DESTINATION", "OTEL_EXPORTER_OTLP_HEADERS") + stale_names = ( + "MLFLOW_MODEL_CATALOG_URI", + "MLFLOW_TRACING_DESTINATION", + "OTEL_EXPORTER_OTLP_HEADERS", + "MLFLOW_ENABLE_OTEL_GENAI_SEMCONV", + ) with Receiver() as receiver, _Recorder() as catalog: env = { "PATH": os.environ.get("PATH", ""), @@ -1027,6 +1075,7 @@ def test_readme_run_block_replaces_a_stale_environment(tmp_path: Path) -> None: "MLFLOW_TRACING_DESTINATION": "0", "OTEL_EXPORTER_OTLP_HEADERS": "x-api-key=other-vendor-placeholder", "MLFLOW_MODEL_CATALOG_URI": catalog.origin + "/catalog", + "MLFLOW_ENABLE_OTEL_GENAI_SEMCONV": "false", } app = " ".join(shlex.quote(part) for part in (sys.executable, str(GUARD), str(APP), QUESTION)) script = "\n".join( @@ -1047,7 +1096,7 @@ def test_readme_run_block_replaces_a_stale_environment(tmp_path: Path) -> None: assert result.returncode == 0, stderr assert ANSWER in result.stdout.decode("utf-8", "replace") assert requests, "no export reached the intended receiver" - assert {span["name"] for span in spans} == {ROOT, LLM} + assert {span["name"] for span in spans} == EXPORTED for request in requests: assert request["path"] == "/tracer/v1/traces" assert request["headers"].get("x-api-key") == FI_API_KEY @@ -1060,6 +1109,7 @@ def test_readme_run_block_replaces_a_stale_environment(tmp_path: Path) -> None: "MLFLOW_MODEL_CATALOG_URI": "", "MLFLOW_TRACING_DESTINATION": None, "OTEL_EXPORTER_OTLP_HEADERS": None, + "MLFLOW_ENABLE_OTEL_GENAI_SEMCONV": "true", } assert not guard_log.exists(), guard_log.read_text() # No ./mlflow.db (destination) and no telemetry file under HOME. @@ -1108,29 +1158,15 @@ def test_readme_header_line_stops_when_a_key_is_missing( # -------------------------------------------------------------------------- -# MLFLOW_ENABLE_OTEL_GENAI_SEMCONV=true: MLflow's own GenAI translation. +# MLFLOW_ENABLE_OTEL_GENAI_SEMCONV=true (the Run block): MLflow's own GenAI +# translation. # -------------------------------------------------------------------------- -@pytest.fixture(scope="module") -def semconv_run(tmp_path_factory: pytest.TempPathFactory) -> dict[str, Any]: - record = _run_script( - tmp_path_factory.mktemp("semconv"), - APP, - [QUESTION], - MLFLOW_ENABLE_OTEL_GENAI_SEMCONV="true", - ) - _assert_ran(record) - return record - - -def test_semconv_exports_plain_gen_ai_values(semconv_run: dict[str, Any]) -> None: - spans = _by_name(semconv_run["spans"]) - chat_name = "chat " + MODEL - # The model span is renamed " "; a span with a type but - # no model is renamed to its type. - assert set(spans) == {chat_name, "CHAIN"} - chat, chain = spans[chat_name], spans["CHAIN"] +def test_recipe_exports_plain_gen_ai_values(recipe_run: dict[str, Any]) -> None: + spans = _by_name(recipe_run["spans"]) + assert set(spans) == EXPORTED + chat, chain = spans[CHAT], spans[CHAIN] assert chat["parentSpanId"] == chain["spanId"] assert chat["kind"] == "SPAN_KIND_CLIENT" assert chain["kind"] == "SPAN_KIND_INTERNAL" @@ -1154,20 +1190,23 @@ def test_semconv_exports_plain_gen_ai_values(semconv_run: dict[str, Any]) -> Non assert _raw(chain) == {"gen_ai.operation.name": {"stringValue": "CHAIN"}} -def test_semconv_moves_chat_content_to_gen_ai_messages(semconv_run: dict[str, Any]) -> None: - attrs = _attrs(_by_name(semconv_run["spans"])["chat " + MODEL]) +def test_recipe_puts_chat_content_in_gen_ai_messages(recipe_run: dict[str, Any]) -> None: + attrs = _attrs(_by_name(recipe_run["spans"])[CHAT]) assert "QMARK7c1e" in attrs["gen_ai.input.messages"] assert POLICY in attrs["gen_ai.system_instructions"] # the system message assert POLICY not in attrs["gen_ai.input.messages"] assert ANSWER in attrs["gen_ai.output.messages"] - dump = json.dumps(semconv_run["spans"]) + dump = json.dumps(recipe_run["spans"]) for marker in CONTENT_MARKERS: assert dump.count(marker) == 1, marker -def test_readme_lists_every_semconv_key(semconv_run: dict[str, Any]) -> None: +def test_readme_lists_every_exported_key(recipe_run: dict[str, Any]) -> None: text = README.read_text(encoding="utf-8") - keys = set().union(*(set(_raw(span)) for span in semconv_run["spans"])) + keys = set().union(*(set(_raw(span)) for span in recipe_run["spans"])) + for request in recipe_run["requests"]: + for resource in request["resource_attributes"]: + keys |= set(resource) assert sorted(key for key in keys if "`{0}`".format(key) not in text) == [] @@ -1206,13 +1245,13 @@ def test_hand_built_gen_ai_span_arrives_with_keys_intact() -> None: def test_fixture_spans_are_what_mlflow_sends( - recipe_run: dict[str, Any], semconv_run: dict[str, Any] + recipe_run: dict[str, Any], switch_off_run: dict[str, Any] ) -> None: """The fixture is hand-built, but each span's keys and values are MLflow's own.""" fixture = _fixture_spans() live = { - "chat " + MODEL: _raw(_by_name(semconv_run["spans"])["chat " + MODEL]), - LLM: _raw(_by_name(recipe_run["spans"])[LLM]), + CHAT: _raw(_by_name(recipe_run["spans"])[CHAT]), + LLM: _raw(_by_name(switch_off_run["spans"])[LLM]), } for name, span in fixture.items(): for key, value in _raw(span).items(): From 780ca7d60cc543de02d7f4cdbf756ef512409ba4 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Fri, 16 Oct 2026 19:44:33 -0700 Subject: [PATCH 15/15] fix(examples): drop MLflow exception events and error text from the recipe export (TH-8334 re-verification) Plain MLflow exports an exception event with the stack trace (local paths) and the exception text as the span status message. src/app.py now always drops the exception events; --drop-content also clears the status message and keeps ERROR. Contract test no longer hard-codes the mlflow version (passes on 3.16.1 and 3.17.1); new test with a positive control. Co-Authored-By: Claude Sonnet 5.5 --- python/examples/mlflow/README.md | 38 ++++++++++++++++--- python/examples/mlflow/src/app.py | 30 +++++++++++++-- python/examples/mlflow/tests/failing_app.py | 36 ++++++++++++++++++ .../mlflow/tests/test_mlflow_recipe.py | 37 +++++++++++++++++- 4 files changed, 132 insertions(+), 9 deletions(-) create mode 100644 python/examples/mlflow/tests/failing_app.py diff --git a/python/examples/mlflow/README.md b/python/examples/mlflow/README.md index 550a911c6..f473c73de 100644 --- a/python/examples/mlflow/README.md +++ b/python/examples/mlflow/README.md @@ -500,11 +500,15 @@ tests). Whatever MLflow captures lands in Future AGI. MLflow's own hook for removing it is a span processor: ```python +from mlflow.entities import SpanStatusCode + def drop_content(span): if span.inputs is not None: span.set_inputs("[REDACTED]") if span.outputs is not None: span.set_outputs("[REDACTED]") + if span.status.status_code == SpanStatusCode.ERROR: + span.set_status("ERROR") # keeps ERROR, drops the exception text mlflow.tracing.configure(span_processors=[drop_content]) ``` @@ -515,11 +519,24 @@ switch off and on: none of the question, the policy text or the answer is anywhere in the export; with the switch off the two keys remain, holding `"[REDACTED]"`, and with it on no message key is exported. -Not covered by that processor (source reading, not tested): when code -inside a span raises, MLflow sets the span status to `ERROR` and adds an -`exception` event with the message and stack trace -(`mlflow/tracing/provider.py:309-311`, `mlflow/entities/span.py:1062-1073`); -attributes your code sets with other keys are exported as they are; and +**Errors and stack traces (tested, mlflow 3.16.1 and 3.17.1).** When code inside a +span raises, MLflow sets the span status to `ERROR`, puts the exception text +(`": "`; a provider fills it with request details) in the +status message, and adds an `exception` event holding the message and +`exception.stacktrace`, whose frames are local file paths (so the user +name). Plain MLflow exports all of it. `src/app.py` therefore always +registers `drop_exception_events`, which removes the `exception` events +(MLflow has no public call for that: it clears the underlying OpenTelemetry +span's event list, and does nothing if that attribute is missing), and +`--drop-content` additionally clears the status message +(`drop_content` above). The span stays `ERROR` either way. Copy +`drop_exception_events` from `src/app.py` into your own app if you do not +use it; without it your stack traces and local paths are exported. + +Not covered by those processors (source reading, not tested): attributes +your code sets with other keys are exported as they are; autolog also +exports the tool schemas you pass to the model (`tools`, +`gen_ai.tool.definitions`), which are your code, not user content; and trace tags are written on the root span as `mlflow.traceTag.` after the span processors have run (`mlflow/tracing/processor/otel.py:68-79`). @@ -553,6 +570,17 @@ the app's output (tested); they travel only as request headers. - **Old traces never arrived.** Expected: this is live export, not a backfill. +## Re-verification (2026-10-16) + +Run against mlflow **3.17.1** (the latest stable on PyPI) as well as the +pinned 3.16.1, with `opentelemetry-exporter-otlp-proto-http` 1.45.1 and +1.45.0, Python 3.12. The Run block works unchanged on 3.17.1 and the contract +test passes on both. With 3.17.1 and the real `openai` client against an +OpenAI-compatible endpoint, `mlflow.openai.autolog()` produced the same span +shapes as in the table above (agent `unknown`, tool `tool`, model calls `llm` +with model and tokens, a failed call `ERROR`). Only `requirements.txt` stays on +3.16.1, the version the recipe was written against. + ## Tests `tests/test_mlflow_recipe.py` runs `src/app.py` as written, in a subprocess, diff --git a/python/examples/mlflow/src/app.py b/python/examples/mlflow/src/app.py index c2190af47..e69a808d4 100644 --- a/python/examples/mlflow/src/app.py +++ b/python/examples/mlflow/src/app.py @@ -24,7 +24,7 @@ from urllib.parse import unquote, urlparse import mlflow -from mlflow.entities import LiveSpan, SpanType +from mlflow.entities import LiveSpan, SpanStatusCode, SpanType MODEL = "gpt-4o-mini" PROVIDER = "openai" @@ -111,16 +111,38 @@ def check_environment() -> None: raise SystemExit("Add project_name= to OTEL_RESOURCE_ATTRIBUTES.") +def drop_exception_events(span: LiveSpan) -> None: + """Remove the ``exception`` events from a span before it is exported. + + When a span fails, MLflow records an ``exception`` event holding the + exception message and ``exception.stacktrace``, which contains the local + file paths of the failing process (and so the user name). Nothing in + MLflow's own settings turns that off, and the status stays ERROR without + it. MLflow has no public call that removes an event, so this clears the + underlying OpenTelemetry span's event list (the span has ended, so nothing + appends to it again); the processor is a no-op if that attribute is not there. ``main()`` always registers it. + """ + otel_span = getattr(span, "_span", None) + if getattr(otel_span, "_events", None): + otel_span._events = [] + + def drop_content(span: LiveSpan) -> None: - """Replace a span's inputs and outputs before it is exported. + """Replace a span's inputs and outputs, and the text of its error, before it + is exported. MLflow has no setting that keeps tracing on and content off; a span processor registered with ``mlflow.tracing.configure`` is its own hook. + A failed span keeps status ERROR, but MLflow would otherwise export the + exception text (``": "``, which providers fill with request + details such as model names and sometimes prompt text) as its status message. """ if span.inputs is not None: span.set_inputs(REDACTED) if span.outputs is not None: span.set_outputs(REDACTED) + if span.status.status_code == SpanStatusCode.ERROR: + span.set_status("ERROR") def call_model(messages: list[dict[str, str]]) -> tuple[str, dict[str, int]]: @@ -173,9 +195,11 @@ def answer(question: str) -> str: def main(argv: list[str]) -> int: args = argv[1:] check_environment() + processors = [drop_exception_events] if args[:1] == ["--drop-content"]: - mlflow.tracing.configure(span_processors=[drop_content]) + processors.append(drop_content) args = args[1:] + mlflow.tracing.configure(span_processors=processors) question = args[0] if args else "What is the refund window?" print(answer(question)) # MLflow's OTLP exporter runs in an OpenTelemetry BatchSpanProcessor, and diff --git a/python/examples/mlflow/tests/failing_app.py b/python/examples/mlflow/tests/failing_app.py new file mode 100644 index 000000000..718528a4b --- /dev/null +++ b/python/examples/mlflow/tests/failing_app.py @@ -0,0 +1,36 @@ +"""Test fixture: a failing span, with and without the recipe's ``--drop-content``. + +The contract test runs this to show that MLflow exports the exception text +as the span's status message, and that ``drop_content`` in ``src/app.py`` +removes the text and keeps status ERROR. Not part of the recipe. +""" + +from __future__ import annotations + +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src")) + +import mlflow # noqa: E402 + +import app # noqa: E402 + +app.check_environment() +args = sys.argv[1:] +processors = {"--drop-content": [app.drop_exception_events, app.drop_content], "--drop-events": [app.drop_exception_events]} +if args[:1] and args[0] in processors: + mlflow.tracing.configure(span_processors=processors[args[0]]) + args = args[1:] +marker = args[0] + + +@mlflow.trace(name="failing_step") +def failing_step(text: str) -> str: + raise ValueError("provider rejected the request: " + text) + + +try: + failing_step(marker) +except ValueError: + print("failed as expected") diff --git a/python/examples/mlflow/tests/test_mlflow_recipe.py b/python/examples/mlflow/tests/test_mlflow_recipe.py index 9944e0a3f..726fff532 100644 --- a/python/examples/mlflow/tests/test_mlflow_recipe.py +++ b/python/examples/mlflow/tests/test_mlflow_recipe.py @@ -19,6 +19,7 @@ import base64 import contextlib +import importlib.metadata import importlib.util import json import os @@ -44,6 +45,7 @@ BARE_SPAN = TESTS_DIR / "bare_span.py" SET_DESTINATION = TESTS_DIR / "set_destination_app.py" GUARD = TESTS_DIR / "_guarded_run.py" +FAILING_APP = TESTS_DIR / "failing_app.py" GUARD_PROBE = TESTS_DIR / "_guard_probe.py" GEN_AI_FIXTURE = TESTS_DIR / "fixtures" / "gen_ai_span.otlp.json" @@ -387,7 +389,7 @@ def test_resource_carries_the_project(recipe_run: dict[str, Any]) -> None: assert resource.get("project_type") == "observe" assert resource.get("service.name") == SERVICE_NAME assert resource.get("telemetry.sdk.name") == "mlflow" - assert resource.get("telemetry.sdk.version") == "3.16.1" + assert resource.get("telemetry.sdk.version") == importlib.metadata.version("mlflow") def test_span_tree(recipe_run: dict[str, Any]) -> None: @@ -527,6 +529,39 @@ def test_drop_content_processor_removes_inputs_and_outputs( assert "gen_ai.system_instructions" not in attrs +ERROR_MARKER = "ERRTEXT-8334-9f1c" + + +@pytest.mark.parametrize("mode", ["none", "events", "content"]) +def test_error_text_and_stack_trace_are_dropped_and_status_stays_error( + tmp_path: Path, mode: str +) -> None: + flags = {"none": [], "events": ["--drop-events"], "content": ["--drop-content"]}[mode] + record = _run_script(tmp_path, FAILING_APP, flags + [ERROR_MARKER]) + _assert_exited_cleanly(record) + assert "failed as expected" in record["stdout"] + assert len(record["spans"]) == 1 + span = record["spans"][0] + assert span["status"]["code"] == "STATUS_CODE_ERROR", "the failed span must stay ERROR" + dump = _export_dump(record) + if mode == "none": + # positive control: plain MLflow exports the message, the stack trace and local paths + assert ERROR_MARKER in span["status"]["message"] + assert "exception.stacktrace" in dump + assert "Traceback" in dump + else: + # no stack trace, no local path, no exception event in either processor mode + assert "exception.stacktrace" not in dump + assert "Traceback" not in dump + assert str(Path(__file__).resolve().parents[4]) not in dump + assert not span.get("events") + if mode == "events": + assert ERROR_MARKER in span["status"]["message"], "--drop-events keeps the status message" + if mode == "content": + assert ERROR_MARKER not in dump + assert not span["status"].get("message") + + # -------------------------------------------------------------------------- # Endpoint, protocol and the settings src/app.py refuses. # --------------------------------------------------------------------------