From 5936e0960f01a23e842d3df90b7c101874ed8149 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Sat, 3 Oct 2026 18:58:31 -0700 Subject: [PATCH 01/16] test(harness): add shared OTLP harness for TH-8103 contract tests --- python/tests/harness/__init__.py | 184 +++++++++++++++++++++++++++ python/tests/harness/test_harness.py | 101 +++++++++++++++ 2 files changed, 285 insertions(+) create mode 100644 python/tests/harness/__init__.py create mode 100644 python/tests/harness/test_harness.py diff --git a/python/tests/harness/__init__.py b/python/tests/harness/__init__.py new file mode 100644 index 000000000..dfd9198e4 --- /dev/null +++ b/python/tests/harness/__init__.py @@ -0,0 +1,184 @@ +"""Small, loopback-only utilities for OTLP contract tests.""" + +from __future__ import annotations + +import copy +import difflib +import json +import subprocess +import threading +from http import HTTPStatus +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from typing import Any, Mapping, Optional, Sequence, Union +from urllib.parse import urlsplit +from urllib.request import Request, urlopen + + +class Receiver: + """An in-memory OTLP/HTTP receiver for tests. + + The receiver only listens on loopback and never forwards received spans. + """ + + def __init__(self) -> None: + self._spans: list[dict[str, Any]] = [] + self._lock = threading.Lock() + owner = self + + class Handler(BaseHTTPRequestHandler): + def do_POST(self) -> None: # noqa: N802 - required by BaseHTTPRequestHandler + if urlsplit(self.path).path != "/v1/traces": + self.send_error(HTTPStatus.NOT_FOUND) + return + + try: + length = int(self.headers.get("Content-Length", "0")) + body = self.rfile.read(length) + content_type = self.headers.get("Content-Type", "").lower() + + if content_type.startswith("application/json"): + request = json.loads(body.decode("utf-8")) + elif content_type.startswith(("application/x-protobuf", "application/protobuf")): + from google.protobuf.json_format import MessageToDict + from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ( + ExportTraceServiceRequest, + ) + + protobuf_request = ExportTraceServiceRequest() + protobuf_request.ParseFromString(body) + request = MessageToDict(protobuf_request) + else: + self.send_error(HTTPStatus.UNSUPPORTED_MEDIA_TYPE) + return + + spans: list[dict[str, Any]] = [] + for resource_spans in request.get( + "resourceSpans", request.get("resource_spans", []) + ): + for scope_spans in resource_spans.get( + "scopeSpans", resource_spans.get("scope_spans", []) + ): + spans.extend(scope_spans.get("spans", [])) + except (ImportError, TypeError, UnicodeDecodeError, ValueError) as error: + self.send_error(HTTPStatus.BAD_REQUEST, str(error)) + return + + with owner._lock: + owner._spans.extend(copy.deepcopy(spans)) + self.send_response(HTTPStatus.OK) + self.send_header("Content-Length", "0") + self.end_headers() + + def log_message(self, _format: str, *args: Any) -> None: + return + + self._server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + self.origin = "http://127.0.0.1:{0}".format(self._server.server_port) + self.endpoint = "{0}/v1/traces".format(self.origin) + self._thread = threading.Thread(target=self._server.serve_forever, daemon=True) + self._thread.start() + + def __enter__(self) -> "Receiver": + return self + + def __exit__(self, _type: Any, _value: Any, _traceback: Any) -> None: + self._server.shutdown() + self._server.server_close() + self._thread.join() + + def spans(self) -> list[dict[str, Any]]: + """Return a copy of the decoded spans received so far.""" + with self._lock: + return copy.deepcopy(self._spans) + + def clear(self) -> None: + """Remove every decoded span received so far.""" + with self._lock: + self._spans.clear() + + +def run( + argv: Sequence[str], + env: Mapping[str, str], + stdin: Optional[bytes], + timeout: float, +) -> subprocess.CompletedProcess[bytes]: + """Run one process and return its captured output, including after a timeout.""" + process = subprocess.Popen( + argv, + env=dict(env), + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + timed_out = False + try: + stdout, stderr = process.communicate(input=stdin, timeout=timeout) + except subprocess.TimeoutExpired: + timed_out = True + process.kill() + stdout, stderr = process.communicate() + + result = subprocess.CompletedProcess(argv, process.returncode, stdout, stderr) + result.timed_out = timed_out + return result + + +def post_otlp(spans: dict[str, Any], endpoint: str) -> int: + """Post one JSON-encoded OTLP request to a loopback receiver.""" + parsed = urlsplit(endpoint) + if parsed.scheme != "http" or parsed.hostname != "127.0.0.1": + raise ValueError("post_otlp only sends to a loopback HTTP endpoint") + + request = Request( + endpoint, + data=json.dumps(spans).encode("utf-8"), + headers={"Content-Type": "application/json"}, + method="POST", + ) + with urlopen(request, timeout=5) as response: + return response.status + + +def compare(actual: Sequence[dict[str, Any]], golden_path: Union[str, Path]) -> None: + """Fail with a diff unless span names, attributes, and statuses match a golden.""" + with Path(golden_path).open(encoding="utf-8") as golden_file: + expected = json.load(golden_file) + + def canonical(spans: Sequence[dict[str, Any]]) -> list[dict[str, Any]]: + normalized = [] + for span in spans: + attributes = span.get("attributes", []) + if isinstance(attributes, dict): + attributes = [ + {"key": key, "value": value} for key, value in attributes.items() + ] + normalized.append( + { + "name": span.get("name", ""), + "attributes": sorted( + attributes, + key=lambda attribute: json.dumps( + attribute, sort_keys=True, separators=(",", ":") + ), + ), + "status": span.get("status", {}), + } + ) + return normalized + + actual_normalized = canonical(actual) + expected_normalized = canonical(expected) + if actual_normalized != expected_normalized: + diff = "\n".join( + difflib.unified_diff( + json.dumps(expected_normalized, indent=2, sort_keys=True).splitlines(), + json.dumps(actual_normalized, indent=2, sort_keys=True).splitlines(), + fromfile=str(golden_path), + tofile="actual", + lineterm="", + ) + ) + print(diff) + raise AssertionError("spans do not match golden") diff --git a/python/tests/harness/test_harness.py b/python/tests/harness/test_harness.py new file mode 100644 index 000000000..4e76ceabe --- /dev/null +++ b/python/tests/harness/test_harness.py @@ -0,0 +1,101 @@ +"""Contract tests for the shared loopback-only harness.""" + +import json +import sys + +import pytest + +from tests.harness import Receiver, compare, post_otlp, run + + +def test_receiver_stores_posted_span_and_can_clear() -> None: + payload = { + "resourceSpans": [ + { + "scopeSpans": [ + { + "spans": [ + { + "traceId": "trace-id", + "spanId": "span-id", + "name": "harness.receiver", + "attributes": [ + {"key": "component", "value": {"stringValue": "test"}} + ], + "status": {"code": 1}, + } + ] + } + ] + } + ] + } + + with Receiver() as receiver: + assert post_otlp(payload, receiver.endpoint) == 200 + assert receiver.spans() == payload["resourceSpans"][0]["scopeSpans"][0]["spans"] + receiver.clear() + assert receiver.spans() == [] + + +def test_run_captures_output_and_kills_timed_out_process() -> None: + completed = run( + [ + sys.executable, + "-c", + "import sys; print('stdout'); sys.stderr.write('stderr\\n')", + ], + env={}, + stdin=b"", + timeout=1, + ) + assert completed.returncode == 0 + assert completed.stdout == b"stdout\n" + assert completed.stderr == b"stderr\n" + assert completed.timed_out is False + + timed_out = run( + [sys.executable, "-c", "import time; time.sleep(30)"], + env={}, + stdin=b"", + timeout=0.1, + ) + assert timed_out.timed_out is True + assert timed_out.returncode != 0 + + +def test_compare_accepts_normalized_match_and_prints_mismatch_diff(tmp_path, capsys) -> None: + golden = [ + { + "traceId": "golden-trace", + "spanId": "golden-span", + "startTimeUnixNano": "1", + "endTimeUnixNano": "2", + "name": "harness.compare", + "attributes": [ + {"key": "alpha", "value": {"stringValue": "a"}}, + {"key": "beta", "value": {"intValue": "2"}}, + ], + "status": {"code": 1}, + } + ] + golden_path = tmp_path / "spans.golden.json" + golden_path.write_text(json.dumps(golden), encoding="utf-8") + + actual = [ + { + "traceId": "other-trace", + "spanId": "other-span", + "startTimeUnixNano": "3", + "endTimeUnixNano": "4", + "name": "harness.compare", + "attributes": list(reversed(golden[0]["attributes"])), + "status": {"code": 1}, + } + ] + compare(actual, golden_path) + + actual[0]["name"] = "harness.mismatch" + with pytest.raises(AssertionError, match="spans do not match golden"): + compare(actual, golden_path) + assert "---" in capsys.readouterr().out From 32f5c9d2b3b80f97a70e00b3f3ca1b408b14dc5c Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Sat, 3 Oct 2026 23:21:29 -0700 Subject: [PATCH 02/16] test(harness): accept the collector /tracer/v1/traces path --- python/tests/harness/__init__.py | 3 ++- python/tests/harness/test_harness.py | 12 ++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/python/tests/harness/__init__.py b/python/tests/harness/__init__.py index dfd9198e4..cb510ad10 100644 --- a/python/tests/harness/__init__.py +++ b/python/tests/harness/__init__.py @@ -28,7 +28,7 @@ def __init__(self) -> None: class Handler(BaseHTTPRequestHandler): def do_POST(self) -> None: # noqa: N802 - required by BaseHTTPRequestHandler - if urlsplit(self.path).path != "/v1/traces": + if urlsplit(self.path).path not in ("/v1/traces", "/tracer/v1/traces"): self.send_error(HTTPStatus.NOT_FOUND) return @@ -76,6 +76,7 @@ def log_message(self, _format: str, *args: Any) -> None: self._server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) self.origin = "http://127.0.0.1:{0}".format(self._server.server_port) self.endpoint = "{0}/v1/traces".format(self.origin) + self.collector_endpoint = "{0}/tracer/v1/traces".format(self.origin) self._thread = threading.Thread(target=self._server.serve_forever, daemon=True) self._thread.start() diff --git a/python/tests/harness/test_harness.py b/python/tests/harness/test_harness.py index 4e76ceabe..6a32c2af8 100644 --- a/python/tests/harness/test_harness.py +++ b/python/tests/harness/test_harness.py @@ -38,6 +38,18 @@ def test_receiver_stores_posted_span_and_can_clear() -> None: assert receiver.spans() == [] +def test_receiver_accepts_the_collector_path() -> None: + payload = { + "resourceSpans": [ + {"scopeSpans": [{"spans": [{"name": "collector.path"}]}]} + ] + } + + with Receiver() as receiver: + assert post_otlp(payload, receiver.collector_endpoint) == 200 + assert receiver.spans() == [{"name": "collector.path"}] + + def test_run_captures_output_and_kills_timed_out_process() -> None: completed = run( [ From 3eaadc8bdbf428a472db6779091bbce3d6d89015 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Sun, 4 Oct 2026 15:59:06 -0700 Subject: [PATCH 03/16] fix(harness): decode chunked OTLP bodies and record request metadata The Node OTLP exporter streams with Transfer-Encoding: chunked and sends no Content-Length. The Receiver read Content-Length only, so every Node export came back 400 with zero spans; two TH-8103 children had to put a de-chunking relay in front of it. Receiver now accepts both framings. It also records one entry per accepted export (path, lower-cased headers, flattened resource attributes) via requests(), so a contract test can assert X-Api-Key/X-Secret-Key and project_name through the shared harness instead of a private recorder. Verified: 6 harness tests pass, and the TanStack example's real Node exporter delivered 4 chunked exports (4 spans, collector path, both auth headers, project_name/project_type) with no relay. Refs: TH-8339, TH-8103 --- python/tests/harness/__init__.py | 61 ++++++++++++++++++++-- python/tests/harness/test_harness.py | 76 ++++++++++++++++++++++++++++ 2 files changed, 133 insertions(+), 4 deletions(-) diff --git a/python/tests/harness/__init__.py b/python/tests/harness/__init__.py index cb510ad10..cf79047f1 100644 --- a/python/tests/harness/__init__.py +++ b/python/tests/harness/__init__.py @@ -23,18 +23,36 @@ class Receiver: def __init__(self) -> None: self._spans: list[dict[str, Any]] = [] + self._requests: list[dict[str, Any]] = [] self._lock = threading.Lock() owner = self class Handler(BaseHTTPRequestHandler): + def _read_body(self) -> bytes: + # The Node OTLP exporter streams with Transfer-Encoding: chunked + # and sends no Content-Length, so both framings are accepted. + if "chunked" in self.headers.get("Transfer-Encoding", "").lower(): + chunks = [] + while True: + size_line = self.rfile.readline().split(b";", 1)[0].strip() + size = int(size_line, 16) + if size == 0: + while self.rfile.readline() not in (b"\r\n", b"\n", b""): + pass + return b"".join(chunks) + chunks.append(self.rfile.read(size)) + self.rfile.readline() + length = int(self.headers.get("Content-Length", "0")) + return self.rfile.read(length) + def do_POST(self) -> None: # noqa: N802 - required by BaseHTTPRequestHandler - if urlsplit(self.path).path not in ("/v1/traces", "/tracer/v1/traces"): + path = urlsplit(self.path).path + if path not in ("/v1/traces", "/tracer/v1/traces"): self.send_error(HTTPStatus.NOT_FOUND) return try: - length = int(self.headers.get("Content-Length", "0")) - body = self.rfile.read(length) + body = self._read_body() content_type = self.headers.get("Content-Type", "").lower() if content_type.startswith("application/json"): @@ -53,9 +71,15 @@ def do_POST(self) -> None: # noqa: N802 - required by BaseHTTPRequestHandler return spans: list[dict[str, Any]] = [] + resource_attributes: list[dict[str, Any]] = [] for resource_spans in request.get( "resourceSpans", request.get("resource_spans", []) ): + resource_attributes.append( + _flatten_attributes( + resource_spans.get("resource", {}).get("attributes", []) + ) + ) for scope_spans in resource_spans.get( "scopeSpans", resource_spans.get("scope_spans", []) ): @@ -66,6 +90,15 @@ def do_POST(self) -> None: # noqa: N802 - required by BaseHTTPRequestHandler with owner._lock: owner._spans.extend(copy.deepcopy(spans)) + owner._requests.append( + { + "path": path, + "headers": { + key.lower(): value for key, value in self.headers.items() + }, + "resource_attributes": resource_attributes, + } + ) self.send_response(HTTPStatus.OK) self.send_header("Content-Length", "0") self.end_headers() @@ -93,10 +126,30 @@ def spans(self) -> list[dict[str, Any]]: with self._lock: return copy.deepcopy(self._spans) + def requests(self) -> list[dict[str, Any]]: + """Return one record per accepted export: path, lower-cased headers, resource attributes.""" + with self._lock: + return copy.deepcopy(self._requests) + def clear(self) -> None: - """Remove every decoded span received so far.""" + """Remove every decoded span and request record received so far.""" with self._lock: self._spans.clear() + self._requests.clear() + + +def _flatten_attributes(attributes: Sequence[Mapping[str, Any]]) -> dict[str, Any]: + """Turn OTLP key/value attributes into a plain dict of scalar values.""" + flat: dict[str, Any] = {} + for attribute in attributes: + value = attribute.get("value", {}) + for kind in ("stringValue", "boolValue", "intValue", "doubleValue"): + if kind in value: + flat[attribute.get("key", "")] = value[kind] + break + else: + flat[attribute.get("key", "")] = value + return flat def run( diff --git a/python/tests/harness/test_harness.py b/python/tests/harness/test_harness.py index 6a32c2af8..bd61d5adc 100644 --- a/python/tests/harness/test_harness.py +++ b/python/tests/harness/test_harness.py @@ -50,6 +50,82 @@ def test_receiver_accepts_the_collector_path() -> None: assert receiver.spans() == [{"name": "collector.path"}] +def _post_chunked(url: str, body: bytes, headers: dict) -> int: + """Send a body with Transfer-Encoding: chunked, as the Node OTLP exporter does.""" + import http.client + from urllib.parse import urlsplit + + parts = urlsplit(url) + connection = http.client.HTTPConnection(parts.hostname, parts.port, timeout=5) + try: + chunks = [body[: len(body) // 2], body[len(body) // 2 :]] + connection.request( + "POST", + parts.path, + body=iter(chunks), + headers={**headers, "Transfer-Encoding": "chunked"}, + encode_chunked=True, + ) + return connection.getresponse().status + finally: + connection.close() + + +def test_receiver_decodes_a_chunked_body() -> None: + payload = { + "resourceSpans": [ + {"scopeSpans": [{"spans": [{"name": "chunked.body"}]}]} + ] + } + + with Receiver() as receiver: + status = _post_chunked( + receiver.collector_endpoint, + json.dumps(payload).encode("utf-8"), + {"Content-Type": "application/json"}, + ) + assert status == 200 + assert receiver.spans() == [{"name": "chunked.body"}] + + +def test_receiver_records_path_headers_and_resource_attributes() -> None: + payload = { + "resourceSpans": [ + { + "resource": { + "attributes": [ + {"key": "project_name", "value": {"stringValue": "harness"}}, + {"key": "project_type", "value": {"stringValue": "observe"}}, + ] + }, + "scopeSpans": [{"spans": [{"name": "with.resource"}]}], + } + ] + } + + with Receiver() as receiver: + status = _post_chunked( + receiver.collector_endpoint, + json.dumps(payload).encode("utf-8"), + { + "Content-Type": "application/json", + "X-Api-Key": "test-key", + "X-Secret-Key": "test-secret", + }, + ) + assert status == 200 + [request] = receiver.requests() + assert request["path"] == "/tracer/v1/traces" + assert request["headers"]["x-api-key"] == "test-key" + assert request["headers"]["x-secret-key"] == "test-secret" + assert "authorization" not in request["headers"] + assert request["resource_attributes"] == [ + {"project_name": "harness", "project_type": "observe"} + ] + receiver.clear() + assert receiver.requests() == [] + + def test_run_captures_output_and_kills_timed_out_process() -> None: completed = run( [ From 67982e50147c45c3bd65ad528bcd50e5e6834909 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Mon, 5 Oct 2026 09:21:15 -0700 Subject: [PATCH 04/16] feat(examples): add OpenLIT recipe app openlit.init() with the collector origin, the Future AGI keys as percent-encoded otlp_headers, content, metrics and events off, and a local empty price table so init() does not download one from GitHub. The project comes from OTEL_RESOURCE_ATTRIBUTES; the app refuses to start without it or without an endpoint. Pins openlit 1.45.0 and openai 2.54.0. approved: Nikhil 2026-10-03 blanket Refs: TH-8338 Co-Authored-By: Claude Opus 5.5 --- python/examples/openlit/requirements.txt | 4 + python/examples/openlit/src/app.py | 84 +++++++++++++++++++++ python/examples/openlit/src/no_pricing.json | 1 + 3 files changed, 89 insertions(+) create mode 100644 python/examples/openlit/requirements.txt create mode 100644 python/examples/openlit/src/app.py create mode 100644 python/examples/openlit/src/no_pricing.json diff --git a/python/examples/openlit/requirements.txt b/python/examples/openlit/requirements.txt new file mode 100644 index 000000000..7d067fff6 --- /dev/null +++ b/python/examples/openlit/requirements.txt @@ -0,0 +1,4 @@ +# Not a traceAI package: these are the example's own dependencies. +openlit==1.45.0 +# openlit 1.45.0 requires openai>=1.92.0,<3.0.0; this is the version the contract test ran with. +openai==2.54.0 diff --git a/python/examples/openlit/src/app.py b/python/examples/openlit/src/app.py new file mode 100644 index 000000000..fd08fe4f7 --- /dev/null +++ b/python/examples/openlit/src/app.py @@ -0,0 +1,84 @@ +"""Send OpenLIT traces to Future AGI. + +Recipe example, not a package. OpenLIT already exports OTLP/HTTP; this file +only calls ``openlit.init`` with the collector endpoint, the Future AGI keys +and the switches README.md explains, then makes one OpenAI call that +OpenLIT's auto-instrumentation traces. The project comes from the standard +``OTEL_RESOURCE_ATTRIBUTES`` variable (see README.md). + + python src/app.py "What is the refund window?" +""" + +from __future__ import annotations + +import os +import sys +from pathlib import Path +from urllib.parse import quote + +import openlit +from openai import OpenAI +from opentelemetry.sdk.resources import OTELResourceDetector + +APP_NAME = "openlit-recipe" # exported as the service.name resource attribute +MODEL = os.environ.get("OPENAI_MODEL", "gpt-4o-mini") +# An empty price table. Without pricing_json, openlit.init() downloads one +# from raw.githubusercontent.com on every start. +NO_PRICING = Path(__file__).resolve().with_name("no_pricing.json") + +_POLICY = "Refunds are accepted within 30 days of purchase." + + +def init_tracing() -> None: + """Start OpenLIT's exporter. Call once per process, before any model call.""" + endpoint = os.environ.get("OTEL_EXPORTER_OTLP_ENDPOINT") + if not endpoint: + # Without an endpoint OpenLIT prints every span to stdout instead. + raise SystemExit("Set OTEL_EXPORTER_OTLP_ENDPOINT to the fi-collector origin.") + # openlit.init() has no resource argument; its Resource.create() merges + # OTEL_RESOURCE_ATTRIBUTES. fi-collector rejects (HTTP 400) a batch whose + # resource has no project_name, so stop here instead. + if not OTELResourceDetector().detect().attributes.get("project_name"): + raise SystemExit( + "Set OTEL_RESOURCE_ATTRIBUTES=project_name=,project_type=observe" + ) + openlit.init( + service_name=APP_NAME, + # The origin, no path: the OTLP/HTTP exporter appends /v1/traces. + otlp_endpoint=endpoint, + # OpenLIT joins this dict into OTEL_EXPORTER_OTLP_HEADERS, which the + # exporter splits on commas and percent-decodes: encode the values. + otlp_headers={ + "x-api-key": quote(os.environ["FI_API_KEY"], safe=""), + "x-secret-key": quote(os.environ["FI_SECRET_KEY"], safe=""), + }, + capture_message_content=False, # OpenLIT's default is True + disable_metrics=True, # fi-collector serves no /v1/metrics + disable_events=True, # events go to /v1/logs, which fi-collector does not serve + pricing_json=str(NO_PRICING), + ) + + +def ask(question: str) -> str: + # OpenAI() reads OPENAI_API_KEY and OPENAI_BASE_URL. + response = OpenAI().chat.completions.create( + model=MODEL, + messages=[ + {"role": "system", "content": "Answer using only this policy: " + _POLICY}, + {"role": "user", "content": question}, + ], + ) + return response.choices[0].message.content or "" + + +def main(argv: list[str]) -> int: + init_tracing() + question = argv[1] if len(argv) > 1 else "What is the refund window?" + print(ask(question)) + # The SDK TracerProvider flushes its batch processor at exit, so a short + # script needs no more. A long-lived server keeps exporting in batches. + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/python/examples/openlit/src/no_pricing.json b/python/examples/openlit/src/no_pricing.json new file mode 100644 index 000000000..0967ef424 --- /dev/null +++ b/python/examples/openlit/src/no_pricing.json @@ -0,0 +1 @@ +{} From 186792b6b4283ef6063312fbe6c6352424e76e79 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Mon, 5 Oct 2026 09:21:15 -0700 Subject: [PATCH 05/16] test(examples): add loopback OpenAI fake and network guard for OpenLIT The same helpers as the OpenLLMetry recipe: a 127.0.0.1 Chat Completions fake, a runner that refuses and logs every non-loopback connection, and a probe that the positive-control test runs through it. approved: Nikhil 2026-10-03 blanket Refs: TH-8338 Co-Authored-By: Claude Opus 5.5 --- python/examples/openlit/tests/_fake_openai.py | 76 +++++++++++++++++ python/examples/openlit/tests/_guard_probe.py | 26 ++++++ python/examples/openlit/tests/_guarded_run.py | 81 +++++++++++++++++++ 3 files changed, 183 insertions(+) create mode 100644 python/examples/openlit/tests/_fake_openai.py create mode 100644 python/examples/openlit/tests/_guard_probe.py create mode 100644 python/examples/openlit/tests/_guarded_run.py diff --git a/python/examples/openlit/tests/_fake_openai.py b/python/examples/openlit/tests/_fake_openai.py new file mode 100644 index 000000000..e3426e180 --- /dev/null +++ b/python/examples/openlit/tests/_fake_openai.py @@ -0,0 +1,76 @@ +"""A loopback stand-in for the OpenAI Chat Completions API (non-streaming). + +Every POST to /v1/chat/completions gets one assistant message whose text is +``answer_text``, with fixed token usage, so the instrumentor records model +and usage attributes. Each request body and Authorization header is kept for +assertions. Listens on 127.0.0.1 only. +""" + +from __future__ import annotations + +import json +import threading +from http import HTTPStatus +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from typing import Any + +RESPONSE_MODEL = "gpt-4o-mini-2024-07-18" +USAGE = {"prompt_tokens": 11, "completion_tokens": 7, "total_tokens": 18} + + +class FakeOpenAI: + """Serve /v1/chat/completions on 127.0.0.1 and record each request.""" + + def __init__(self, answer_text: str) -> None: + self.requests: list[dict[str, Any]] = [] + self.authorizations: list[str | None] = [] + lock = threading.Lock() + owner = self + + class Handler(BaseHTTPRequestHandler): + def do_POST(self) -> None: # noqa: N802 - BaseHTTPRequestHandler API + if self.path.rstrip("/") != "/v1/chat/completions": + self.send_error(HTTPStatus.NOT_FOUND) + return + length = int(self.headers.get("Content-Length", "0")) + request = json.loads(self.rfile.read(length).decode("utf-8")) + with lock: + owner.requests.append(request) + owner.authorizations.append(self.headers.get("Authorization")) + body = json.dumps( + { + "id": "chatcmpl-fake", + "object": "chat.completion", + "created": 1_700_000_000, + "model": RESPONSE_MODEL, + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": answer_text}, + "finish_reason": "stop", + } + ], + "usage": USAGE, + } + ).encode("utf-8") + self.send_response(HTTPStatus.OK) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, format: str, *args: Any) -> None: # noqa: A002 + return + + self._server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + self.base_url = "http://127.0.0.1:{0}/v1".format(self._server.server_port) + self._thread = threading.Thread(target=self._server.serve_forever, daemon=True) + self._thread.start() + + def __enter__(self) -> "FakeOpenAI": + return self + + def __exit__(self, *_exc: Any) -> None: + self._server.shutdown() + self._server.server_close() + self._thread.join() diff --git a/python/examples/openlit/tests/_guard_probe.py b/python/examples/openlit/tests/_guard_probe.py new file mode 100644 index 000000000..5ebeff59d --- /dev/null +++ b/python/examples/openlit/tests/_guard_probe.py @@ -0,0 +1,26 @@ +"""A test-only probe: try three non-loopback connections under the loopback guard. + +The contract test runs this through ``_guarded_run.py`` to prove the guard +really refuses and logs a non-loopback attempt, so the ``guard_attempts == []`` +assertions elsewhere cannot pass because the guard silently stopped working. +The targets are reserved and never reachable: TEST-NET-1 (RFC 5737), the IPv6 +documentation prefix (RFC 3849) and the ``.invalid`` TLD (RFC 2606). +Not part of the recipe. +""" + +from __future__ import annotations + +import socket + +refused = [] +for label, attempt in ( + ("connect", lambda: socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect(("192.0.2.1", 80))), + ("connect_ex", lambda: socket.socket(socket.AF_INET6, socket.SOCK_STREAM).connect_ex(("2001:db8::1", 80, 0, 0))), + ("getaddrinfo", lambda: socket.getaddrinfo("example.invalid", 443)), +): + try: + attempt() + except OSError as error: + if "blocked by loopback guard" in str(error): + refused.append(label) +print("refused:", ",".join(refused)) diff --git a/python/examples/openlit/tests/_guarded_run.py b/python/examples/openlit/tests/_guarded_run.py new file mode 100644 index 000000000..2295b36fc --- /dev/null +++ b/python/examples/openlit/tests/_guarded_run.py @@ -0,0 +1,81 @@ +"""Run a script with every non-loopback network connection blocked and logged. + +Usage: python _guarded_run.py [args...] + +Before the script starts, ``socket.socket.connect``/``connect_ex`` and +``socket.getaddrinfo`` refuse any host other than 127.0.0.1, ::1 or +localhost. Each refused attempt is appended as a JSON line to the file named +by ``LOOPBACK_GUARD_LOG``, so a test can assert that the SDK, its +instrumentors and the example tried to reach nothing else (no pricing +download, no OpenLIT host, no real OpenAI). Unix sockets are not affected. +""" + +from __future__ import annotations + +import json +import os +import runpy +import socket +import sys + +_ALLOWED_HOSTS = {"127.0.0.1", "::1", "localhost"} +_LOG_PATH = os.environ.get("LOOPBACK_GUARD_LOG", "") + + +def _host(value: object) -> str: + if isinstance(value, bytes): + return value.decode("ascii", "replace") + return str(value) + + +def _refuse(kind: str, target: object) -> None: + if _LOG_PATH: + with open(_LOG_PATH, "a", encoding="utf-8") as log: + log.write(json.dumps({"kind": kind, "target": repr(target)}) + "\n") + + +_original_connect = socket.socket.connect +_original_connect_ex = socket.socket.connect_ex +_original_getaddrinfo = socket.getaddrinfo + + +def _is_blocked(sock: socket.socket, address: object) -> bool: + if sock.family not in (socket.AF_INET, socket.AF_INET6): + return False + host = address[0] if isinstance(address, tuple) else address + return _host(host) not in _ALLOWED_HOSTS + + +def _connect(self: socket.socket, address: object) -> None: + if _is_blocked(self, address): + _refuse("connect", address) + raise OSError("blocked by loopback guard: {0!r}".format(address)) + return _original_connect(self, address) + + +def _connect_ex(self: socket.socket, address: object) -> int: + if _is_blocked(self, address): + _refuse("connect_ex", address) + raise OSError("blocked by loopback guard: {0!r}".format(address)) + return _original_connect_ex(self, address) + + +def _getaddrinfo(host: object, *args: object, **kwargs: object): # type: ignore[no-untyped-def] + if host is not None and _host(host) not in _ALLOWED_HOSTS: + _refuse("getaddrinfo", host) + raise socket.gaierror("blocked by loopback guard: {0!r}".format(host)) + return _original_getaddrinfo(host, *args, **kwargs) + + +def main() -> None: + socket.socket.connect = _connect # type: ignore[method-assign] + socket.socket.connect_ex = _connect_ex # type: ignore[method-assign] + socket.getaddrinfo = _getaddrinfo # type: ignore[assignment] + script = sys.argv[1] + sys.argv = sys.argv[1:] + sys.path.insert(0, os.path.dirname(os.path.abspath(script))) + runpy.run_path(script, run_name="__main__") + + +if __name__ == "__main__": + main() From 2a819c4361af443d5b115c318601bb4df75250a6 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Mon, 5 Oct 2026 09:27:16 -0700 Subject: [PATCH 06/16] test(examples): contract-test the OpenLIT recipe with openlit 1.45.0 Runs src/app.py as written against the shared Receiver and a loopback OpenAI fake, under the network guard. Checks the endpoint forms (origin gets /v1/traces appended; a full path is doubled), x-api-key and x-secret-key from percent-encoded otlp_headers, project_name and project_type from OTEL_RESOURCE_ATTRIBUTES, the exact key inventory and values, content off and on, metrics and events paths, the GitHub price download as the only outbound attempt, @openlit.trace ignoring the content switch, a 401 reply, and OpenLIT plus traceai-openai giving two model-call spans for one call. An opt-in test reads fi-collector's alias lists and checks the README display table against them. approved: Nikhil 2026-10-03 blanket Refs: TH-8338 Co-Authored-By: Claude Opus 5.5 --- .../examples/openlit/tests/recipe_variant.py | 40 + .../openlit/tests/test_openlit_recipe.py | 742 ++++++++++++++++++ .../examples/openlit/tests/traced_function.py | 26 + 3 files changed, 808 insertions(+) create mode 100644 python/examples/openlit/tests/recipe_variant.py create mode 100644 python/examples/openlit/tests/test_openlit_recipe.py create mode 100644 python/examples/openlit/tests/traced_function.py diff --git a/python/examples/openlit/tests/recipe_variant.py b/python/examples/openlit/tests/recipe_variant.py new file mode 100644 index 000000000..1b51d7af3 --- /dev/null +++ b/python/examples/openlit/tests/recipe_variant.py @@ -0,0 +1,40 @@ +"""A test-only fixture: ``src/app.py`` as written, with one thing changed. + +``RECIPE_INIT_OVERRIDES`` is a JSON object of ``openlit.init()`` keyword +arguments to replace; a ``null`` value removes the argument, so OpenLIT's +own default applies. With ``RECIPE_ADD_TRACEAI_OPENAI=1``, traceAI's +``OpenAIInstrumentor`` is also enabled right after ``openlit.init()``, on the +tracer provider OpenLIT created, so both instrument the same client. +Everything else, including the model call, is the recipe's own code. +Not part of the recipe. +""" + +from __future__ import annotations + +import json +import os +import runpy +from pathlib import Path + +import openlit + +APP = Path(__file__).resolve().parent.parent / "src" / "app.py" +_OVERRIDES = json.loads(os.environ.get("RECIPE_INIT_OVERRIDES", "{}")) +_real_init = openlit.init + + +def _init(**kwargs): # type: ignore[no-untyped-def] + for key, value in _OVERRIDES.items(): + if value is None: + kwargs.pop(key, None) + else: + kwargs[key] = value + _real_init(**kwargs) + if os.environ.get("RECIPE_ADD_TRACEAI_OPENAI") == "1": + from traceai_openai import OpenAIInstrumentor + + OpenAIInstrumentor().instrument() + + +openlit.init = _init +runpy.run_path(str(APP), run_name="__main__") diff --git a/python/examples/openlit/tests/test_openlit_recipe.py b/python/examples/openlit/tests/test_openlit_recipe.py new file mode 100644 index 000000000..9aa4f6a2c --- /dev/null +++ b/python/examples/openlit/tests/test_openlit_recipe.py @@ -0,0 +1,742 @@ +"""Contract test for the OpenLIT recipe. + +Runs ``src/app.py`` as written with the real ``openlit`` 1.45.0 and a real +``openai`` client, in a subprocess whose network is limited to 127.0.0.1 +(``_guarded_run.py``). The model is a loopback fake of the OpenAI Chat +Completions API; spans go to the shared harness ``Receiver``, which serves +``/v1/traces`` and ``/tracer/v1/traces`` like fi-collector's HTTP mux but +does not authenticate, stamp projects or store anything. + +All keys are placeholders. Nothing here contacts OpenLIT, GitHub, OpenAI or +Future AGI. +""" + +from __future__ import annotations + +import importlib.metadata +import importlib.util +import json +import os +import re +import sys +import threading +from http import HTTPStatus +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from typing import Any, Optional +from urllib.parse import quote, urlsplit + +import pytest + +from harness import Receiver, run + +TESTS_DIR = Path(__file__).resolve().parent +RECIPE_DIR = TESTS_DIR.parent +APP = RECIPE_DIR / "src" / "app.py" +README = RECIPE_DIR / "README.md" +VARIANT = TESTS_DIR / "recipe_variant.py" +TRACED_FUNCTION = TESTS_DIR / "traced_function.py" +GUARD = TESTS_DIR / "_guarded_run.py" +GUARD_PROBE = TESTS_DIR / "_guard_probe.py" + +sys.path.insert(0, str(TESTS_DIR)) +from _fake_openai import RESPONSE_MODEL, USAGE, FakeOpenAI # noqa: E402 + +PROJECT = "openlit-recipe-contract" +SERVICE_NAME = "openlit-recipe" +FI_API_KEY = "fi-api-placeholder-0000" +# A comma and an equals sign: the recipe's percent-encoding must carry them. +FI_SECRET_KEY = "fi-secret-placeholder,part=0000" +OPENAI_KEY = "sk-openai-placeholder-0000" + +# Content markers: the question (input), the fake model's answer (output) +# and the policy text in the system prompt. None may reach an export with +# content off. +QUESTION = "QMARK7c1e what is the refund window?" +ANSWER = "AMARK5d2b refunds are accepted for 30 days." +POLICY = "Refunds are accepted within 30 days of purchase." +CONTENT_MARKERS = ("QMARK7c1e", "AMARK5d2b", POLICY) + +LLM_SPAN = "chat gpt-4o-mini" +HTTP_SPAN = "POST" + +# The key inventory: every attribute key openlit 1.45.0 put on each span of +# one non-streaming chat call, content off. README.md lists the same keys. +LLM_SPAN_KEYS = { + "deployment.environment", + "gen_ai.client.token.usage", + "gen_ai.operation.name", + "gen_ai.output.type", + "gen_ai.provider.name", + "gen_ai.request.frequency_penalty", + "gen_ai.request.model", + "gen_ai.request.presence_penalty", + "gen_ai.request.seed", + "gen_ai.request.stream", + "gen_ai.request.temperature", + "gen_ai.request.top_p", + "gen_ai.request.user", + "gen_ai.response.finish_reasons", + "gen_ai.response.id", + "gen_ai.response.model", + "gen_ai.sdk.version", + "gen_ai.server.time_per_output_token", + "gen_ai.server.time_to_first_token", + "gen_ai.usage.cache_creation.input_tokens", + "gen_ai.usage.cache_read.input_tokens", + "gen_ai.usage.cost", + "gen_ai.usage.input_tokens", + "gen_ai.usage.output_tokens", + "openai.api.type", + "openlit.agent.version_hash", + "server.address", + "server.port", + "service.name", + "telemetry.sdk.name", +} +# Added to the model-call span with content on (OpenLIT's default). +CONTENT_KEYS = { + "gen_ai.input.messages", + "gen_ai.output.messages", + "gen_ai.system_instructions", +} +# OpenLIT also enables OpenTelemetry's httpx instrumentor; the OpenAI +# client's HTTP request becomes this child span. +HTTP_SPAN_KEYS = {"http.method", "http.status_code", "http.url"} +RESOURCE_KEYS = { + "deployment.environment", + "project_name", + "project_type", + "service.instance.id", + "service.name", + "telemetry.sdk.language", + "telemetry.sdk.name", + "telemetry.sdk.version", +} + +# A hang guard, not a speed check: one run imports OpenLIT and its +# instrumentors. +RUN_TIMEOUT_SECONDS = 180 + + +def fi_headers() -> dict[str, str]: + """The otlp_headers dict exactly as src/app.py builds it.""" + return { + "x-api-key": quote(FI_API_KEY, safe=""), + "x-secret-key": quote(FI_SECRET_KEY, safe=""), + } + + +class _PathRecorder: + """Loopback catch-all that records every request's method, path and reply. + + By default it answers /v1/traces and /tracer/v1/traces with 200 and + anything else with 404, as fi-collector's HTTP mux does + (pkg/server/server.go:233-234). ``status`` forces one reply for all. + """ + + def __init__(self, status: Optional[HTTPStatus] = None) -> None: + self.calls: list[tuple[str, str, int]] = [] + lock = threading.Lock() + owner = self + + class Handler(BaseHTTPRequestHandler): + def do_POST(self) -> None: # noqa: N802 + path = urlsplit(self.path).path + self.rfile.read(int(self.headers.get("Content-Length", "0"))) + if status is not None: + reply = status + elif path in ("/v1/traces", "/tracer/v1/traces"): + reply = HTTPStatus.OK + else: + reply = HTTPStatus.NOT_FOUND + with lock: + owner.calls.append(("POST", path, int(reply))) + self.send_response(reply) + self.send_header("Content-Length", "0") + self.end_headers() + + def log_message(self, format: str, *args: Any) -> None: # noqa: A002 + return + + self._server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + self.origin = "http://127.0.0.1:{0}".format(self._server.server_port) + self._thread = threading.Thread(target=self._server.serve_forever, daemon=True) + self._thread.start() + + def __enter__(self) -> "_PathRecorder": + return self + + def __exit__(self, *_exc: Any) -> None: + self._server.shutdown() + self._server.server_close() + self._thread.join() + + def paths(self) -> set[str]: + return {path for _method, path, _status in self.calls} + + +def _child_env( + home: Path, guard_log: Path, endpoint: str, openai_base_url: str, **overrides: Optional[str] +) -> dict[str, str]: + """The recipe's environment, built from scratch (nothing inherited but PATH). + + Bytecode writing stays on so later runs reuse the SDK's compiled modules; + the scripts themselves run through runpy and leave no .pyc in the repo. + """ + env = { + "PATH": os.environ.get("PATH", ""), + "HOME": str(home), + "LOOPBACK_GUARD_LOG": str(guard_log), + "OTEL_EXPORTER_OTLP_ENDPOINT": endpoint, + "OTEL_RESOURCE_ATTRIBUTES": "project_name={0},project_type=observe".format(PROJECT), + "FI_API_KEY": FI_API_KEY, + "FI_SECRET_KEY": FI_SECRET_KEY, + "OPENAI_API_KEY": OPENAI_KEY, + "OPENAI_BASE_URL": openai_base_url, + } + for key, value in overrides.items(): + if value is None: + env.pop(key, None) + else: + env[key] = value + return env + + +def _guard_attempts(guard_log: Path) -> list[dict[str, Any]]: + if not guard_log.exists(): + return [] + return [json.loads(line) for line in guard_log.read_text().splitlines()] + + +def _run_script(tmp_path: Path, script: Path, **overrides: Optional[str]) -> dict[str, Any]: + """Run one script under the loopback guard; return everything the tests read.""" + tmp_path.mkdir(parents=True, exist_ok=True) + guard_log = tmp_path / "guard.jsonl" + with Receiver() as receiver, FakeOpenAI(ANSWER) as fake: + env = _child_env(tmp_path, guard_log, receiver.origin, fake.base_url, **overrides) + result = run( + [sys.executable, str(GUARD), str(script), QUESTION], + env=env, + stdin=None, + timeout=RUN_TIMEOUT_SECONDS, + ) + record = { + "result": result, + "stdout": result.stdout.decode("utf-8", "replace"), + "stderr": result.stderr.decode("utf-8", "replace"), + "requests": receiver.requests(), + "spans": receiver.spans(), + "model_requests": list(fake.requests), + "model_authorizations": list(fake.authorizations), + } + record["guard_attempts"] = _guard_attempts(guard_log) + return record + + +def _run_variant(tmp_path: Path, **init_overrides: Any) -> dict[str, Any]: + """Run src/app.py with some openlit.init() arguments replaced (None removes one).""" + return _run_script(tmp_path, VARIANT, RECIPE_INIT_OVERRIDES=json.dumps(init_overrides)) + + +def _run_against_recorder( + tmp_path: Path, suffix: str = "", status: Optional[HTTPStatus] = None, **overrides: Optional[str] +) -> tuple[Any, list[tuple[str, str, int]], list[dict[str, Any]]]: + """Run a script with a _PathRecorder as the collector; return result, calls, guard log.""" + tmp_path.mkdir(parents=True, exist_ok=True) + guard_log = tmp_path / "guard.jsonl" + script = VARIANT if "RECIPE_INIT_OVERRIDES" in overrides else APP + with _PathRecorder(status) as recorder, FakeOpenAI(ANSWER) as fake: + env = _child_env( + tmp_path, guard_log, recorder.origin + suffix, fake.base_url, **overrides + ) + result = run( + [sys.executable, str(GUARD), str(script), QUESTION], + env=env, + stdin=None, + timeout=RUN_TIMEOUT_SECONDS, + ) + calls = list(recorder.calls) + return result, calls, _guard_attempts(guard_log) + + +def _value(value: dict[str, Any]) -> Any: + for kind in ("stringValue", "boolValue", "doubleValue"): + if kind in value: + return value[kind] + if "intValue" in value: + return int(value["intValue"]) + if "arrayValue" in value: + return [_value(item) for item in value["arrayValue"].get("values", [])] + return value + + +def _attrs(span: dict[str, Any]) -> dict[str, Any]: + return {item["key"]: _value(item.get("value", {})) for item in span.get("attributes", [])} + + +def _by_name(spans: list[dict[str, Any]]) -> dict[str, dict[str, Any]]: + named = {span["name"]: span for span in spans} + assert len(named) == len(spans), "span names repeat: {0}".format( + sorted(span["name"] for span in spans) + ) + return named + + +def _export_dump(record: dict[str, Any]) -> str: + """Everything the export carried except HTTP headers: spans and resources.""" + return json.dumps( + { + "spans": record["spans"], + "resources": [r["resource_attributes"] for r in record["requests"]], + }, + sort_keys=True, + ) + + +def _assert_ran(record: dict[str, Any]) -> None: + result = record["result"] + assert not result.timed_out, record["stderr"] + assert result.returncode == 0, record["stderr"] + assert record["guard_attempts"] == [], record["guard_attempts"] + assert record["requests"], "no export reached the receiver" + assert record["spans"], "the export carried no spans" + + +# -------------------------------------------------------------------------- +# The recipe as documented: content, metrics and events off, local prices. +# -------------------------------------------------------------------------- + + +@pytest.fixture(scope="module") +def recipe_run(tmp_path_factory: pytest.TempPathFactory) -> dict[str, Any]: + record = _run_script(tmp_path_factory.mktemp("recipe"), APP) + _assert_ran(record) + return record + + +def test_recipe_runs_with_loopback_network_only(recipe_run: dict[str, Any]) -> None: + assert recipe_run["guard_attempts"] == [] + assert ANSWER in recipe_run["stdout"] + # The model call reached the fake, with the OpenAI key the client was given. + assert len(recipe_run["model_requests"]) == 1 + assert recipe_run["model_authorizations"] == ["Bearer " + OPENAI_KEY] + + +def test_guard_refuses_and_logs_non_loopback_connections(tmp_path: Path) -> None: + """Positive control for every ``guard_attempts == []`` assertion in this file.""" + guard_log = tmp_path / "guard.jsonl" + result = run( + [sys.executable, str(GUARD), str(GUARD_PROBE)], + env={"PATH": os.environ.get("PATH", ""), "HOME": str(tmp_path), "LOOPBACK_GUARD_LOG": str(guard_log)}, + stdin=None, + timeout=60, + ) + assert not result.timed_out + assert result.returncode == 0, result.stderr.decode("utf-8", "replace") + assert "refused: connect,connect_ex,getaddrinfo" in result.stdout.decode("utf-8", "replace") + logged = _guard_attempts(guard_log) + assert [entry["kind"] for entry in logged] == ["connect", "connect_ex", "getaddrinfo"] + assert "192.0.2.1" in logged[0]["target"] + assert "2001:db8::1" in logged[1]["target"] + assert "example.invalid" in logged[2]["target"] + + +def test_origin_endpoint_posts_to_v1_traces(recipe_run: dict[str, Any]) -> None: + assert recipe_run["requests"], "no export reached the receiver" + assert {request["path"] for request in recipe_run["requests"]} == {"/v1/traces"} + + +def test_otlp_headers_carry_the_future_agi_keys(recipe_run: dict[str, Any]) -> None: + assert recipe_run["requests"], "no export reached the receiver" + for request in recipe_run["requests"]: + headers = request["headers"] + assert headers.get("x-api-key") == FI_API_KEY + assert headers.get("x-secret-key") == FI_SECRET_KEY + assert "authorization" not in headers + assert all(OPENAI_KEY not in value for value in headers.values()) + + +def test_resource_carries_the_project(recipe_run: dict[str, Any]) -> None: + assert recipe_run["requests"], "no export reached the receiver" + for request in recipe_run["requests"]: + assert request["resource_attributes"], "export without a resource" + for resource in request["resource_attributes"]: + assert set(resource) == RESOURCE_KEYS + assert resource["project_name"] == PROJECT + assert resource["project_type"] == "observe" + assert resource["service.name"] == SERVICE_NAME + assert resource["telemetry.sdk.name"] == "openlit" + assert resource["deployment.environment"] == "default" + + +def test_one_trace_model_call_with_an_http_child(recipe_run: dict[str, Any]) -> None: + spans = _by_name(recipe_run["spans"]) + assert set(spans) == {LLM_SPAN, HTTP_SPAN} + assert len({span["traceId"] for span in spans.values()}) == 1 + llm, http = spans[LLM_SPAN], spans[HTTP_SPAN] + assert not llm.get("parentSpanId") + assert http["parentSpanId"] == llm["spanId"] + assert llm["kind"] == http["kind"] == "SPAN_KIND_CLIENT" + assert llm["status"].get("code") == "STATUS_CODE_OK" + + +def test_key_inventory(recipe_run: dict[str, Any]) -> None: + spans = _by_name(recipe_run["spans"]) + llm = _attrs(spans[LLM_SPAN]) + assert set(llm) == LLM_SPAN_KEYS + assert set(_attrs(spans[HTTP_SPAN])) == HTTP_SPAN_KEYS + + # The values the collector's alias lists can read. + assert llm["gen_ai.operation.name"] == "chat" + assert llm["gen_ai.provider.name"] == "openai" + assert llm["gen_ai.request.model"] == "gpt-4o-mini" + assert llm["gen_ai.response.model"] == RESPONSE_MODEL + assert llm["gen_ai.usage.input_tokens"] == USAGE["prompt_tokens"] + assert llm["gen_ai.usage.output_tokens"] == USAGE["completion_tokens"] + # The total is sent under OpenLIT's own key, not gen_ai.usage.total_tokens. + assert llm["gen_ai.client.token.usage"] == USAGE["total_tokens"] + # With an empty price table OpenLIT still sends a cost, of 0. + assert llm["gen_ai.usage.cost"] == 0 + assert llm["gen_ai.usage.cache_read.input_tokens"] == 0 + assert llm["gen_ai.usage.cache_creation.input_tokens"] == 0 + # Request parameters the call did not set are sent with OpenLIT's defaults. + assert llm["gen_ai.request.temperature"] == 1.0 + assert llm["gen_ai.request.top_p"] == 1.0 + assert llm["gen_ai.request.frequency_penalty"] == 0.0 + assert llm["gen_ai.request.presence_penalty"] == 0.0 + assert llm["gen_ai.request.seed"] == 0 + assert llm["gen_ai.request.user"] == "" + assert llm["gen_ai.request.stream"] is False + assert llm["gen_ai.response.id"] == "chatcmpl-fake" + assert llm["gen_ai.response.finish_reasons"] == ["stop"] + assert llm["gen_ai.output.type"] == "text" + assert llm["gen_ai.sdk.version"] == importlib.metadata.version("openai") + assert isinstance(llm["gen_ai.server.time_to_first_token"], float) + assert llm["gen_ai.server.time_per_output_token"] == 0 + assert llm["openai.api.type"] == "chat_completions" + assert re.fullmatch(r"[0-9a-f]{16}", llm["openlit.agent.version_hash"]) + assert llm["server.address"] == "127.0.0.1" + assert isinstance(llm["server.port"], int) + assert llm["service.name"] == SERVICE_NAME + assert llm["deployment.environment"] == "default" + assert llm["telemetry.sdk.name"] == "openlit" + # None of the keys fi-collector reads a span kind from (converter.go:79-84). + for key in ("fi.span.kind", "gen_ai.span.kind", "llm.request.type", "openinference.span.kind"): + assert key not in llm + assert "gen_ai.system" not in llm + + http = _attrs(spans[HTTP_SPAN]) + assert http["http.method"] == "POST" + assert http["http.url"].startswith("http://127.0.0.1:") + assert http["http.url"].endswith("/v1/chat/completions") + assert http["http.status_code"] == 200 + + +def test_content_off_exports_no_content(recipe_run: dict[str, Any]) -> None: + dump = _export_dump(recipe_run) + for marker in CONTENT_MARKERS: + assert marker not in dump, marker + for span in recipe_run["spans"]: + assert not CONTENT_KEYS & set(_attrs(span)), span["name"] + assert not span.get("events"), span["name"] + + +def test_no_secret_in_the_export_or_output(recipe_run: dict[str, Any]) -> None: + dump = _export_dump(recipe_run) + output = recipe_run["stdout"] + recipe_run["stderr"] + secrets = (FI_API_KEY, FI_SECRET_KEY, quote(FI_SECRET_KEY, safe=""), OPENAI_KEY) + for secret in secrets: + assert secret not in dump + assert secret not in output + + +# -------------------------------------------------------------------------- +# Controls and the documented edges. +# -------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + ("suffix", "expected"), + [ + ("", ("/v1/traces", 200)), + ("/", ("/v1/traces", 200)), + # The exporter always appends v1/traces, so a full path doubles it. + ("/v1/traces", ("/v1/traces/v1/traces", 404)), + # fi-collector also serves /tracer/v1/traces. + ("/tracer", ("/tracer/v1/traces", 200)), + ], +) +def test_endpoint_is_the_origin_and_v1_traces_is_appended( + tmp_path: Path, suffix: str, expected: tuple[str, int] +) -> None: + result, calls, guard_attempts = _run_against_recorder(tmp_path, suffix) + assert not result.timed_out + assert result.returncode == 0, result.stderr.decode("utf-8", "replace") + assert guard_attempts == [] + assert calls, "no export reached the recorder" + assert {(path, status) for _method, path, status in calls} == {expected} + + +def test_unencoded_comma_in_a_header_value_truncates_it(tmp_path: Path) -> None: + """Why the recipe percent-encodes: the exporter splits the headers on commas.""" + record = _run_variant( + tmp_path, otlp_headers={"x-api-key": FI_API_KEY, "x-secret-key": FI_SECRET_KEY} + ) + _assert_ran(record) + for request in record["requests"]: + assert request["headers"].get("x-api-key") == FI_API_KEY + assert request["headers"].get("x-secret-key") == FI_SECRET_KEY.split(",")[0] + + +def test_content_is_exported_when_the_argument_is_left_out(tmp_path: Path) -> None: + """OpenLIT's default is content ON. Control for the content-off test.""" + record = _run_variant(tmp_path, capture_message_content=None) + _assert_ran(record) + llm = _attrs(_by_name(record["spans"])[LLM_SPAN]) + assert set(llm) == LLM_SPAN_KEYS | CONTENT_KEYS + assert "QMARK7c1e" in llm["gen_ai.input.messages"] + assert POLICY in llm["gen_ai.input.messages"] # the system prompt + assert POLICY in llm["gen_ai.system_instructions"] + assert "AMARK5d2b" in llm["gen_ai.output.messages"] + dump = _export_dump(record) + for secret in (FI_API_KEY, FI_SECRET_KEY, OPENAI_KEY): + assert secret not in dump + + +def test_env_false_turns_content_off_over_an_explicit_true(tmp_path: Path) -> None: + record = _run_script( + tmp_path, + VARIANT, + RECIPE_INIT_OVERRIDES=json.dumps({"capture_message_content": True}), + OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT="false", + ) + _assert_ran(record) + dump = _export_dump(record) + for marker in CONTENT_MARKERS: + assert marker not in dump, marker + + +@pytest.mark.parametrize( + ("left_out", "extra_path"), + [ + (None, None), + ("disable_metrics", "/v1/metrics"), + ("disable_events", "/v1/logs"), + ], +) +def test_metrics_and_events_go_to_paths_fi_collector_does_not_serve( + tmp_path: Path, left_out: Optional[str], extra_path: Optional[str] +) -> None: + overrides = {left_out: None} if left_out else {} + result, calls, guard_attempts = _run_against_recorder( + tmp_path, RECIPE_INIT_OVERRIDES=json.dumps(overrides) + ) + assert not result.timed_out + assert result.returncode == 0, result.stderr.decode("utf-8", "replace") + assert guard_attempts == [] + paths = {path for _method, path, _status in calls} + if extra_path is None: + assert paths == {"/v1/traces"} + else: + assert paths == {"/v1/traces", extra_path} + assert {status for _method, path, status in calls if path == extra_path} == {404} + + +def test_default_pricing_download_is_the_only_outbound_attempt(tmp_path: Path) -> None: + """Without pricing_json, init() fetches prices from GitHub. The guard refuses it.""" + record = _run_variant(tmp_path, pricing_json=None) + assert not record["result"].timed_out + assert record["result"].returncode == 0, record["stderr"] + assert [entry["kind"] for entry in record["guard_attempts"]] == ["getaddrinfo"] + assert "raw.githubusercontent.com" in record["guard_attempts"][0]["target"] + # The run still exported; the recipe's run above made no attempt at all. + assert record["requests"] and record["spans"] + + +def test_trace_decorator_ignores_the_content_switch(tmp_path: Path) -> None: + """@openlit.trace records arguments and return value with content off.""" + record = _run_script(tmp_path, TRACED_FUNCTION) + _assert_ran(record) + spans = _by_name(record["spans"]) + assert set(spans) == {"support_request", LLM_SPAN, HTTP_SPAN} + decorated = _attrs(spans["support_request"]) + assert "QMARK7c1e" in decorated["function.args"] + assert decorated["gen_ai.output.messages"] == ANSWER + # The model call under it stays content-free. + assert not CONTENT_KEYS & set(_attrs(spans[LLM_SPAN])) + assert spans[LLM_SPAN]["parentSpanId"] == spans["support_request"]["spanId"] + + +def test_rejected_export_is_logged_and_the_app_still_exits_zero(tmp_path: Path) -> None: + """fi-collector answers 401 without valid keys; OpenLIT only logs it.""" + result, calls, guard_attempts = _run_against_recorder( + tmp_path, status=HTTPStatus.UNAUTHORIZED + ) + assert not result.timed_out + assert guard_attempts == [] + assert {(path, status) for _method, path, status in calls} == {("/v1/traces", 401)} + assert result.returncode == 0 + stderr = result.stderr.decode("utf-8", "replace") + assert "Failed to export spans batch code: 401" in stderr, stderr + assert ANSWER in result.stdout.decode("utf-8", "replace") + + +@pytest.mark.parametrize( + ("unset", "message"), + [ + ("OTEL_EXPORTER_OTLP_ENDPOINT", "Set OTEL_EXPORTER_OTLP_ENDPOINT"), + ("OTEL_RESOURCE_ATTRIBUTES", "Set OTEL_RESOURCE_ATTRIBUTES"), + ], +) +def test_app_refuses_to_start_without_endpoint_or_project( + tmp_path: Path, unset: str, message: str +) -> None: + record = _run_script(tmp_path, APP, **{unset: None}) + assert not record["result"].timed_out + assert record["result"].returncode != 0 + assert message in record["stderr"] + assert record["requests"] == [] + assert record["model_requests"] == [] + assert record["guard_attempts"] == [] + + +def _skip_without_traceai_openai() -> None: + if importlib.util.find_spec("traceai_openai") is None: + pytest.skip("traceai-openai is not installed; see README.md, Tests") + + +def test_openlit_and_traceai_openai_both_trace_the_same_call(tmp_path: Path) -> None: + """Both enabled: two model-call spans for one call. The README says enable one.""" + _skip_without_traceai_openai() + record = _run_script(tmp_path, VARIANT, RECIPE_ADD_TRACEAI_OPENAI="1") + _assert_ran(record) + spans = _by_name(record["spans"]) + assert set(spans) == {LLM_SPAN, "ChatCompletion", HTTP_SPAN} + assert len(record["model_requests"]) == 1 + assert len({span["traceId"] for span in spans.values()}) == 1 + # OpenLIT's span wraps traceAI's, which wraps the HTTP request. + assert spans["ChatCompletion"]["parentSpanId"] == spans[LLM_SPAN]["spanId"] + assert spans[HTTP_SPAN]["parentSpanId"] == spans["ChatCompletion"]["spanId"] + openlit_llm, traceai_llm = _attrs(spans[LLM_SPAN]), _attrs(spans["ChatCompletion"]) + assert openlit_llm["gen_ai.operation.name"] == "chat" + assert traceai_llm["gen_ai.span.kind"] == "LLM" + # Both carry the same usage, so a sum over spans counts the call twice. + for llm in (openlit_llm, traceai_llm): + assert llm["gen_ai.usage.input_tokens"] == USAGE["prompt_tokens"] + assert llm["gen_ai.usage.output_tokens"] == USAGE["completion_tokens"] + # OpenLIT's content switch does not reach traceAI's span. + assert POLICY in traceai_llm["input.value"] + assert "QMARK7c1e" in traceai_llm["gen_ai.input.messages.1.message.content"] + assert "AMARK5d2b" in traceai_llm["output.value"] + assert not CONTENT_KEYS & set(openlit_llm) + + +def test_disabling_openlits_openai_instrumentor_leaves_one_model_span(tmp_path: Path) -> None: + """The README's way to enable only traceAI's: disabled_instrumentors=["openai"].""" + _skip_without_traceai_openai() + record = _run_script( + tmp_path, + VARIANT, + RECIPE_INIT_OVERRIDES=json.dumps({"disabled_instrumentors": ["openai"]}), + RECIPE_ADD_TRACEAI_OPENAI="1", + ) + _assert_ran(record) + spans = _by_name(record["spans"]) + assert set(spans) == {"ChatCompletion", HTTP_SPAN} + assert spans[HTTP_SPAN]["parentSpanId"] == spans["ChatCompletion"]["spanId"] + + +# -------------------------------------------------------------------------- +# README checks, and an opt-in check against the collector's alias lists. +# -------------------------------------------------------------------------- + + +def _readme_section(title: str) -> str: + text = README.read_text(encoding="utf-8") + start = text.index("\n## {0}\n".format(title)) + end = text.find("\n## ", start + 1) + return text[start : end if end != -1 else len(text)] + + +def test_readme_key_inventory_matches_the_emitted_keys() -> None: + section = _readme_section("Key inventory") + listed = set(re.findall(r"^\| `([a-z_.]+)` \|", section, flags=re.MULTILINE)) + assert listed == LLM_SPAN_KEYS | CONTENT_KEYS | HTTP_SPAN_KEYS + + +def _collector_aliases() -> dict[str, Any]: + src = os.environ.get("FI_COLLECTOR_SRC") + if not src: + pytest.skip("set FI_COLLECTOR_SRC to a future-agi checkout's fi-collector directory") + adapter = (Path(src) / "pkg" / "adapter" / "adapter.go").read_text(encoding="utf-8") + converter = (Path(src) / "exporter" / "clickhouse25exporter" / "converter.go").read_text( + encoding="utf-8" + ) + + def string_slice(source: str, name: str) -> list[str]: + match = re.search(r"\b{0}\s*=\s*\[\]string\{{(.*?)\}}".format(name), source, re.DOTALL) + assert match, name + body = re.sub(r"//[^\n]*", "", match.group(1)) + return re.findall(r'"([^"]+)"', body) + + synonyms_body = re.search( + r"\bspanKindSynonyms\s*=\s*map\[string\]string\{(.*?)\}", converter, re.DOTALL + ) + assert synonyms_body + names = ( + "modelNameKeys", + "providerKeys", + "inputTokenKeys", + "outputTokenKeys", + "totalTokenKeys", + "costTotalKeys", + "costInputKeys", + "costOutputKeys", + ) + aliases: dict[str, Any] = {name: string_slice(adapter, name) for name in names} + aliases["spanKindAttrKeys"] = string_slice(converter, "spanKindAttrKeys") + aliases["operationNameAttrKeys"] = string_slice(converter, "operationNameAttrKeys") + aliases["spanKindSynonyms"] = dict( + re.findall(r'"([^"]+)"\s*:\s*"([^"]+)"', synonyms_body.group(1)) + ) + return aliases + + +def _first(attrs: dict[str, Any], keys: list[str]) -> Optional[str]: + return next((key for key in keys if attrs.get(key) not in (None, "")), None) + + +def test_readme_display_table_matches_the_collector_aliases(recipe_run: dict[str, Any]) -> None: + """Which emitted key fills each fi-collector column, read from the Go source.""" + aliases = _collector_aliases() + llm = _attrs(_by_name(recipe_run["spans"])[LLM_SPAN]) + kind_key = _first(llm, aliases["spanKindAttrKeys"]) or _first( + llm, aliases["operationNameAttrKeys"] + ) + expected = { + "model": _first(llm, aliases["modelNameKeys"]), + "provider": _first(llm, aliases["providerKeys"]), + "prompt_tokens": _first(llm, aliases["inputTokenKeys"]), + "completion_tokens": _first(llm, aliases["outputTokenKeys"]), + "total_tokens": _first(llm, aliases["totalTokenKeys"]) or "derived", + "cost": _first( + llm, + aliases["costTotalKeys"] + aliases["costInputKeys"] + aliases["costOutputKeys"], + ), + "observation_type": kind_key, + "input": "input.value" if "input.value" in llm else None, + "output": "output.value" if "output.value" in llm else None, + } + # gen_ai.operation.name=chat resolves to the llm observation type. + assert aliases["spanKindSynonyms"].get(llm[kind_key]) == "llm" + + rows = { + column: key or word + for column, key, word in re.findall( + r"^\| `([a-z_]+)` \| (?:`([a-z_.]+)`|(derived|none))", + _readme_section("What Future AGI shows"), + flags=re.MULTILINE, + ) + } + assert rows == {column: key or "none" for column, key in expected.items()} diff --git a/python/examples/openlit/tests/traced_function.py b/python/examples/openlit/tests/traced_function.py new file mode 100644 index 000000000..91c0f8016 --- /dev/null +++ b/python/examples/openlit/tests/traced_function.py @@ -0,0 +1,26 @@ +"""A test-only fixture: the recipe's model call inside OpenLIT's ``@openlit.trace``. + +It runs ``init_tracing()`` from ``src/app.py`` as written (content off), so +the contract test can show which content the ``@openlit.trace`` decorator +records anyway. Not part of the recipe. +""" + +from __future__ import annotations + +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "src")) + +import openlit # noqa: E402 +from app import ask, init_tracing # noqa: E402 + +init_tracing() + + +@openlit.trace +def support_request(question: str) -> str: + return ask(question) + + +print(support_request(sys.argv[1])) From e29826abdc50df11002267b25cadaed7bad18db0 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Mon, 5 Oct 2026 09:27:16 -0700 Subject: [PATCH 07/16] docs(examples): add the OpenLIT recipe README Quotes the 1.45.0 init() signature, records the tested endpoint, header and resource behaviour, the key inventory and which keys fi-collector promotes to columns, the SF-1 gaps, what the content switch does not cover, and "enable one" for traceai-openai and Firecrawl. approved: Nikhil 2026-10-03 blanket Refs: TH-8338 Co-Authored-By: Claude Opus 5.5 --- python/examples/openlit/README.md | 469 ++++++++++++++++++++++++++++++ 1 file changed, 469 insertions(+) create mode 100644 python/examples/openlit/README.md diff --git a/python/examples/openlit/README.md b/python/examples/openlit/README.md new file mode 100644 index 000000000..665ca2fb8 --- /dev/null +++ b/python/examples/openlit/README.md @@ -0,0 +1,469 @@ +# OpenLIT → Future AGI (recipe) + +There is no traceAI package for OpenLIT and no OpenLIT adapter in Future +AGI. OpenLIT already exports OTLP/HTTP. This example points `openlit.init()` +at fi-collector, passes the Future AGI keys as OTLP headers, puts the +project on the resource with `OTEL_RESOURCE_ATTRIBUTES`, and turns content +capture, metrics, events and OpenLIT's price download off. It adds no +mapping of its own: the key inventory below lists what OpenLIT sent and +which of those keys fi-collector reads. + +Pinned: `openlit` 1.45.0. Its wheel METADATA declares `License: Apache-2.0` +and the OSI Apache classifier, ships the Apache 2.0 text, and declares +`Requires-Python: >=3.9.0,<4.0.0`. It requires `openai>=1.92.0,<3.0.0`, so +`requirements.txt` pins `openai` 2.54.0, not the 3.x line. Both are +dependencies of this example only, not of any traceAI package. + +The contract test (see "Tests") passes on Python 3.11.12 and 3.13.15 with +`openai` 2.54.0, `opentelemetry-sdk` and +`opentelemetry-exporter-otlp-proto-http` 1.45.0 and +`opentelemetry-instrumentation-httpx` 0.66b0, the versions uv resolved on +2026-10-05. Python 3.9, 3.10 and 3.12 are inside openlit's range but were +not tested here. + +## Run + +```bash +cd python/examples/openlit +pip install -r requirements.txt + +export FI_API_KEY="YOUR_API_KEY" +export FI_SECRET_KEY="YOUR_SECRET_KEY" +export OTEL_EXPORTER_OTLP_ENDPOINT="https://YOUR_FI_COLLECTOR_ORIGIN" # no path +export OTEL_RESOURCE_ATTRIBUTES="project_name=my-chatbot,project_type=observe" +export OPENAI_API_KEY="YOUR_OPENAI_KEY" # optional: OPENAI_BASE_URL, OPENAI_MODEL + +python src/app.py "What is the refund window?" +``` + +## The recipe + +`init_tracing()` in `src/app.py` is the whole integration. It first checks +that the endpoint and the project are set, then calls: + +```python +openlit.init( + service_name="openlit-recipe", # service.name + otlp_endpoint=os.environ["OTEL_EXPORTER_OTLP_ENDPOINT"], + otlp_headers={ + "x-api-key": quote(os.environ["FI_API_KEY"], safe=""), + "x-secret-key": quote(os.environ["FI_SECRET_KEY"], safe=""), + }, + capture_message_content=False, + disable_metrics=True, + disable_events=True, + pricing_json=str(NO_PRICING), # src/no_pricing.json, an empty table +) +``` + +The 1.45.0 signature, quoted from the wheel's `openlit/__init__.py:185-209`: + +```python +def init( + environment="default", + application_name="default", + service_name="default", + otlp_endpoint=None, + otlp_headers=None, + disable_batch=False, + capture_message_content=True, + disabled_instrumentors=None, + disable_metrics=False, + disable_events=False, + pricing_json=None, + controller_mode=None, + collect_gpu_stats=False, + collect_system_metrics=False, + capture_db_parameters=False, + max_content_length=None, + custom_span_attributes=None, + custom_metrics_attributes=None, + openlit_api_key=None, + openlit_url=None, + *, + guards=None, + guard_fail_open=True, +): +``` + +It has no resource argument and no telemetry argument. `init()` logs and +swallows any error raised inside it (`__init__.py:487-488`), so a mistake +there shows up as a log line, not an exception. + +### Endpoint + +Set `otlp_endpoint` to the collector origin, with no path. OpenLIT copies +it into `OTEL_EXPORTER_OTLP_ENDPOINT` (`openlit/otel/tracing.py:72-73`) and +creates OpenTelemetry's OTLP/HTTP span exporter with no arguments +(`tracing.py:121-128`). That exporter (`opentelemetry-exporter-otlp-proto-http` +1.45.0, `_common/__init__.py:129-138`) removes one trailing `/` and always +appends `/v1/traces`. Tested results: + +| `otlp_endpoint` | Request path | Reply from a fi-collector-like mux | +|---|---|---| +| `http://host:port` | `/v1/traces` | 200 | +| `http://host:port/` | `/v1/traces` | 200 | +| `http://host:port/v1/traces` | `/v1/traces/v1/traces` | 404 | +| `http://host:port/tracer` | `/tracer/v1/traces` | 200 | + +fi-collector serves `/v1/traces` and `/tracer/v1/traces` +(`fi-collector/pkg/server/server.go:233-234` at future-agi `main` 4af5338). +These paths were tested against a loopback recorder that answers the way +that mux does; no deployed endpoint was called. + +Read from source, not tested: with `otlp_endpoint` left out, OpenLIT uses +`OTEL_EXPORTER_OTLP_ENDPOINT` from the environment, and with neither it +prints every span to stdout (`tracing.py:121-131`). `src/app.py` refuses to +start without the variable instead (tested). `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT`, +if set, is used as the full URL with nothing appended +(`_common/__init__.py:133-134`). `OTEL_EXPORTER_OTLP_PROTOCOL=grpc` switches +OpenLIT to the gRPC exporter (`tracing.py:23-26`); this recipe does not test +gRPC. + +### Auth headers + +`otlp_headers` is a dict (`__init__.py:222`). OpenLIT joins it into +`name=value,name=value` and writes that to `OTEL_EXPORTER_OTLP_HEADERS` +(`tracing.py:75-83`). The exporter parses the variable with OpenTelemetry's +`parse_env_headers(..., liberal=True)` (`_common/__init__.py:141-153`), +which splits on commas and percent-decodes each value +(`opentelemetry/util/re.py:52-93`). So the recipe percent-encodes both +values. The contract test proves that they arrive as the `x-api-key` and +`x-secret-key` request headers fi-collector reads +(`fi-collector/pkg/auth/middleware.go:45-46`), using a placeholder secret +that contains `,` and `=`. Without the encoding, a comma cuts the value +short (tested). + +Because OpenLIT writes the joined string to `os.environ`, the keys stay in +the process environment after `init()` and any child process the app starts +inherits them (read from source). + +### Project + +fi-collector rejects a batch whose resource has no `project_name` (HTTP 400, +`fi-collector/pkg/auth/stamp.go:31-45`) and creates the project on first +use. `openlit.init()` has no resource argument. It builds its resource with +`Resource.create()` (`tracing.py:62-68`), which merges the standard +`OTEL_RESOURCE_ATTRIBUTES` variable and percent-decodes its values +(`opentelemetry-sdk` 1.45.0, `resources/__init__.py:333-348`). The test sets +`project_name=...,project_type=observe` there and finds exactly these +resource attributes on every export: `project_name`, `project_type`, +`service.name` (`openlit-recipe`), `deployment.environment` (`default`), +`telemetry.sdk.name` (`openlit`), `telemetry.sdk.language`, +`telemetry.sdk.version` and `service.instance.id`. `src/app.py` refuses to +start when the variable has no `project_name` (tested). + +### Metrics and events + +By default OpenLIT also exports metrics and, once an endpoint is set, OTel +log records it calls events (`__init__.py:378-400`, +`otel/events.py:124-136`). Their exporters post to `/v1/metrics` and +`/v1/logs`, which fi-collector does not route, and get 404 (tested against +the recorder). The recipe passes `disable_metrics=True` and +`disable_events=True`; with both, the only request is to `/v1/traces` +(tested). With events off, `init()` still installs a global OTel logger +provider with an OTLP log exporter (`__init__.py:378-385`); nothing is +written to it, and no `/v1/logs` request was seen. + +### Price download and telemetry + +`init()` always loads a price table (`__init__.py:426`). Without +`pricing_json` it downloads +`https://raw.githubusercontent.com/openlit/openlit/main/assets/pricing.json`, +with a 20-second timeout, on every start (`__helpers.py:476-482`). The recipe +passes `src/no_pricing.json`, an empty table, so nothing is downloaded and +OpenLIT sends `gen_ai.usage.cost` as 0 (tested). With the argument left out, +the loopback guard refused exactly one attempt, a DNS lookup of +`raw.githubusercontent.com`, and nothing else (tested). + +`init()` has no telemetry switch. The 1.45.0 wheel has no telemetry client, +and the price download is the only URL the `init()` path calls; +`cloud.openlit.io` appears only in the CLI's help text (read from source). +No recipe run in the contract test made any non-loopback connection +attempt. + +## What is traced + +With content off, `python src/app.py` exports two spans in one trace: + +| Span | Parent | Kind | Instrumentor | +|---|---|---|---| +| `chat gpt-4o-mini` | none | CLIENT | OpenLIT's OpenAI instrumentor | +| `POST` | `chat gpt-4o-mini` | CLIENT | OpenTelemetry's httpx instrumentor, which `openlit.init()` also enables | + +The model-call span is named `chat `. `init()` also enables +OpenTelemetry's `requests`, `urllib`, `urllib3` and `aiohttp` client +instrumentors when those libraries are installed (`_instrumentors.py:78-82`, +`__init__.py:141-145`), so other HTTP calls in your app become spans too +(read from source). `disabled_instrumentors` turns any of them off (not +tested). + +## Key inventory + +Every attribute key OpenLIT 1.45.0 put on the spans of one non-streaming +chat call against the fake, with the value the test checked. "Stored" means +fi-collector keeps the key in the span's raw attribute maps +(`attrs_string`, `attrs_number`, `attrs_bool`, or the `attributes_extra` +JSON for arrays; `fi-collector/pkg/adapter/adapter.go:62-107`) without +putting it in a column. Whether the Future AGI UI shows raw attributes was +not checked. + +Model-call span (`chat gpt-4o-mini`), content off: + +| Key | Value in the test | fi-collector | +|---|---|---| +| `gen_ai.operation.name` | `chat` | `observation_type` (see "What Future AGI shows") | +| `gen_ai.provider.name` | `openai` | `provider` column | +| `gen_ai.request.model` | `gpt-4o-mini` | `model` column | +| `gen_ai.response.model` | `gpt-4o-mini-2024-07-18` | stored; `model` reads the request model first | +| `gen_ai.usage.input_tokens` | `11` | `prompt_tokens` column | +| `gen_ai.usage.output_tokens` | `7` | `completion_tokens` column | +| `gen_ai.client.token.usage` | `18` | stored; not a total-tokens alias | +| `gen_ai.usage.cost` | `0` (empty price table) | stored; not a cost alias | +| `gen_ai.usage.cache_read.input_tokens` | `0` | stored | +| `gen_ai.usage.cache_creation.input_tokens` | `0` | stored | +| `gen_ai.request.temperature` | `1.0` (OpenLIT's default; the call set none) | stored | +| `gen_ai.request.top_p` | `1.0` (same) | stored | +| `gen_ai.request.frequency_penalty` | `0.0` (same) | stored | +| `gen_ai.request.presence_penalty` | `0.0` (same) | stored | +| `gen_ai.request.seed` | `0` (same) | stored | +| `gen_ai.request.user` | empty string | stored | +| `gen_ai.request.stream` | `false` | stored | +| `gen_ai.response.id` | `chatcmpl-fake` | stored | +| `gen_ai.response.finish_reasons` | `["stop"]` | stored (array) | +| `gen_ai.output.type` | `text` | stored | +| `gen_ai.sdk.version` | `2.54.0`, the `openai` package version | stored | +| `gen_ai.server.time_to_first_token` | seconds, a float | stored | +| `gen_ai.server.time_per_output_token` | `0` | stored | +| `openai.api.type` | `chat_completions` | stored | +| `openlit.agent.version_hash` | 16 hex digits | stored | +| `server.address` | `127.0.0.1` | stored | +| `server.port` | the fake's port | stored | +| `service.name` | `openlit-recipe` | stored (also on the resource) | +| `deployment.environment` | `default` | stored (also on the resource) | +| `telemetry.sdk.name` | `openlit` | stored (also on the resource) | + +Added to the model-call span only with content on, OpenLIT's default: + +| Key | Value in the test | fi-collector | +|---|---|---| +| `gen_ai.input.messages` | JSON string: the system and user messages | stored; not read as the span's input | +| `gen_ai.output.messages` | JSON string: the assistant message | stored; not read as the span's output | +| `gen_ai.system_instructions` | JSON string: the system prompt | stored | + +HTTP span (`POST`): + +| Key | Value in the test | fi-collector | +|---|---|---| +| `http.method` | `POST` | stored | +| `http.url` | the full OpenAI request URL | stored | +| `http.status_code` | `200` | stored | + +Not sent (tested): `gen_ai.usage.total_tokens`, `gen_ai.system`, and every +key fi-collector reads a span kind from: `fi.span.kind`, `gen_ai.span.kind`, +`llm.request.type`, `openinference.span.kind` +(`exporter/clickhouse25exporter/converter.go:79-84`). No span events were +sent. Tool calls, streaming, embeddings and errors were not exercised, so +their keys are not in this inventory. + +## What Future AGI shows + +fi-collector promotes a few keys to columns. This table is everything this +recipe calls "displayed". It was read from fi-collector at future-agi `main` +4af5338 (`pkg/adapter/adapter.go`, +`exporter/clickhouse25exporter/converter.go`); the opt-in test (see +"Tests") re-reads the alias lists from that source and checks this table +against them and against the keys the run emitted. No fi-collector was run. + +| Column | Filled from | Source | +|---|---|---| +| `model` | `gen_ai.request.model`, which wins over `gen_ai.response.model` | `adapter.go:279-284` | +| `provider` | `gen_ai.provider.name` | `adapter.go:289-295` | +| `prompt_tokens` | `gen_ai.usage.input_tokens` | `adapter.go:296-300` | +| `completion_tokens` | `gen_ai.usage.output_tokens` | `adapter.go:301-305` | +| `total_tokens` | derived as input plus output, because no total alias arrived | `adapter.go:230-235`, `:306-310` | +| `cost` | none: `gen_ai.usage.cost` is not one of `gen_ai.cost.*`, `llm.cost.*` | `adapter.go:311-313` | +| `observation_type` | `gen_ai.operation.name`: `chat` maps to `llm` | `converter.go:79-131` | +| `input` | none: only `input.value` is read | `converter.go:343` | +| `output` | none: only `output.value` is read | `converter.go:344` | + +Because OpenLIT's cost key is not read, fi-collector treats the span as +having no user cost and prices it from `model` and the token columns with +its own price table, when one is configured (`converter.go:285-294`). That +pricing was not exercised here. This recipe computes no price, and the 0 +OpenLIT sends with the empty table is not used. + +The `POST` span carries none of these keys: its `observation_type` resolves +to `unknown` and its model and token columns stay empty (read from source). + +Gaps for the shared processor work (SF-1), from this inventory: + +- `gen_ai.client.token.usage` is not a total-tokens alias. The column is + derived instead, and for this call the derived value equals it. +- `gen_ai.usage.cost` is not a cost alias. +- `gen_ai.input.messages`, `gen_ai.output.messages` and + `gen_ai.system_instructions` are not read as the span's input or output. +- A `@openlit.trace` span (see "Privacy") has no span-kind key, so it is + `unknown`. + +## Privacy + +Content capture is on unless you turn it off: `capture_message_content` +defaults to `True` (`__init__.py:192`). With it on, the model-call span +carries `gen_ai.input.messages` (system prompt and question), +`gen_ai.output.messages` (the answer) and `gen_ai.system_instructions` +(tested). The recipe passes `False`, and the test finds none of those keys, +no span events, and none of the question, answer or system-prompt text +anywhere in the export. + +`OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT=false` turns content off +even when `init()` is called with `True` (tested; `__init__.py:410-414`). +The variable cannot turn content on when the argument is `False`: OpenLIT +applies it only while the argument is still `True` (`__init__.py:292-293`, +read from source). + +Not covered by the content switch: + +- `@openlit.trace` records the decorated function's arguments + (`function.args`, `function.kwargs`) and its return value + (`gen_ai.output.messages`) whatever the switch says + (`__init__.py:953-965`). With content off, the test finds the question in + `function.args` and the answer in `gen_ai.output.messages`. `src/app.py` + does not use the decorator; `tests/traced_function.py` is the fixture. +- The `POST` span's `http.url` is the full request URL (tested). +- OpenLIT's Firecrawl instrumentor ignores the switch (see "Firecrawl"). +- With traceai-openai also enabled, its span carries the prompt and the + answer whatever OpenLIT's switch says (tested; see "traceai-openai"). +- When an instrumented call raises, OpenLIT records the exception, message + included, on the span (`__helpers.py:492-502`; not tested). + +No Future AGI key, secret or OpenAI key appears in any exported span or +resource, or in the app's output (tested). + +## Using OpenLIT with traceAI + +### traceai-openai + +OpenLIT wraps `Completions.create` +(`instrumentation/openai/__init__.py:185-189`) and traceai-openai wraps +`OpenAI.request` (`python/frameworks/openai/traceai_openai/__init__.py:55-59`), +so with both enabled one call is traced twice. The test runs the recipe and +then enables traceai-openai 0.1.10's `OpenAIInstrumentor` on the tracer +provider OpenLIT created. One call to the fake gave three spans in one +trace: + +| Span | Parent | From | +|---|---|---| +| `chat gpt-4o-mini` | none | OpenLIT | +| `ChatCompletion` | `chat gpt-4o-mini` | traceai-openai (`gen_ai.span.kind` = `LLM`) | +| `POST` | `ChatCompletion` | OpenTelemetry httpx, enabled by OpenLIT | + +Both model-call spans carry the same token counts (tested), so fi-collector +stores two LLM spans with tokens for one call (read from source) and a sum +over spans counts the call twice. OpenLIT's content switch does not reach +traceai-openai's span: with the recipe's content off, that span still +carried the system prompt in `input.value`, the question in +`gen_ai.input.messages.1.message.content` and the answer in `output.value` +(tested). + +Enable one of the two. Neither package disables the other, and this recipe +does not do it for you. To keep only traceai-openai's span, pass +`disabled_instrumentors=["openai"]` to `openlit.init()`; the test then sees +only `ChatCompletion` and `POST`. To keep only OpenLIT's, do not enable +traceai-openai. + +If traceAI's `register()`, or anything else, has already set a global tracer +provider, `openlit.init()` reuses it and adds no exporter, so +`otlp_endpoint` and `otlp_headers` do nothing for spans +(`tracing.py:56-59`; read from source, not tested). + +### Firecrawl + +OpenLIT 1.45.0's `FireCrawlInstrumentor` wraps `FirecrawlApp` in +`firecrawl.firecrawl` (`scrape_url`, `crawl_url`, `map_url`, `search`, +`extract`, `batch_scrape_urls`, `check_crawl_status`) and the matching +`AsyncFirecrawlApp` methods (`instrumentation/firecrawl/__init__.py:40-273`): +the Firecrawl v1 client. The architecture note (D-8338) confirms the same at +OpenLIT commit `897838a`. Per that note, the traceAI Firecrawl package +(TH-8325) wraps the v2 `Firecrawl` client instead. If you use both, pick +one. This recipe does not disable either and does not change the Firecrawl +package. `disabled_instrumentors=["firecrawl"]` turns OpenLIT's off +(`_instrumentors.py:50`, `__init__.py:131-133`; not tested). + +OpenLIT's content switch does not reach its Firecrawl instrumentor. `init()` +passes `capture_message_content` (`__init__.py:148-156`), but the +instrumentor reads `trace_content`, which defaults to `True` +(`instrumentation/firecrawl/__init__.py:33`), and uses it to decide whether +to record scraped content (`instrumentation/firecrawl/utils.py:264`, `:431`). +Read from source; no Firecrawl call was made. + +## Notes + +- OpenLIT uses a `BatchSpanProcessor`, and the SDK tracer provider flushes + it at exit, so a short script exports on exit (tested) and a server + exports in batches. +- When fi-collector rejects an export (401 without valid keys, + `middleware.go:20`), the exporter logs + `Failed to export spans batch code: 401, reason: Unauthorized` to stderr + and the app carries on and exits 0; nothing raises in your code (tested + against a recorder that answers 401). +- No fi-collector, OpenLIT host, GitHub or OpenAI was called. Auth, project + stamping and storage are fi-collector behaviour the harness receiver does + not reproduce. + +## Tests + +`tests/test_openlit_recipe.py` runs `src/app.py` as written, in a +subprocess, with the real `openlit` 1.45.0 instrumenting the real `openai` +client. `tests/_fake_openai.py` is a loopback fake of the Chat Completions +API; the spans go to the shared harness `Receiver` (`python/tests/harness`), +which serves `/v1/traces` and `/tracer/v1/traces` on 127.0.0.1. +`tests/_guarded_run.py` blocks and logs every non-loopback connection; a +positive control (`tests/_guard_probe.py`) proves it refuses and logs IPv4, +IPv6 and DNS attempts. All keys are placeholders. + +`tests/recipe_variant.py` runs `src/app.py` with one `openlit.init()` +argument changed or removed, or with traceai-openai also enabled. +`tests/traced_function.py` wraps the recipe's model call in +`@openlit.trace`. Both are fixtures, not part of the recipe. + +The tests check the request path for each endpoint form, the auth headers, +the resource, the span tree, the exact key inventory and its values, +content off and on, the environment override, the metrics and events paths, +the price download, the decorator, a 401 reply, the app's start-up checks, +both instrumentors together, and that no key reaches the export. + +From the repository root, Python 3.11: + +```bash +env -u PYTHONPATH PYTHONPATH="python/examples/openlit:python:python/tests" \ + uv run --no-project --python 3.11 \ + --with pytest --with jsonschema --with protobuf --with opentelemetry-proto \ + --with 'openlit==1.45.0' --with 'openai==2.54.0' \ + pytest python/examples/openlit/tests -q -p no:cacheprovider \ + --noconftest -o addopts= -rfEs +``` + +For Python 3.13, replace `--python 3.11`. `python` is on the path and +`jsonschema` is installed only because pytest imports `python/__init__.py`, +which imports `fi_instrumentation`; the recipe needs neither. + +The two traceai-openai tests skip in that environment. To run them, add +`--with 'traceai-openai==0.1.10'`. That package requires `wrapt<2` (through +`fi-instrumentation-otel` 1.1.0), so uv resolves `wrapt` 1.17.3 there and +2.5.0 without it. The whole suite passes in both environments. + +One test is opt-in. It reads the alias lists from fi-collector's +`adapter.go` and `converter.go` and checks the "What Future AGI shows" table +against them and against the emitted keys. Point `FI_COLLECTOR_SRC` at the +`fi-collector` directory of a future-agi checkout (tested with `main` +4af5338): + +```bash +export FI_COLLECTOR_SRC=/fi-collector # then run the command above +``` + +Without it the test is skipped. No CI job runs this example, so run it +whenever fi-collector's alias lists or this recipe change; otherwise the +table can drift unnoticed. A full run starts the app in a new process about +20 times and takes under a minute. From 87cfaee99f3acf6f312e527ffb1b0dc74ae281d7 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Mon, 5 Oct 2026 09:40:45 -0700 Subject: [PATCH 08/16] docs(examples): OpenLIT recipe runs on Python 3.10-3.13 Coordinator exact-head run at e29826a passed on 3.10, 3.11, 3.12 and 3.13 (25 passed + 3 skipped README command; 26 passed + 2 skipped with FI_COLLECTOR_SRC; 28 passed with traceai-openai 0.1.10 on 3.11 and 3.13). approved: Nikhil 2026-10-03 blanket Refs: TH-8338 --- python/examples/openlit/README.md | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/python/examples/openlit/README.md b/python/examples/openlit/README.md index 665ca2fb8..5cc0e8dec 100644 --- a/python/examples/openlit/README.md +++ b/python/examples/openlit/README.md @@ -14,12 +14,11 @@ and the OSI Apache classifier, ships the Apache 2.0 text, and declares `requirements.txt` pins `openai` 2.54.0, not the 3.x line. Both are dependencies of this example only, not of any traceAI package. -The contract test (see "Tests") passes on Python 3.11.12 and 3.13.15 with -`openai` 2.54.0, `opentelemetry-sdk` and +The contract test (see "Tests") passes on Python 3.10, 3.11, 3.12 and 3.13 +with `openai` 2.54.0, `opentelemetry-sdk` and `opentelemetry-exporter-otlp-proto-http` 1.45.0 and `opentelemetry-instrumentation-httpx` 0.66b0, the versions uv resolved on -2026-10-05. Python 3.9, 3.10 and 3.12 are inside openlit's range but were -not tested here. +2026-10-05. Python 3.9 is inside openlit's range but was not tested here. ## Run From eefe838ba6d65d2c44feaaf80c7d81daef62858f Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Mon, 5 Oct 2026 10:28:03 -0700 Subject: [PATCH 09/16] docs(examples): list what OpenLIT still exports with content off Review R1 (P2) and V1. With capture_message_content=False, OpenLIT 1.45.0 still exports model-generated tool calls (gen_ai.tool.name, gen_ai.tool.call.id, gen_ai.tool.args), the caller's `user` (gen_ai.request.user), and, for a failed call, two exception events plus the status description carrying the SDK's error body. New fixtures run the recipe's own init_tracing(): tool_call.py (tools and user; the fake answers with a tool call whose arguments carry a planted email) and failed_call.py (a fake 500 with max_retries=0). The tests assert that these markers DO arrive with content off and that no prompt or answer text does. They document exceptions; they are not privacy claims. A README test fails if Privacy stops naming any key those runs exported. The README summary no longer says "content off" without qualification, and Privacy recommends disabled_instrumentors=["openai"] or accepting the export; the recipe filters nothing (D-8338). approved: Nikhil 2026-10-03 blanket Refs: TH-8338 Co-Authored-By: Claude Opus 5.5 --- python/examples/openlit/README.md | 58 +++++- python/examples/openlit/tests/_fake_openai.py | 58 ++++-- python/examples/openlit/tests/failed_call.py | 27 +++ .../openlit/tests/test_openlit_recipe.py | 174 ++++++++++++++++-- python/examples/openlit/tests/tool_call.py | 45 +++++ 5 files changed, 327 insertions(+), 35 deletions(-) create mode 100644 python/examples/openlit/tests/failed_call.py create mode 100644 python/examples/openlit/tests/tool_call.py diff --git a/python/examples/openlit/README.md b/python/examples/openlit/README.md index 5cc0e8dec..1a74abe58 100644 --- a/python/examples/openlit/README.md +++ b/python/examples/openlit/README.md @@ -3,10 +3,13 @@ There is no traceAI package for OpenLIT and no OpenLIT adapter in Future AGI. OpenLIT already exports OTLP/HTTP. This example points `openlit.init()` at fi-collector, passes the Future AGI keys as OTLP headers, puts the -project on the resource with `OTEL_RESOURCE_ATTRIBUTES`, and turns content -capture, metrics, events and OpenLIT's price download off. It adds no -mapping of its own: the key inventory below lists what OpenLIT sent and -which of those keys fi-collector reads. +project on the resource with `OTEL_RESOURCE_ATTRIBUTES`, and turns off +metrics, events, OpenLIT's price download and its message capture. With +message capture off, the prompt and the answer are not exported, but +model-generated tool-call arguments, the `user` argument and error bodies +still are (see "Privacy"). The recipe adds no mapping or filtering of its +own: the key inventory below lists what OpenLIT sent and which of those keys +fi-collector reads. Pinned: `openlit` 1.45.0. Its wheel METADATA declares `License: Apache-2.0` and the OSI Apache classifier, ships the Apache 2.0 text, and declares @@ -262,8 +265,10 @@ Not sent (tested): `gen_ai.usage.total_tokens`, `gen_ai.system`, and every key fi-collector reads a span kind from: `fi.span.kind`, `gen_ai.span.kind`, `llm.request.type`, `openinference.span.kind` (`exporter/clickhouse25exporter/converter.go:79-84`). No span events were -sent. Tool calls, streaming, embeddings and errors were not exercised, so -their keys are not in this inventory. +sent. A tool call adds `gen_ai.tool.name`, `gen_ai.tool.call.id` and +`gen_ai.tool.args`, and a failed call adds `error.type`, two `exception` +events and an error status; "Privacy" covers them because they are sent +with content off (tested). Streaming and embeddings were not exercised. ## What Future AGI shows @@ -313,7 +318,8 @@ carries `gen_ai.input.messages` (system prompt and question), `gen_ai.output.messages` (the answer) and `gen_ai.system_instructions` (tested). The recipe passes `False`, and the test finds none of those keys, no span events, and none of the question, answer or system-prompt text -anywhere in the export. +anywhere in the export. The switch covers those messages only. The list +below is what it does not cover; the recipe filters nothing. `OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT=false` turns content off even when `init()` is called with `True` (tested; `__init__.py:410-414`). @@ -323,6 +329,33 @@ read from source). Not covered by the content switch: +- Tool calls. When the request passes `tools` and the model answers with + tool calls, the model-call span carries `gen_ai.tool.name`, + `gen_ai.tool.call.id` and `gen_ai.tool.args`, the arguments the model + generated, verbatim (`instrumentation/openai/utils.py:1524-1551`, before + the content check at `:1616`). Those arguments often repeat what the user + typed: an email address, an order number. The test plants an email address + in the fake model's tool-call arguments and finds it in `gen_ai.tool.args` + (`tests/tool_call.py`). The Responses API sets the same keys + (`utils.py:991-1022`; not tested). +- `user`. The `user` argument of `chat.completions.create()` is sent as + `gen_ai.request.user` (`utils.py:1484-1487`). The test passes an email + address and finds it there. Without the argument the key is an empty + string. +- Errors. When the OpenAI call raises, OpenLIT records the exception on the + model-call span (`instrumentation/openai/openai.py:186-209`, + `utils.py:77-84`) and re-raises inside `start_as_current_span`, whose + context manager records it again and sets the span status + (`opentelemetry-api` 1.45.0 `trace/__init__.py:609-622`, called from + `opentelemetry-sdk` `trace/__init__.py:1130-1136`). So a failed call + carries two `exception` events, each with `exception.message` and + `exception.stacktrace`, and the same text as the status description, + `status.message`. The `openai` SDK puts the whole error response body in + that text (`openai/_base_client.py:430-435`). Tested with a fake 500 and + `max_retries=0` (`tests/failed_call.py`): the fake's error body appeared + in all three places, and no prompt or answer text did. An + OpenAI-compatible server or proxy behind `OPENAI_BASE_URL` whose error + bodies echo the request would put prompt text there (not tested). - `@openlit.trace` records the decorated function's arguments (`function.args`, `function.kwargs`) and its return value (`gen_ai.output.messages`) whatever the switch says @@ -333,8 +366,15 @@ Not covered by the content switch: - OpenLIT's Firecrawl instrumentor ignores the switch (see "Firecrawl"). - With traceai-openai also enabled, its span carries the prompt and the answer whatever OpenLIT's switch says (tested; see "traceai-openai"). -- When an instrumented call raises, OpenLIT records the exception, message - included, on the span (`__helpers.py:492-502`; not tested). + +If your tool arguments or `user` values can carry personal data, choose one: + +- Turn OpenLIT's OpenAI instrumentor off with + `disabled_instrumentors=["openai"]`. OpenLIT then makes no model-call span, + so none of the tool, `user` or error keys above is recorded by it (tested + with traceai-openai enabled; see "traceai-openai"). The httpx `POST` span + and its `http.url` stay. +- Or accept that these keys reach Future AGI. No Future AGI key, secret or OpenAI key appears in any exported span or resource, or in the app's output (tested). diff --git a/python/examples/openlit/tests/_fake_openai.py b/python/examples/openlit/tests/_fake_openai.py index e3426e180..2b9f96058 100644 --- a/python/examples/openlit/tests/_fake_openai.py +++ b/python/examples/openlit/tests/_fake_openai.py @@ -2,8 +2,10 @@ Every POST to /v1/chat/completions gets one assistant message whose text is ``answer_text``, with fixed token usage, so the instrumentor records model -and usage attributes. Each request body and Authorization header is kept for -assertions. Listens on 127.0.0.1 only. +and usage attributes. With ``tool_call`` the message is that one tool call +instead of text; with ``error`` every request gets that status and JSON body. +Each request body and Authorization header is kept for assertions. Listens +on 127.0.0.1 only. """ from __future__ import annotations @@ -12,21 +14,50 @@ import threading from http import HTTPStatus from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer -from typing import Any +from typing import Any, Optional RESPONSE_MODEL = "gpt-4o-mini-2024-07-18" USAGE = {"prompt_tokens": 11, "completion_tokens": 7, "total_tokens": 18} class FakeOpenAI: - """Serve /v1/chat/completions on 127.0.0.1 and record each request.""" + """Serve /v1/chat/completions on 127.0.0.1 and record each request. - def __init__(self, answer_text: str) -> None: + ``tool_call`` is ``{"id": ..., "name": ..., "arguments": }``; + ``error`` is ``[status, body]``. + """ + + def __init__( + self, + answer_text: str, + tool_call: Optional[dict[str, str]] = None, + error: Optional[tuple[int, dict[str, Any]]] = None, + ) -> None: self.requests: list[dict[str, Any]] = [] self.authorizations: list[str | None] = [] lock = threading.Lock() owner = self + if tool_call is not None: + message: dict[str, Any] = { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": tool_call["id"], + "type": "function", + "function": { + "name": tool_call["name"], + "arguments": tool_call["arguments"], + }, + } + ], + } + finish_reason = "tool_calls" + else: + message = {"role": "assistant", "content": answer_text} + finish_reason = "stop" + class Handler(BaseHTTPRequestHandler): def do_POST(self) -> None: # noqa: N802 - BaseHTTPRequestHandler API if self.path.rstrip("/") != "/v1/chat/completions": @@ -37,23 +68,22 @@ def do_POST(self) -> None: # noqa: N802 - BaseHTTPRequestHandler API with lock: owner.requests.append(request) owner.authorizations.append(self.headers.get("Authorization")) - body = json.dumps( - { + if error is not None: + status, reply = HTTPStatus(error[0]), error[1] + else: + status = HTTPStatus.OK + reply = { "id": "chatcmpl-fake", "object": "chat.completion", "created": 1_700_000_000, "model": RESPONSE_MODEL, "choices": [ - { - "index": 0, - "message": {"role": "assistant", "content": answer_text}, - "finish_reason": "stop", - } + {"index": 0, "message": message, "finish_reason": finish_reason} ], "usage": USAGE, } - ).encode("utf-8") - self.send_response(HTTPStatus.OK) + body = json.dumps(reply).encode("utf-8") + self.send_response(status) self.send_header("Content-Type", "application/json") self.send_header("Content-Length", str(len(body))) self.end_headers() diff --git a/python/examples/openlit/tests/failed_call.py b/python/examples/openlit/tests/failed_call.py new file mode 100644 index 000000000..c702c98ec --- /dev/null +++ b/python/examples/openlit/tests/failed_call.py @@ -0,0 +1,27 @@ +"""A test-only fixture: the recipe's model call against a server error. + +It runs ``init_tracing()`` and ``ask()`` from ``src/app.py`` as written +(content off). The one change: the OpenAI client is built with +``max_retries=0``, so the fake's error is raised at once instead of being +retried twice with back-off. The error is caught and its type printed, so +the process exits 0. Not part of the recipe. +""" + +from __future__ import annotations + +import functools +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "src")) + +from openai import APIStatusError # noqa: E402 + +import app # noqa: E402 + +app.OpenAI = functools.partial(app.OpenAI, max_retries=0) +app.init_tracing() +try: + app.ask(sys.argv[1]) +except APIStatusError as error: + print("raised", type(error).__name__, error.status_code) diff --git a/python/examples/openlit/tests/test_openlit_recipe.py b/python/examples/openlit/tests/test_openlit_recipe.py index 9aa4f6a2c..7914bdcd3 100644 --- a/python/examples/openlit/tests/test_openlit_recipe.py +++ b/python/examples/openlit/tests/test_openlit_recipe.py @@ -36,6 +36,8 @@ README = RECIPE_DIR / "README.md" VARIANT = TESTS_DIR / "recipe_variant.py" TRACED_FUNCTION = TESTS_DIR / "traced_function.py" +TOOL_CALL_SCRIPT = TESTS_DIR / "tool_call.py" +FAILED_CALL = TESTS_DIR / "failed_call.py" GUARD = TESTS_DIR / "_guarded_run.py" GUARD_PROBE = TESTS_DIR / "_guard_probe.py" @@ -57,6 +59,26 @@ POLICY = "Refunds are accepted within 30 days of purchase." CONTENT_MARKERS = ("QMARK7c1e", "AMARK5d2b", POLICY) +# Markers that DO reach the export with content off; README.md, Privacy, +# lists them as exceptions. A tool call whose generated arguments carry an +# email, the caller's ``user`` parameter, and a server error body. +TOOL_ARGS_MARKER = "TMARK3f9a" +TOOL_CALL = { + "id": "call_fake0001", + "name": "lookup_order", + "arguments": json.dumps({"email": TOOL_ARGS_MARKER + "@example.com"}), +} +END_USER = "UMARK8e4c@example.com" +ERROR_MARKER = "EMARK41d7" +ERROR_BODY = { + "error": { + "message": ERROR_MARKER + " The server had an error processing your request.", + "type": "server_error", + "param": None, + "code": None, + } +} + LLM_SPAN = "chat gpt-4o-mini" HTTP_SPAN = "POST" @@ -209,17 +231,32 @@ def _guard_attempts(guard_log: Path) -> list[dict[str, Any]]: return [json.loads(line) for line in guard_log.read_text().splitlines()] -def _run_script(tmp_path: Path, script: Path, **overrides: Optional[str]) -> dict[str, Any]: - """Run one script under the loopback guard; return everything the tests read.""" +def _launch( + tmp_path: Path, script: Path, endpoint: str, fake: FakeOpenAI, **overrides: Optional[str] +) -> tuple[Any, list[dict[str, Any]]]: + """Run one script under the loopback guard; return its result and the guard log.""" tmp_path.mkdir(parents=True, exist_ok=True) guard_log = tmp_path / "guard.jsonl" - with Receiver() as receiver, FakeOpenAI(ANSWER) as fake: - env = _child_env(tmp_path, guard_log, receiver.origin, fake.base_url, **overrides) - result = run( - [sys.executable, str(GUARD), str(script), QUESTION], - env=env, - stdin=None, - timeout=RUN_TIMEOUT_SECONDS, + env = _child_env(tmp_path, guard_log, endpoint, fake.base_url, **overrides) + result = run( + [sys.executable, str(GUARD), str(script), QUESTION], + env=env, + stdin=None, + timeout=RUN_TIMEOUT_SECONDS, + ) + return result, _guard_attempts(guard_log) + + +def _run_script( + tmp_path: Path, script: Path, fake: Optional[dict[str, Any]] = None, **overrides: Optional[str] +) -> dict[str, Any]: + """Run one script against a Receiver; return everything the tests read. + + ``fake`` holds extra FakeOpenAI arguments (``tool_call``, ``error``). + """ + with Receiver() as receiver, FakeOpenAI(ANSWER, **(fake or {})) as fake_openai: + result, guard_attempts = _launch( + tmp_path, script, receiver.origin, fake_openai, **overrides ) record = { "result": result, @@ -227,10 +264,10 @@ def _run_script(tmp_path: Path, script: Path, **overrides: Optional[str]) -> dic "stderr": result.stderr.decode("utf-8", "replace"), "requests": receiver.requests(), "spans": receiver.spans(), - "model_requests": list(fake.requests), - "model_authorizations": list(fake.authorizations), + "model_requests": list(fake_openai.requests), + "model_authorizations": list(fake_openai.authorizations), + "guard_attempts": guard_attempts, } - record["guard_attempts"] = _guard_attempts(guard_log) return record @@ -294,6 +331,17 @@ def _export_dump(record: dict[str, Any]) -> str: ) +def _keys_carrying(record: dict[str, Any], marker: str) -> set[str]: + """Span attribute, event attribute and status keys whose value contains ``marker``.""" + keys: set[str] = set() + for span in record["spans"]: + for attributes in [_attrs(span)] + [_attrs(event) for event in span.get("events", [])]: + keys |= {key for key, value in attributes.items() if marker in str(value)} + if marker in span.get("status", {}).get("message", ""): + keys.add("status.message") + return keys + + def _assert_ran(record: dict[str, Any]) -> None: result = record["result"] assert not result.timed_out, record["stderr"] @@ -568,6 +616,90 @@ def test_trace_decorator_ignores_the_content_switch(tmp_path: Path) -> None: assert spans[LLM_SPAN]["parentSpanId"] == spans["support_request"]["spanId"] +# The next tests prove the exceptions README.md lists under Privacy. They +# show what still leaks with content off; they are not privacy guarantees. + + +@pytest.fixture(scope="module") +def tool_call_run(tmp_path_factory: pytest.TempPathFactory) -> dict[str, Any]: + record = _run_script( + tmp_path_factory.mktemp("tool_call"), + TOOL_CALL_SCRIPT, + fake={"tool_call": TOOL_CALL}, + RECIPE_USER=END_USER, + ) + _assert_ran(record) + return record + + +def test_content_off_still_exports_the_models_tool_calls(tool_call_run: dict[str, Any]) -> None: + """Tool name, call ID and the model's generated arguments (utils.py:1524-1551).""" + request = tool_call_run["model_requests"][0] + assert [tool["function"]["name"] for tool in request["tools"]] == [TOOL_CALL["name"]] + assert tool_call_run["stdout"].strip() == TOOL_CALL["name"] + llm = _attrs(_by_name(tool_call_run["spans"])[LLM_SPAN]) + assert llm["gen_ai.tool.name"] == TOOL_CALL["name"] + assert llm["gen_ai.tool.call.id"] == TOOL_CALL["id"] + # The arguments arrive verbatim, planted email included. + assert llm["gen_ai.tool.args"] == TOOL_CALL["arguments"] + assert TOOL_ARGS_MARKER in llm["gen_ai.tool.args"] + # The messages themselves stay out. + assert not CONTENT_KEYS & set(llm) + dump = _export_dump(tool_call_run) + for marker in CONTENT_MARKERS: + assert marker not in dump, marker + + +def test_content_off_still_exports_the_request_user(tool_call_run: dict[str, Any]) -> None: + """The ``user`` argument goes to gen_ai.request.user (utils.py:1484-1487).""" + assert tool_call_run["model_requests"][0]["user"] == END_USER + llm = _attrs(_by_name(tool_call_run["spans"])[LLM_SPAN]) + assert llm["gen_ai.request.user"] == END_USER + + +@pytest.fixture(scope="module") +def failed_call_run(tmp_path_factory: pytest.TempPathFactory) -> dict[str, Any]: + record = _run_script( + tmp_path_factory.mktemp("failed_call"), + FAILED_CALL, + fake={"error": (HTTPStatus.INTERNAL_SERVER_ERROR, ERROR_BODY)}, + ) + _assert_ran(record) + return record + + +def test_failed_call_records_the_error_body_but_no_content(failed_call_run: dict[str, Any]) -> None: + """OpenLIT records the exception, then the SDK's span context records it again.""" + assert failed_call_run["stdout"].strip() == "raised InternalServerError 500" + assert len(failed_call_run["model_requests"]) == 1 # max_retries=0 + spans = _by_name(failed_call_run["spans"]) + assert set(spans) == {LLM_SPAN, HTTP_SPAN} + llm = spans[LLM_SPAN] + # openai.py:186-209 -> utils.py:77-84, then opentelemetry-api use_span. + events = llm.get("events", []) + assert [event["name"] for event in events] == ["exception", "exception"] + for event in events: + attributes = _attrs(event) + assert attributes["exception.type"] == "openai.InternalServerError" + # The openai SDK puts the whole response body in the message. + assert attributes["exception.message"] == "Error code: 500 - {0}".format(ERROR_BODY) + assert attributes["exception.message"] in attributes["exception.stacktrace"] + assert llm["status"]["code"] == "STATUS_CODE_ERROR" + assert llm["status"]["message"] == "InternalServerError: Error code: 500 - {0}".format( + ERROR_BODY + ) + assert _attrs(llm)["error.type"] == "InternalServerError" + assert _keys_carrying(failed_call_run, ERROR_MARKER) == { + "exception.message", + "exception.stacktrace", + "status.message", + } + # The fake's error body echoes no input, so no prompt or answer text arrives. + dump = _export_dump(failed_call_run) + for marker in CONTENT_MARKERS: + assert marker not in dump, marker + + def test_rejected_export_is_logged_and_the_app_still_exits_zero(tmp_path: Path) -> None: """fi-collector answers 401 without valid keys; OpenLIT only logs it.""" result, calls, guard_attempts = _run_against_recorder( @@ -665,6 +797,24 @@ def test_readme_key_inventory_matches_the_emitted_keys() -> None: assert listed == LLM_SPAN_KEYS | CONTENT_KEYS | HTTP_SPAN_KEYS +def test_readme_privacy_names_what_content_off_still_exports( + tool_call_run: dict[str, Any], failed_call_run: dict[str, Any] +) -> None: + """Each key a tool call adds, and each key a planted marker reached, is listed.""" + privacy = _readme_section("Privacy") + not_covered = privacy[privacy.index("Not covered by the content switch") :] + tool_llm = _attrs(_by_name(tool_call_run["spans"])[LLM_SPAN]) + exported = ( + (set(tool_llm) - LLM_SPAN_KEYS) + | _keys_carrying(tool_call_run, TOOL_ARGS_MARKER) + | _keys_carrying(tool_call_run, END_USER) + | _keys_carrying(failed_call_run, ERROR_MARKER) + ) + assert {"gen_ai.tool.args", "gen_ai.request.user", "exception.message"} <= exported + missing = sorted(key for key in exported if "`{0}`".format(key) not in not_covered) + assert missing == [], "not listed in README Privacy: {0}".format(missing) + + def _collector_aliases() -> dict[str, Any]: src = os.environ.get("FI_COLLECTOR_SRC") if not src: diff --git a/python/examples/openlit/tests/tool_call.py b/python/examples/openlit/tests/tool_call.py new file mode 100644 index 000000000..161239a8d --- /dev/null +++ b/python/examples/openlit/tests/tool_call.py @@ -0,0 +1,45 @@ +"""A test-only fixture: one tool-using model call after the recipe's ``init_tracing()``. + +It runs ``init_tracing()`` from ``src/app.py`` as written (content off), then +makes one Chat Completions call with the recipe's messages, a ``tools`` +list and ``user`` set to ``RECIPE_USER``. The fake answers with a tool call, +so the contract test can show what OpenLIT exports for tool calls and for +``user`` with content off. Not part of the recipe. +""" + +from __future__ import annotations + +import os +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "src")) + +from openai import OpenAI # noqa: E402 + +from app import _POLICY, MODEL, init_tracing # noqa: E402 + +LOOKUP_ORDER = { + "type": "function", + "function": { + "name": "lookup_order", + "description": "Find a customer's latest order.", + "parameters": { + "type": "object", + "properties": {"email": {"type": "string"}}, + "required": ["email"], + }, + }, +} + +init_tracing() +response = OpenAI().chat.completions.create( + model=MODEL, + messages=[ + {"role": "system", "content": "Answer using only this policy: " + _POLICY}, + {"role": "user", "content": sys.argv[1]}, + ], + tools=[LOOKUP_ORDER], + user=os.environ["RECIPE_USER"], +) +print(response.choices[0].message.tool_calls[0].function.name) From 9e4b125d3eeb027cc11f9a89505da4f73d0a2dee Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Mon, 5 Oct 2026 10:28:07 -0700 Subject: [PATCH 10/16] fix(examples): OpenLIT recipe exits with a message without the FI keys Review R5. init_tracing() read FI_API_KEY and FI_SECRET_KEY with os.environ[...], so a missing key ended in a KeyError traceback, unlike the endpoint and project checks. It now checks both next to them and raises SystemExit("Set FI_API_KEY and FI_SECRET_KEY ..."). The start-up test covers both keys and asserts no traceback and no key value in stderr. approved: Nikhil 2026-10-03 blanket Refs: TH-8338 Co-Authored-By: Claude Opus 5.5 --- python/examples/openlit/README.md | 4 +++- python/examples/openlit/src/app.py | 9 +++++++-- python/examples/openlit/tests/test_openlit_recipe.py | 7 ++++++- 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/python/examples/openlit/README.md b/python/examples/openlit/README.md index 1a74abe58..aedcd3410 100644 --- a/python/examples/openlit/README.md +++ b/python/examples/openlit/README.md @@ -41,7 +41,9 @@ python src/app.py "What is the refund window?" ## The recipe `init_tracing()` in `src/app.py` is the whole integration. It first checks -that the endpoint and the project are set, then calls: +that `OTEL_EXPORTER_OTLP_ENDPOINT`, `FI_API_KEY`, `FI_SECRET_KEY` and the +project are set, and otherwise exits with a message saying what to set +(tested). Then it calls: ```python openlit.init( diff --git a/python/examples/openlit/src/app.py b/python/examples/openlit/src/app.py index fd08fe4f7..2fbe1289b 100644 --- a/python/examples/openlit/src/app.py +++ b/python/examples/openlit/src/app.py @@ -35,6 +35,11 @@ def init_tracing() -> None: if not endpoint: # Without an endpoint OpenLIT prints every span to stdout instead. raise SystemExit("Set OTEL_EXPORTER_OTLP_ENDPOINT to the fi-collector origin.") + api_key = os.environ.get("FI_API_KEY") + secret_key = os.environ.get("FI_SECRET_KEY") + if not (api_key and secret_key): + # fi-collector answers 401 without both, and OpenLIT only logs that. + raise SystemExit("Set FI_API_KEY and FI_SECRET_KEY to your Future AGI keys.") # openlit.init() has no resource argument; its Resource.create() merges # OTEL_RESOURCE_ATTRIBUTES. fi-collector rejects (HTTP 400) a batch whose # resource has no project_name, so stop here instead. @@ -49,8 +54,8 @@ def init_tracing() -> None: # OpenLIT joins this dict into OTEL_EXPORTER_OTLP_HEADERS, which the # exporter splits on commas and percent-decodes: encode the values. otlp_headers={ - "x-api-key": quote(os.environ["FI_API_KEY"], safe=""), - "x-secret-key": quote(os.environ["FI_SECRET_KEY"], safe=""), + "x-api-key": quote(api_key, safe=""), + "x-secret-key": quote(secret_key, safe=""), }, capture_message_content=False, # OpenLIT's default is True disable_metrics=True, # fi-collector serves no /v1/metrics diff --git a/python/examples/openlit/tests/test_openlit_recipe.py b/python/examples/openlit/tests/test_openlit_recipe.py index 7914bdcd3..adb837423 100644 --- a/python/examples/openlit/tests/test_openlit_recipe.py +++ b/python/examples/openlit/tests/test_openlit_recipe.py @@ -719,15 +719,20 @@ def test_rejected_export_is_logged_and_the_app_still_exits_zero(tmp_path: Path) [ ("OTEL_EXPORTER_OTLP_ENDPOINT", "Set OTEL_EXPORTER_OTLP_ENDPOINT"), ("OTEL_RESOURCE_ATTRIBUTES", "Set OTEL_RESOURCE_ATTRIBUTES"), + ("FI_API_KEY", "Set FI_API_KEY and FI_SECRET_KEY"), + ("FI_SECRET_KEY", "Set FI_API_KEY and FI_SECRET_KEY"), ], ) -def test_app_refuses_to_start_without_endpoint_or_project( +def test_app_refuses_to_start_without_endpoint_keys_or_project( tmp_path: Path, unset: str, message: str ) -> None: record = _run_script(tmp_path, APP, **{unset: None}) assert not record["result"].timed_out assert record["result"].returncode != 0 assert message in record["stderr"] + assert "Traceback" not in record["stderr"] + for secret in (FI_API_KEY, FI_SECRET_KEY): + assert secret not in record["stderr"] assert record["requests"] == [] assert record["model_requests"] == [] assert record["guard_attempts"] == [] From fcc86665078483133135894d8a40876b19e71ac6 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Mon, 5 Oct 2026 10:28:18 -0700 Subject: [PATCH 11/16] test(examples): check the README's openlit.init() call against app.py Review V2. Nothing tied the README's copy-paste surfaces to src/app.py. A test now extracts the openlit.init(...) keyword arguments from the README code block and from app.py with ast and asserts they are equal, including capture_message_content=False. Another asserts that the Run block exports the variables app.py checks and that its OTEL_RESOURCE_ATTRIBUTES gives a project_name under the parser app.py uses. The README block now shows app.py's call verbatim, with comments naming each variable. approved: Nikhil 2026-10-03 blanket Refs: TH-8338 Co-Authored-By: Claude Opus 5.5 --- python/examples/openlit/README.md | 11 ++-- .../openlit/tests/test_openlit_recipe.py | 50 +++++++++++++++++++ 2 files changed, 57 insertions(+), 4 deletions(-) diff --git a/python/examples/openlit/README.md b/python/examples/openlit/README.md index aedcd3410..e4f21db78 100644 --- a/python/examples/openlit/README.md +++ b/python/examples/openlit/README.md @@ -47,11 +47,11 @@ project are set, and otherwise exits with a message saying what to set ```python openlit.init( - service_name="openlit-recipe", # service.name - otlp_endpoint=os.environ["OTEL_EXPORTER_OTLP_ENDPOINT"], + service_name=APP_NAME, # "openlit-recipe", exported as service.name + otlp_endpoint=endpoint, # OTEL_EXPORTER_OTLP_ENDPOINT otlp_headers={ - "x-api-key": quote(os.environ["FI_API_KEY"], safe=""), - "x-secret-key": quote(os.environ["FI_SECRET_KEY"], safe=""), + "x-api-key": quote(api_key, safe=""), # FI_API_KEY + "x-secret-key": quote(secret_key, safe=""), # FI_SECRET_KEY }, capture_message_content=False, disable_metrics=True, @@ -60,6 +60,9 @@ openlit.init( ) ``` +A test checks that this call has the same arguments as the one in +`src/app.py`, and that the "Run" block sets a `project_name`. + The 1.45.0 signature, quoted from the wheel's `openlit/__init__.py:185-209`: ```python diff --git a/python/examples/openlit/tests/test_openlit_recipe.py b/python/examples/openlit/tests/test_openlit_recipe.py index adb837423..7b4c5781b 100644 --- a/python/examples/openlit/tests/test_openlit_recipe.py +++ b/python/examples/openlit/tests/test_openlit_recipe.py @@ -13,6 +13,7 @@ from __future__ import annotations +import ast import importlib.metadata import importlib.util import json @@ -820,6 +821,55 @@ def test_readme_privacy_names_what_content_off_still_exports( assert missing == [], "not listed in README Privacy: {0}".format(missing) +def _code_blocks(section: str, language: str) -> list[str]: + return re.findall(r"```{0}\n(.*?)```".format(language), section, flags=re.DOTALL) + + +def _init_keywords(source: str) -> dict[Optional[str], ast.expr]: + """The keyword arguments of the one ``openlit.init(...)`` call in ``source``.""" + calls = [ + node + for node in ast.walk(ast.parse(source)) + if isinstance(node, ast.Call) and ast.unparse(node.func) == "openlit.init" + ] + assert len(calls) == 1, source + assert calls[0].args == [] + return {keyword.arg: keyword.value for keyword in calls[0].keywords} + + +def test_readme_init_call_is_the_one_in_app_py() -> None: + blocks = [ + block + for block in _code_blocks(_readme_section("The recipe"), "python") + if "openlit.init(" in block + ] + assert len(blocks) == 1 + readme = _init_keywords(blocks[0]) + app = _init_keywords(APP.read_text(encoding="utf-8")) + assert {key: ast.dump(value) for key, value in readme.items()} == { + key: ast.dump(value) for key, value in app.items() + } + assert ast.literal_eval(app["capture_message_content"]) is False + + +def test_readme_run_block_sets_what_app_py_checks(monkeypatch: pytest.MonkeyPatch) -> None: + from opentelemetry.sdk.resources import OTELResourceDetector + + (block,) = _code_blocks(_readme_section("Run"), "bash") + exports = dict(re.findall(r'^export (\w+)="([^"]*)"', block, flags=re.MULTILINE)) + for name in ( + "FI_API_KEY", + "FI_SECRET_KEY", + "OTEL_EXPORTER_OTLP_ENDPOINT", + "OTEL_RESOURCE_ATTRIBUTES", + "OPENAI_API_KEY", + ): + assert name in exports, name + # The parser src/app.py checks the project with. + monkeypatch.setenv("OTEL_RESOURCE_ATTRIBUTES", exports["OTEL_RESOURCE_ATTRIBUTES"]) + assert OTELResourceDetector().detect().attributes.get("project_name") + + def _collector_aliases() -> dict[str, Any]: src = os.environ.get("FI_COLLECTOR_SRC") if not src: From 78b2a9c321f099a7363d5cdd40b56dd7b64b773c Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Mon, 5 Oct 2026 10:28:27 -0700 Subject: [PATCH 12/16] docs(examples): spell out both register() orders for the OpenLIT recipe Review R2. The README implied that traceAI's register() sets a global tracer provider; by default it does not. tests/register_first.py calls register() before the recipe's init_tracing() and runs the model call inside a register() span, against two loopback receivers. Tested: - register(set_global_tracer_provider=True): openlit.init() reuses it; otlp_endpoint, otlp_headers and OTEL_RESOURCE_ATTRIBUTES are ignored for spans, and all spans go to FI_BASE_URL under register()'s project. - default register(): two providers and two exports, OpenLIT's spans under the OTEL_RESOURCE_ATTRIBUTES project and the register() span under register()'s, still one trace. approved: Nikhil 2026-10-03 blanket Refs: TH-8338 Co-Authored-By: Claude Opus 5.5 --- python/examples/openlit/README.md | 32 ++++++++--- .../examples/openlit/tests/register_first.py | 35 ++++++++++++ .../openlit/tests/test_openlit_recipe.py | 53 +++++++++++++++++++ 3 files changed, 114 insertions(+), 6 deletions(-) create mode 100644 python/examples/openlit/tests/register_first.py diff --git a/python/examples/openlit/README.md b/python/examples/openlit/README.md index e4f21db78..a50993d45 100644 --- a/python/examples/openlit/README.md +++ b/python/examples/openlit/README.md @@ -416,10 +416,29 @@ does not do it for you. To keep only traceai-openai's span, pass only `ChatCompletion` and `POST`. To keep only OpenLIT's, do not enable traceai-openai. -If traceAI's `register()`, or anything else, has already set a global tracer -provider, `openlit.init()` reuses it and adds no exporter, so -`otlp_endpoint` and `otlp_headers` do nothing for spans -(`tracing.py:56-59`; read from source, not tested). +### traceAI's `register()` + +`openlit.init()` reuses the global tracer provider if it is an SDK +`TracerProvider`, and otherwise creates its own (`tracing.py:56-59`). +traceAI's `register()` makes its provider global only when asked +(`set_global_tracer_provider: bool = False`, +`python/fi_instrumentation/otel.py:102`, `:245-246`). The test +(`tests/register_first.py`) calls `register()` first, then the recipe's +`init_tracing()`, and makes the model call inside a span from `register()`'s +provider. Both cases were tested: + +- `register(set_global_tracer_provider=True)` first: `openlit.init()` reuses + that provider and adds no exporter, so `otlp_endpoint`, `otlp_headers` and + `OTEL_RESOURCE_ATTRIBUTES` are ignored for spans. All three spans went to + `register()`'s endpoint (`FI_BASE_URL` plus `/tracer/v1/traces`) with + `register()`'s keys and project. +- `register()` with its default first: the process has two providers and + two exports. OpenLIT's spans (`chat gpt-4o-mini`, `POST`) went to + `otlp_endpoint` under the `OTEL_RESOURCE_ATTRIBUTES` project, and the + `register()` span went to `FI_BASE_URL` under `register()`'s project. They + are still one trace: the model-call span's parent is the `register()` span, + which is in the other export, and in the other project unless the two + project names match. ### Firecrawl @@ -489,8 +508,9 @@ env -u PYTHONPATH PYTHONPATH="python/examples/openlit:python:python/tests" \ ``` For Python 3.13, replace `--python 3.11`. `python` is on the path and -`jsonschema` is installed only because pytest imports `python/__init__.py`, -which imports `fi_instrumentation`; the recipe needs neither. +`jsonschema` is installed because pytest imports `python/__init__.py`, which +imports `fi_instrumentation`, and because the `register()` test imports +`fi_instrumentation` from `python/`. The recipe needs neither. The two traceai-openai tests skip in that environment. To run them, add `--with 'traceai-openai==0.1.10'`. That package requires `wrapt<2` (through diff --git a/python/examples/openlit/tests/register_first.py b/python/examples/openlit/tests/register_first.py new file mode 100644 index 000000000..f7eba6389 --- /dev/null +++ b/python/examples/openlit/tests/register_first.py @@ -0,0 +1,35 @@ +"""A test-only fixture: traceAI's ``register()`` before the recipe's ``init_tracing()``. + +``register()`` runs first: with ``set_global_tracer_provider=True`` when +``REGISTER_GLOBAL=1``, else with its default (``False``). Then +``init_tracing()`` from ``src/app.py`` runs as written, and the recipe's +model call runs inside one span from ``register()``'s provider, standing in +for a traceAI instrumentor's span. ``register()`` exports to ``FI_BASE_URL`` +under the project ``REGISTER_PROJECT``. traceAI is imported from this +repository's ``python/`` directory. Not part of the recipe. +""" + +from __future__ import annotations + +import os +import sys +from pathlib import Path + +HERE = Path(__file__).resolve() +sys.path.insert(0, str(HERE.parents[3])) # python/, for fi_instrumentation +sys.path.insert(0, str(HERE.parents[1] / "src")) + +from fi_instrumentation import register # noqa: E402 +from fi_instrumentation.fi_types import ProjectType # noqa: E402 + +from app import ask, init_tracing # noqa: E402 + +provider = register( + project_name=os.environ["REGISTER_PROJECT"], + project_type=ProjectType.OBSERVE, + set_global_tracer_provider=os.environ.get("REGISTER_GLOBAL") == "1", + verbose=False, +) +init_tracing() +with provider.get_tracer("register_first").start_as_current_span("traceai_span"): + print(ask(sys.argv[1])) diff --git a/python/examples/openlit/tests/test_openlit_recipe.py b/python/examples/openlit/tests/test_openlit_recipe.py index 7b4c5781b..fd52fb3ce 100644 --- a/python/examples/openlit/tests/test_openlit_recipe.py +++ b/python/examples/openlit/tests/test_openlit_recipe.py @@ -39,6 +39,7 @@ TRACED_FUNCTION = TESTS_DIR / "traced_function.py" TOOL_CALL_SCRIPT = TESTS_DIR / "tool_call.py" FAILED_CALL = TESTS_DIR / "failed_call.py" +REGISTER_FIRST = TESTS_DIR / "register_first.py" GUARD = TESTS_DIR / "_guarded_run.py" GUARD_PROBE = TESTS_DIR / "_guard_probe.py" @@ -785,6 +786,58 @@ def test_disabling_openlits_openai_instrumentor_leaves_one_model_span(tmp_path: assert spans[HTTP_SPAN]["parentSpanId"] == spans["ChatCompletion"]["spanId"] +REGISTER_PROJECT = "register-project" + + +@pytest.mark.parametrize("global_provider", [False, True], ids=["default", "global"]) +def test_register_before_openlit_init(tmp_path: Path, global_provider: bool) -> None: + """traceAI's register() first: one provider only if it was made global.""" + with Receiver() as recipe_collector, Receiver() as register_collector, FakeOpenAI( + ANSWER + ) as fake: + result, guard_attempts = _launch( + tmp_path, + REGISTER_FIRST, + recipe_collector.origin, + fake, + FI_BASE_URL=register_collector.origin, + REGISTER_PROJECT=REGISTER_PROJECT, + REGISTER_GLOBAL="1" if global_provider else None, + ) + recipe_requests, recipe_spans = recipe_collector.requests(), recipe_collector.spans() + register_requests = register_collector.requests() + register_spans = register_collector.spans() + assert not result.timed_out + assert result.returncode == 0, result.stderr.decode("utf-8", "replace") + assert guard_attempts == [] + assert ANSWER in result.stdout.decode("utf-8", "replace") + + def projects(requests: list[dict[str, Any]]) -> set[str]: + return {r["project_name"] for request in requests for r in request["resource_attributes"]} + + # One trace either way: the model call is a child of register()'s span. + spans = _by_name(recipe_spans + register_spans) + assert set(spans) == {"traceai_span", LLM_SPAN, HTTP_SPAN} + assert len({span["traceId"] for span in spans.values()}) == 1 + assert spans[LLM_SPAN]["parentSpanId"] == spans["traceai_span"]["spanId"] + if global_provider: + # openlit.init() reused register()'s provider: otlp_endpoint, + # otlp_headers and OTEL_RESOURCE_ATTRIBUTES did nothing for spans. + assert recipe_requests == [] + assert {span["name"] for span in register_spans} == set(spans) + else: + # Two providers, two exports, two projects. + assert {span["name"] for span in recipe_spans} == {LLM_SPAN, HTTP_SPAN} + assert {request["path"] for request in recipe_requests} == {"/v1/traces"} + assert projects(recipe_requests) == {PROJECT} + assert {span["name"] for span in register_spans} == {"traceai_span"} + assert {request["path"] for request in register_requests} == {"/tracer/v1/traces"} + assert projects(register_requests) == {REGISTER_PROJECT} + for request in register_requests: + assert request["headers"].get("x-api-key") == FI_API_KEY + assert request["headers"].get("x-secret-key") == FI_SECRET_KEY + + # -------------------------------------------------------------------------- # README checks, and an opt-in check against the collector's alias lists. # -------------------------------------------------------------------------- From b80887e2cf0e12c6b6b4d0ec26e8e89e4ed4e323 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Mon, 5 Oct 2026 10:28:33 -0700 Subject: [PATCH 13/16] test(examples): read fi-collector's input and output keys from converter.go Review R4. The opt-in drift test hardcoded input.value/output.value, so a collector change to the keys behind the input and output columns left the README rows stale while the test passed. It now extracts the keys passed to overflowAsString(...) for input and output from converter.go, expects the README rows to name exactly those keys, checks that none was emitted, and checks that the content-on keys are not among them. Against a converter.go copy with input.value changed to gen_ai.input.messages, the old test passes and the new one fails. approved: Nikhil 2026-10-03 blanket Refs: TH-8338 Co-Authored-By: Claude Opus 5.5 --- python/examples/openlit/README.md | 14 +++++++----- .../openlit/tests/test_openlit_recipe.py | 22 ++++++++++++++++--- 2 files changed, 27 insertions(+), 9 deletions(-) diff --git a/python/examples/openlit/README.md b/python/examples/openlit/README.md index a50993d45..74c7b5ddb 100644 --- a/python/examples/openlit/README.md +++ b/python/examples/openlit/README.md @@ -281,8 +281,9 @@ fi-collector promotes a few keys to columns. This table is everything this recipe calls "displayed". It was read from fi-collector at future-agi `main` 4af5338 (`pkg/adapter/adapter.go`, `exporter/clickhouse25exporter/converter.go`); the opt-in test (see -"Tests") re-reads the alias lists from that source and checks this table -against them and against the keys the run emitted. No fi-collector was run. +"Tests") re-reads from that source the alias lists and the keys `input` and +`output` are read from, and checks this table against them and against the +keys the run emitted. No fi-collector was run. | Column | Filled from | Source | |---|---|---| @@ -518,10 +519,11 @@ The two traceai-openai tests skip in that environment. To run them, add 2.5.0 without it. The whole suite passes in both environments. One test is opt-in. It reads the alias lists from fi-collector's -`adapter.go` and `converter.go` and checks the "What Future AGI shows" table -against them and against the emitted keys. Point `FI_COLLECTOR_SRC` at the -`fi-collector` directory of a future-agi checkout (tested with `main` -4af5338): +`adapter.go` and `converter.go`, and the keys passed to `overflowAsString` +for `input` and `output` in `converter.go`, and checks the "What Future AGI +shows" table against them and against the emitted keys. Point +`FI_COLLECTOR_SRC` at the `fi-collector` directory of a future-agi checkout +(tested with `main` 4af5338): ```bash export FI_COLLECTOR_SRC=/fi-collector # then run the command above diff --git a/python/examples/openlit/tests/test_openlit_recipe.py b/python/examples/openlit/tests/test_openlit_recipe.py index fd52fb3ce..1020bef24 100644 --- a/python/examples/openlit/tests/test_openlit_recipe.py +++ b/python/examples/openlit/tests/test_openlit_recipe.py @@ -958,6 +958,13 @@ def string_slice(source: str, name: str) -> list[str]: aliases["spanKindSynonyms"] = dict( re.findall(r'"([^"]+)"\s*:\s*"([^"]+)"', synonyms_body.group(1)) ) + # The input and output columns: `input := overflowAsString(overflow, "...")`. + for column in ("input", "output"): + aliases[column] = re.findall( + r'\b{0}\s*:?=\s*overflowAsString\(\s*\w+\s*,\s*"([^"]+)"\s*\)'.format(column), + converter, + ) + assert aliases[column], column return aliases @@ -983,18 +990,27 @@ def test_readme_display_table_matches_the_collector_aliases(recipe_run: dict[str aliases["costTotalKeys"] + aliases["costInputKeys"] + aliases["costOutputKeys"], ), "observation_type": kind_key, - "input": "input.value" if "input.value" in llm else None, - "output": "output.value" if "output.value" in llm else None, + "input": _first(llm, aliases["input"]), + "output": _first(llm, aliases["output"]), } # gen_ai.operation.name=chat resolves to the llm observation type. assert aliases["spanKindSynonyms"].get(llm[kind_key]) == "llm" + section = _readme_section("What Future AGI shows") rows = { column: key or word for column, key, word in re.findall( r"^\| `([a-z_]+)` \| (?:`([a-z_.]+)`|(derived|none))", - _readme_section("What Future AGI shows"), + section, flags=re.MULTILINE, ) } assert rows == {column: key or "none" for column, key in expected.items()} + # The input and output rows name exactly the keys converter.go reads, and + # the content-on keys (Key inventory: "not read as the span's input") are + # not among them. + for column in ("input", "output"): + row = re.search(r"^\| `{0}` \|([^|]*)\|".format(column), section, flags=re.MULTILINE) + assert row, column + assert set(re.findall(r"`([a-z_.]+)`", row.group(1))) == set(aliases[column]), column + assert not CONTENT_KEYS & set(aliases["input"] + aliases["output"]) From ebbdf072218a41ced5d1c94e201ea831e6a14186 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Mon, 5 Oct 2026 10:28:42 -0700 Subject: [PATCH 14/16] docs(examples): list the environment that overrides the OpenLIT recipe Review R3, plus the verifier's addition. New README subsection, each item tested unless marked: - OTEL_EXPORTER_OTLP_TRACES_HEADERS replaces OpenLIT's header string, so the FI keys are dropped and exports get 401 (a recorder that answers 401 without the keys, as fi-collector's auth middleware does). - OTEL_EXPORTER_OTLP_TRACES_ENDPOINT wins over otlp_endpoint, is used as is, and the FI keys go to it. - OTEL_TRACES_EXPORTER=none|console adds no OTLP exporter. - OTEL_EXPORTER_OTLP_HEADERS is overwritten (read from source). - In-process inheritance: an OTLP exporter built later in the same process without headers (tests/second_exporter.py) sends the FI keys to whatever endpoint it was given. A README test fails if the subsection stops naming these variables. approved: Nikhil 2026-10-03 blanket Refs: TH-8338 Co-Authored-By: Claude Opus 5.5 --- python/examples/openlit/README.md | 46 +++++- .../examples/openlit/tests/second_exporter.py | 31 ++++ .../openlit/tests/test_openlit_recipe.py | 138 +++++++++++++++--- 3 files changed, 192 insertions(+), 23 deletions(-) create mode 100644 python/examples/openlit/tests/second_exporter.py diff --git a/python/examples/openlit/README.md b/python/examples/openlit/README.md index 74c7b5ddb..a5aa6b60d 100644 --- a/python/examples/openlit/README.md +++ b/python/examples/openlit/README.md @@ -121,9 +121,9 @@ that mux does; no deployed endpoint was called. Read from source, not tested: with `otlp_endpoint` left out, OpenLIT uses `OTEL_EXPORTER_OTLP_ENDPOINT` from the environment, and with neither it prints every span to stdout (`tracing.py:121-131`). `src/app.py` refuses to -start without the variable instead (tested). `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT`, -if set, is used as the full URL with nothing appended -(`_common/__init__.py:133-134`). `OTEL_EXPORTER_OTLP_PROTOCOL=grpc` switches +start without the variable instead (tested). A set +`OTEL_EXPORTER_OTLP_TRACES_ENDPOINT` wins over all of this (see "Environment +that overrides the recipe"). `OTEL_EXPORTER_OTLP_PROTOCOL=grpc` switches OpenLIT to the gRPC exporter (`tracing.py:23-26`); this recipe does not test gRPC. @@ -142,8 +142,9 @@ that contains `,` and `=`. Without the encoding, a comma cuts the value short (tested). Because OpenLIT writes the joined string to `os.environ`, the keys stay in -the process environment after `init()` and any child process the app starts -inherits them (read from source). +the process environment after `init()`. Child processes inherit them (read +from source), and so does any OTLP exporter created later in the same +process (tested; see "Environment that overrides the recipe"). ### Project @@ -189,6 +190,41 @@ and the price download is the only URL the `init()` path calls; No recipe run in the contract test made any non-loopback connection attempt. +### Environment that overrides the recipe + +If these variables are already set when `openlit.init()` runs, they +replace or widen what the recipe configures. `src/app.py` does not check +them. + +- `OTEL_EXPORTER_OTLP_TRACES_HEADERS` replaces the header string OpenLIT + built: the exporter reads it instead of `OTEL_EXPORTER_OTLP_HEADERS` + (`_common/__init__.py:146-147`). The Future AGI keys are then not sent, + and every export gets 401, which is only logged. Tested with + `x-team=support` against a recorder that answers 401 without the keys, as + fi-collector's auth middleware does (`middleware.go:18-21`). +- `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT` wins over `otlp_endpoint` and is used + as the full URL, with nothing appended (`_common/__init__.py:133-134`). + The Future AGI keys go to that URL, wherever it points (tested). +- `OTEL_TRACES_EXPORTER` makes OpenLIT add an OTLP exporter only if the list + includes `otlp` (`tracing.py:86-119`). With `none` nothing is exported; + with `console` each span is printed to stdout instead (both tested). +- `OTEL_EXPORTER_OTLP_HEADERS` is overwritten by OpenLIT with the recipe's + headers (`tracing.py:83`), so headers you put there are dropped (read from + source). + +The keys also leak the other way. OpenLIT writes the endpoint and the +joined keys to `OTEL_EXPORTER_OTLP_ENDPOINT` and `OTEL_EXPORTER_OTLP_HEADERS` +in `os.environ` (`tracing.py:72-83`). Any OTLP/HTTP exporter created later +in the same process without its own headers reads them and sends the Future +AGI keys to whatever endpoint it was given. The test builds a second +exporter for another loopback receiver after `init_tracing()`, and that +receiver got both keys. An exporter built without an endpoint also sends its +spans to fi-collector (read from source). + +When `openlit.init()` reuses an existing tracer provider, it builds no +exporter and writes neither variable (`tracing.py:56-59`; see "traceAI's +`register()`"). + ## What is traced With content off, `python src/app.py` exports two spans in one trace: diff --git a/python/examples/openlit/tests/second_exporter.py b/python/examples/openlit/tests/second_exporter.py new file mode 100644 index 000000000..160203760 --- /dev/null +++ b/python/examples/openlit/tests/second_exporter.py @@ -0,0 +1,31 @@ +"""A test-only fixture: another OTLP exporter in the same process, after ``init_tracing()``. + +It runs ``init_tracing()`` from ``src/app.py`` as written, then builds a +plain OpenTelemetry OTLP/HTTP span exporter for ``SECOND_COLLECTOR`` with no +headers, as another library in the app might, and exports one span through +it. Not part of the recipe. +""" + +from __future__ import annotations + +import os +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "src")) + +from opentelemetry.exporter.otlp.proto.http.trace_exporter import ( # noqa: E402 + OTLPSpanExporter, +) +from opentelemetry.sdk.trace import TracerProvider # noqa: E402 +from opentelemetry.sdk.trace.export import SimpleSpanProcessor # noqa: E402 + +from app import init_tracing # noqa: E402 + +init_tracing() +other = TracerProvider() +other.add_span_processor( + SimpleSpanProcessor(OTLPSpanExporter(endpoint=os.environ["SECOND_COLLECTOR"])) +) +other.get_tracer("second_exporter").start_span("other_library_span").end() +other.shutdown() diff --git a/python/examples/openlit/tests/test_openlit_recipe.py b/python/examples/openlit/tests/test_openlit_recipe.py index 1020bef24..19b6f14dc 100644 --- a/python/examples/openlit/tests/test_openlit_recipe.py +++ b/python/examples/openlit/tests/test_openlit_recipe.py @@ -40,6 +40,7 @@ TOOL_CALL_SCRIPT = TESTS_DIR / "tool_call.py" FAILED_CALL = TESTS_DIR / "failed_call.py" REGISTER_FIRST = TESTS_DIR / "register_first.py" +SECOND_EXPORTER = TESTS_DIR / "second_exporter.py" GUARD = TESTS_DIR / "_guarded_run.py" GUARD_PROBE = TESTS_DIR / "_guard_probe.py" @@ -157,9 +158,12 @@ class _PathRecorder: By default it answers /v1/traces and /tracer/v1/traces with 200 and anything else with 404, as fi-collector's HTTP mux does (pkg/server/server.go:233-234). ``status`` forces one reply for all. + With ``require_keys``, a request without the placeholder Future AGI keys + gets 401 first, as fi-collector's auth middleware does + (pkg/auth/middleware.go:18-21). """ - def __init__(self, status: Optional[HTTPStatus] = None) -> None: + def __init__(self, status: Optional[HTTPStatus] = None, require_keys: bool = False) -> None: self.calls: list[tuple[str, str, int]] = [] lock = threading.Lock() owner = self @@ -168,8 +172,11 @@ class Handler(BaseHTTPRequestHandler): def do_POST(self) -> None: # noqa: N802 path = urlsplit(self.path).path self.rfile.read(int(self.headers.get("Content-Length", "0"))) + keys = (self.headers.get("x-api-key"), self.headers.get("x-secret-key")) if status is not None: reply = status + elif require_keys and keys != (FI_API_KEY, FI_SECRET_KEY): + reply = HTTPStatus.UNAUTHORIZED elif path in ("/v1/traces", "/tracer/v1/traces"): reply = HTTPStatus.OK else: @@ -279,24 +286,20 @@ def _run_variant(tmp_path: Path, **init_overrides: Any) -> dict[str, Any]: def _run_against_recorder( - tmp_path: Path, suffix: str = "", status: Optional[HTTPStatus] = None, **overrides: Optional[str] + tmp_path: Path, + suffix: str = "", + status: Optional[HTTPStatus] = None, + require_keys: bool = False, + **overrides: Optional[str], ) -> tuple[Any, list[tuple[str, str, int]], list[dict[str, Any]]]: """Run a script with a _PathRecorder as the collector; return result, calls, guard log.""" - tmp_path.mkdir(parents=True, exist_ok=True) - guard_log = tmp_path / "guard.jsonl" script = VARIANT if "RECIPE_INIT_OVERRIDES" in overrides else APP - with _PathRecorder(status) as recorder, FakeOpenAI(ANSWER) as fake: - env = _child_env( - tmp_path, guard_log, recorder.origin + suffix, fake.base_url, **overrides - ) - result = run( - [sys.executable, str(GUARD), str(script), QUESTION], - env=env, - stdin=None, - timeout=RUN_TIMEOUT_SECONDS, + with _PathRecorder(status, require_keys) as recorder, FakeOpenAI(ANSWER) as fake: + result, guard_attempts = _launch( + tmp_path, script, recorder.origin + suffix, fake, **overrides ) calls = list(recorder.calls) - return result, calls, _guard_attempts(guard_log) + return result, calls, guard_attempts def _value(value: dict[str, Any]) -> Any: @@ -716,6 +719,91 @@ def test_rejected_export_is_logged_and_the_app_still_exits_zero(tmp_path: Path) assert ANSWER in result.stdout.decode("utf-8", "replace") +# Environment that overrides the recipe (README.md, The recipe). + + +@pytest.mark.parametrize( + ("traces_headers", "expected_status"), + [(None, HTTPStatus.OK), ("x-team=support", HTTPStatus.UNAUTHORIZED)], +) +def test_traces_headers_variable_replaces_the_recipes_headers( + tmp_path: Path, traces_headers: Optional[str], expected_status: HTTPStatus +) -> None: + """The exporter reads OTEL_EXPORTER_OTLP_TRACES_HEADERS instead of OpenLIT's string.""" + result, calls, guard_attempts = _run_against_recorder( + tmp_path, require_keys=True, OTEL_EXPORTER_OTLP_TRACES_HEADERS=traces_headers + ) + assert not result.timed_out + assert result.returncode == 0, result.stderr.decode("utf-8", "replace") + assert guard_attempts == [] + assert {(path, status) for _method, path, status in calls} == { + ("/v1/traces", int(expected_status)) + } + stderr = result.stderr.decode("utf-8", "replace") + rejected = "Failed to export spans batch code: 401" in stderr + assert rejected == (expected_status == HTTPStatus.UNAUTHORIZED), stderr + + +def test_traces_endpoint_variable_wins_over_otlp_endpoint(tmp_path: Path) -> None: + """A set OTEL_EXPORTER_OTLP_TRACES_ENDPOINT is used as is, and the keys go there.""" + with _PathRecorder() as recipe_target, Receiver() as elsewhere, FakeOpenAI(ANSWER) as fake: + result, guard_attempts = _launch( + tmp_path, + APP, + recipe_target.origin, + fake, + OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=elsewhere.collector_endpoint, + ) + calls, requests = list(recipe_target.calls), elsewhere.requests() + assert not result.timed_out + assert result.returncode == 0, result.stderr.decode("utf-8", "replace") + assert guard_attempts == [] + assert calls == [] + # The full URL, nothing appended: /tracer/v1/traces/v1/traces would 404. + assert requests, "no export reached the other collector" + assert {request["path"] for request in requests} == {"/tracer/v1/traces"} + for request in requests: + assert request["headers"].get("x-api-key") == FI_API_KEY + assert request["headers"].get("x-secret-key") == FI_SECRET_KEY + + +@pytest.mark.parametrize("exporter", ["none", "console"]) +def test_traces_exporter_variable_can_leave_out_the_otlp_exporter( + tmp_path: Path, exporter: str +) -> None: + result, calls, guard_attempts = _run_against_recorder( + tmp_path, OTEL_TRACES_EXPORTER=exporter + ) + assert not result.timed_out + assert result.returncode == 0, result.stderr.decode("utf-8", "replace") + assert guard_attempts == [] + assert calls == [] + stdout = result.stdout.decode("utf-8", "replace") + assert ANSWER in stdout + # console prints each span to stdout instead. + assert ('"name": "{0}"'.format(LLM_SPAN) in stdout) == (exporter == "console") + + +def test_a_later_otlp_exporter_in_the_process_sends_the_future_agi_keys(tmp_path: Path) -> None: + """OpenLIT leaves the keys in OTEL_EXPORTER_OTLP_HEADERS for every later exporter.""" + with Receiver() as recipe_collector, Receiver() as elsewhere, FakeOpenAI(ANSWER) as fake: + result, guard_attempts = _launch( + tmp_path, + SECOND_EXPORTER, + recipe_collector.origin, + fake, + SECOND_COLLECTOR=elsewhere.endpoint, + ) + requests, spans = elsewhere.requests(), elsewhere.spans() + assert not result.timed_out + assert result.returncode == 0, result.stderr.decode("utf-8", "replace") + assert guard_attempts == [] + assert [span["name"] for span in spans] == ["other_library_span"] + for request in requests: + assert request["headers"].get("x-api-key") == FI_API_KEY + assert request["headers"].get("x-secret-key") == FI_SECRET_KEY + + @pytest.mark.parametrize( ("unset", "message"), [ @@ -843,11 +931,13 @@ def projects(requests: list[dict[str, Any]]) -> set[str]: # -------------------------------------------------------------------------- -def _readme_section(title: str) -> str: +def _readme_section(title: str, level: int = 2) -> str: + """The README text from a heading of this level to the next of the same or higher.""" text = README.read_text(encoding="utf-8") - start = text.index("\n## {0}\n".format(title)) - end = text.find("\n## ", start + 1) - return text[start : end if end != -1 else len(text)] + start = text.index("\n{0} {1}\n".format("#" * level, title)) + ends = [text.find("\n{0} ".format("#" * n), start + 1) for n in range(2, level + 1)] + ends = [end for end in ends if end != -1] + return text[start : min(ends) if ends else len(text)] def test_readme_key_inventory_matches_the_emitted_keys() -> None: @@ -874,6 +964,18 @@ def test_readme_privacy_names_what_content_off_still_exports( assert missing == [], "not listed in README Privacy: {0}".format(missing) +def test_readme_names_the_environment_that_overrides_the_recipe() -> None: + """The variables the tests above show replacing or widening the recipe's settings.""" + section = _readme_section("Environment that overrides the recipe", level=3) + for name in ( + "OTEL_EXPORTER_OTLP_TRACES_HEADERS", + "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", + "OTEL_TRACES_EXPORTER", + "OTEL_EXPORTER_OTLP_HEADERS", + ): + assert "`{0}`".format(name) in section, name + + def _code_blocks(section: str, language: str) -> list[str]: return re.findall(r"```{0}\n(.*?)```".format(language), section, flags=re.DOTALL) From a8d302349860257b2663f62a8968384c394cbc95 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Mon, 5 Oct 2026 10:28:42 -0700 Subject: [PATCH 15/16] docs(examples): describe the new OpenLIT recipe fixtures and checks The Tests section now names the fake's tool-call and error replies, lists every fixture script (tool_call, failed_call, register_first, second_exporter), lists the new checks, and says a full run starts the app about 30 times. approved: Nikhil 2026-10-03 blanket Refs: TH-8338 Co-Authored-By: Claude Opus 5.5 --- python/examples/openlit/README.md | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/python/examples/openlit/README.md b/python/examples/openlit/README.md index a5aa6b60d..90dafac7b 100644 --- a/python/examples/openlit/README.md +++ b/python/examples/openlit/README.md @@ -516,22 +516,32 @@ Read from source; no Firecrawl call was made. `tests/test_openlit_recipe.py` runs `src/app.py` as written, in a subprocess, with the real `openlit` 1.45.0 instrumenting the real `openai` client. `tests/_fake_openai.py` is a loopback fake of the Chat Completions -API; the spans go to the shared harness `Receiver` (`python/tests/harness`), -which serves `/v1/traces` and `/tracer/v1/traces` on 127.0.0.1. +API that answers with text, a tool call or an error; the spans go to the +shared harness `Receiver` (`python/tests/harness`), which serves +`/v1/traces` and `/tracer/v1/traces` on 127.0.0.1. `tests/_guarded_run.py` blocks and logs every non-loopback connection; a positive control (`tests/_guard_probe.py`) proves it refuses and logs IPv4, IPv6 and DNS attempts. All keys are placeholders. `tests/recipe_variant.py` runs `src/app.py` with one `openlit.init()` -argument changed or removed, or with traceai-openai also enabled. -`tests/traced_function.py` wraps the recipe's model call in -`@openlit.trace`. Both are fixtures, not part of the recipe. +argument changed or removed, or with traceai-openai also enabled. The other +scripts run the recipe's own `init_tracing()` first: +`tests/traced_function.py` wraps the model call in `@openlit.trace`, +`tests/tool_call.py` makes a call with `tools` and `user`, +`tests/failed_call.py` makes the call with `max_retries=0` against an error, +`tests/register_first.py` calls traceAI's `register()` before it, and +`tests/second_exporter.py` builds another OTLP exporter after it. All are +fixtures, not part of the recipe. The tests check the request path for each endpoint form, the auth headers, the resource, the span tree, the exact key inventory and its values, -content off and on, the environment override, the metrics and events paths, -the price download, the decorator, a 401 reply, the app's start-up checks, -both instrumentors together, and that no key reaches the export. +content off and on, the content environment override, the metrics and +events paths, the price download, what content off still exports (the +decorator, tool calls, `user`, a failed call), a 401 reply, the environment +that overrides the recipe, the app's start-up checks, both instrumentors +together, `register()` before `openlit.init()`, that no key reaches the +export, and that the README's `openlit.init()` call and "Run" block match +`src/app.py`. From the repository root, Python 3.11: @@ -568,4 +578,4 @@ export FI_COLLECTOR_SRC=/fi-collector # then run the comm Without it the test is skipped. No CI job runs this example, so run it whenever fi-collector's alias lists or this recipe change; otherwise the table can drift unnoticed. A full run starts the app in a new process about -20 times and takes under a minute. +30 times and takes under a minute. From 5d94039598915273d35b3da98045a432adf3d089 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Fri, 16 Oct 2026 20:09:49 -0700 Subject: [PATCH 16/16] docs(examples): record live Future AGI check for the OpenLIT recipe (TH-8338) Co-Authored-By: Claude Sonnet 5.5 --- python/examples/openlit/README.md | 38 +++++++++++++++++++++++++++---- 1 file changed, 34 insertions(+), 4 deletions(-) diff --git a/python/examples/openlit/README.md b/python/examples/openlit/README.md index 90dafac7b..a00eeb6c3 100644 --- a/python/examples/openlit/README.md +++ b/python/examples/openlit/README.md @@ -319,7 +319,8 @@ recipe calls "displayed". It was read from fi-collector at future-agi `main` `exporter/clickhouse25exporter/converter.go`); the opt-in test (see "Tests") re-reads from that source the alias lists and the keys `input` and `output` are read from, and checks this table against them and against the -keys the run emitted. No fi-collector was run. +keys the run emitted. The contract test runs no fi-collector; "Live check" below +records a run against a local Future AGI stack. | Column | Filled from | Source | |---|---|---| @@ -507,9 +508,38 @@ Read from source; no Firecrawl call was made. `Failed to export spans batch code: 401, reason: Unauthorized` to stderr and the app carries on and exits 0; nothing raises in your code (tested against a recorder that answers 401). -- No fi-collector, OpenLIT host, GitHub or OpenAI was called. Auth, project - stamping and storage are fi-collector behaviour the harness receiver does - not reproduce. +- The contract test calls no fi-collector, OpenLIT host, GitHub or OpenAI. + Auth, project stamping and storage are fi-collector behaviour the harness + receiver does not reproduce; see "Live check" for a run that did. + +## Live check + +Run on 2026-10-16 against a local Future AGI stack (fi-collector on `:14338`, +ClickHouse spans table, trace UI), with the recipe's `src/app.py` +`init_tracing()` unchanged, `openlit` 1.45.0 (the latest release on PyPI that +day), `openai` 2.54.0 (the latest 2.x; `openlit` requires `<3`), Python 3.12. +One agent run made two chat calls with a tool in between (the tool function +wrapped in `@openlit.trace`), plus one call to an unknown model. + +- The export was accepted with the origin as `OTEL_EXPORTER_OTLP_ENDPOINT` + and both percent-encoded key headers. The project came from + `OTEL_RESOURCE_ATTRIBUTES`. +- Stored tree: `agent` > two `chat ` spans (each with a child `POST`) and + `lookup_order`. The chat spans had `observation_type=llm`, `model`, `provider`, + and prompt, completion and total tokens. The `agent`, `lookup_order` and + `POST` spans were `unknown` kind, as listed under "What Future AGI shows". +- The failed call was stored with status `ERROR`; its status message holds + the provider's error body (see "Privacy"). The stored `span_events` kept only + the two `exception` event names and timestamps, with no `exception.message` + or `exception.stacktrace` attributes. The status message, tool arguments and + `user` were stored as the "Privacy" section says. +- With `capture_message_content=True`, the prompt and answer were stored only + in `attrs_string` (`gen_ai.input.messages`, `gen_ai.output.messages`); the + span's `input` and `output` columns stayed empty, so the trace UI shows no + input or output for the model-call span. +- `gen_ai.usage.cost` (0 with the recipe's empty price table) was not in the stored + attributes of this local stack, and the `cost` column was 0. The key + inventory table above records what OpenLIT sent, not what this stack kept. ## Tests