From 5936e0960f01a23e842d3df90b7c101874ed8149 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Sat, 3 Oct 2026 18:58:31 -0700 Subject: [PATCH 01/14] test(harness): add shared OTLP harness for TH-8103 contract tests --- python/tests/harness/__init__.py | 184 +++++++++++++++++++++++++++ python/tests/harness/test_harness.py | 101 +++++++++++++++ 2 files changed, 285 insertions(+) create mode 100644 python/tests/harness/__init__.py create mode 100644 python/tests/harness/test_harness.py diff --git a/python/tests/harness/__init__.py b/python/tests/harness/__init__.py new file mode 100644 index 000000000..dfd9198e4 --- /dev/null +++ b/python/tests/harness/__init__.py @@ -0,0 +1,184 @@ +"""Small, loopback-only utilities for OTLP contract tests.""" + +from __future__ import annotations + +import copy +import difflib +import json +import subprocess +import threading +from http import HTTPStatus +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from typing import Any, Mapping, Optional, Sequence, Union +from urllib.parse import urlsplit +from urllib.request import Request, urlopen + + +class Receiver: + """An in-memory OTLP/HTTP receiver for tests. + + The receiver only listens on loopback and never forwards received spans. + """ + + def __init__(self) -> None: + self._spans: list[dict[str, Any]] = [] + self._lock = threading.Lock() + owner = self + + class Handler(BaseHTTPRequestHandler): + def do_POST(self) -> None: # noqa: N802 - required by BaseHTTPRequestHandler + if urlsplit(self.path).path != "/v1/traces": + self.send_error(HTTPStatus.NOT_FOUND) + return + + try: + length = int(self.headers.get("Content-Length", "0")) + body = self.rfile.read(length) + content_type = self.headers.get("Content-Type", "").lower() + + if content_type.startswith("application/json"): + request = json.loads(body.decode("utf-8")) + elif content_type.startswith(("application/x-protobuf", "application/protobuf")): + from google.protobuf.json_format import MessageToDict + from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ( + ExportTraceServiceRequest, + ) + + protobuf_request = ExportTraceServiceRequest() + protobuf_request.ParseFromString(body) + request = MessageToDict(protobuf_request) + else: + self.send_error(HTTPStatus.UNSUPPORTED_MEDIA_TYPE) + return + + spans: list[dict[str, Any]] = [] + for resource_spans in request.get( + "resourceSpans", request.get("resource_spans", []) + ): + for scope_spans in resource_spans.get( + "scopeSpans", resource_spans.get("scope_spans", []) + ): + spans.extend(scope_spans.get("spans", [])) + except (ImportError, TypeError, UnicodeDecodeError, ValueError) as error: + self.send_error(HTTPStatus.BAD_REQUEST, str(error)) + return + + with owner._lock: + owner._spans.extend(copy.deepcopy(spans)) + self.send_response(HTTPStatus.OK) + self.send_header("Content-Length", "0") + self.end_headers() + + def log_message(self, _format: str, *args: Any) -> None: + return + + self._server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + self.origin = "http://127.0.0.1:{0}".format(self._server.server_port) + self.endpoint = "{0}/v1/traces".format(self.origin) + self._thread = threading.Thread(target=self._server.serve_forever, daemon=True) + self._thread.start() + + def __enter__(self) -> "Receiver": + return self + + def __exit__(self, _type: Any, _value: Any, _traceback: Any) -> None: + self._server.shutdown() + self._server.server_close() + self._thread.join() + + def spans(self) -> list[dict[str, Any]]: + """Return a copy of the decoded spans received so far.""" + with self._lock: + return copy.deepcopy(self._spans) + + def clear(self) -> None: + """Remove every decoded span received so far.""" + with self._lock: + self._spans.clear() + + +def run( + argv: Sequence[str], + env: Mapping[str, str], + stdin: Optional[bytes], + timeout: float, +) -> subprocess.CompletedProcess[bytes]: + """Run one process and return its captured output, including after a timeout.""" + process = subprocess.Popen( + argv, + env=dict(env), + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + timed_out = False + try: + stdout, stderr = process.communicate(input=stdin, timeout=timeout) + except subprocess.TimeoutExpired: + timed_out = True + process.kill() + stdout, stderr = process.communicate() + + result = subprocess.CompletedProcess(argv, process.returncode, stdout, stderr) + result.timed_out = timed_out + return result + + +def post_otlp(spans: dict[str, Any], endpoint: str) -> int: + """Post one JSON-encoded OTLP request to a loopback receiver.""" + parsed = urlsplit(endpoint) + if parsed.scheme != "http" or parsed.hostname != "127.0.0.1": + raise ValueError("post_otlp only sends to a loopback HTTP endpoint") + + request = Request( + endpoint, + data=json.dumps(spans).encode("utf-8"), + headers={"Content-Type": "application/json"}, + method="POST", + ) + with urlopen(request, timeout=5) as response: + return response.status + + +def compare(actual: Sequence[dict[str, Any]], golden_path: Union[str, Path]) -> None: + """Fail with a diff unless span names, attributes, and statuses match a golden.""" + with Path(golden_path).open(encoding="utf-8") as golden_file: + expected = json.load(golden_file) + + def canonical(spans: Sequence[dict[str, Any]]) -> list[dict[str, Any]]: + normalized = [] + for span in spans: + attributes = span.get("attributes", []) + if isinstance(attributes, dict): + attributes = [ + {"key": key, "value": value} for key, value in attributes.items() + ] + normalized.append( + { + "name": span.get("name", ""), + "attributes": sorted( + attributes, + key=lambda attribute: json.dumps( + attribute, sort_keys=True, separators=(",", ":") + ), + ), + "status": span.get("status", {}), + } + ) + return normalized + + actual_normalized = canonical(actual) + expected_normalized = canonical(expected) + if actual_normalized != expected_normalized: + diff = "\n".join( + difflib.unified_diff( + json.dumps(expected_normalized, indent=2, sort_keys=True).splitlines(), + json.dumps(actual_normalized, indent=2, sort_keys=True).splitlines(), + fromfile=str(golden_path), + tofile="actual", + lineterm="", + ) + ) + print(diff) + raise AssertionError("spans do not match golden") diff --git a/python/tests/harness/test_harness.py b/python/tests/harness/test_harness.py new file mode 100644 index 000000000..4e76ceabe --- /dev/null +++ b/python/tests/harness/test_harness.py @@ -0,0 +1,101 @@ +"""Contract tests for the shared loopback-only harness.""" + +import json +import sys + +import pytest + +from tests.harness import Receiver, compare, post_otlp, run + + +def test_receiver_stores_posted_span_and_can_clear() -> None: + payload = { + "resourceSpans": [ + { + "scopeSpans": [ + { + "spans": [ + { + "traceId": "trace-id", + "spanId": "span-id", + "name": "harness.receiver", + "attributes": [ + {"key": "component", "value": {"stringValue": "test"}} + ], + "status": {"code": 1}, + } + ] + } + ] + } + ] + } + + with Receiver() as receiver: + assert post_otlp(payload, receiver.endpoint) == 200 + assert receiver.spans() == payload["resourceSpans"][0]["scopeSpans"][0]["spans"] + receiver.clear() + assert receiver.spans() == [] + + +def test_run_captures_output_and_kills_timed_out_process() -> None: + completed = run( + [ + sys.executable, + "-c", + "import sys; print('stdout'); sys.stderr.write('stderr\\n')", + ], + env={}, + stdin=b"", + timeout=1, + ) + assert completed.returncode == 0 + assert completed.stdout == b"stdout\n" + assert completed.stderr == b"stderr\n" + assert completed.timed_out is False + + timed_out = run( + [sys.executable, "-c", "import time; time.sleep(30)"], + env={}, + stdin=b"", + timeout=0.1, + ) + assert timed_out.timed_out is True + assert timed_out.returncode != 0 + + +def test_compare_accepts_normalized_match_and_prints_mismatch_diff(tmp_path, capsys) -> None: + golden = [ + { + "traceId": "golden-trace", + "spanId": "golden-span", + "startTimeUnixNano": "1", + "endTimeUnixNano": "2", + "name": "harness.compare", + "attributes": [ + {"key": "alpha", "value": {"stringValue": "a"}}, + {"key": "beta", "value": {"intValue": "2"}}, + ], + "status": {"code": 1}, + } + ] + golden_path = tmp_path / "spans.golden.json" + golden_path.write_text(json.dumps(golden), encoding="utf-8") + + actual = [ + { + "traceId": "other-trace", + "spanId": "other-span", + "startTimeUnixNano": "3", + "endTimeUnixNano": "4", + "name": "harness.compare", + "attributes": list(reversed(golden[0]["attributes"])), + "status": {"code": 1}, + } + ] + compare(actual, golden_path) + + actual[0]["name"] = "harness.mismatch" + with pytest.raises(AssertionError, match="spans do not match golden"): + compare(actual, golden_path) + assert "---" in capsys.readouterr().out From 32f5c9d2b3b80f97a70e00b3f3ca1b408b14dc5c Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Sat, 3 Oct 2026 23:21:29 -0700 Subject: [PATCH 02/14] test(harness): accept the collector /tracer/v1/traces path --- python/tests/harness/__init__.py | 3 ++- python/tests/harness/test_harness.py | 12 ++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/python/tests/harness/__init__.py b/python/tests/harness/__init__.py index dfd9198e4..cb510ad10 100644 --- a/python/tests/harness/__init__.py +++ b/python/tests/harness/__init__.py @@ -28,7 +28,7 @@ def __init__(self) -> None: class Handler(BaseHTTPRequestHandler): def do_POST(self) -> None: # noqa: N802 - required by BaseHTTPRequestHandler - if urlsplit(self.path).path != "/v1/traces": + if urlsplit(self.path).path not in ("/v1/traces", "/tracer/v1/traces"): self.send_error(HTTPStatus.NOT_FOUND) return @@ -76,6 +76,7 @@ def log_message(self, _format: str, *args: Any) -> None: self._server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) self.origin = "http://127.0.0.1:{0}".format(self._server.server_port) self.endpoint = "{0}/v1/traces".format(self.origin) + self.collector_endpoint = "{0}/tracer/v1/traces".format(self.origin) self._thread = threading.Thread(target=self._server.serve_forever, daemon=True) self._thread.start() diff --git a/python/tests/harness/test_harness.py b/python/tests/harness/test_harness.py index 4e76ceabe..6a32c2af8 100644 --- a/python/tests/harness/test_harness.py +++ b/python/tests/harness/test_harness.py @@ -38,6 +38,18 @@ def test_receiver_stores_posted_span_and_can_clear() -> None: assert receiver.spans() == [] +def test_receiver_accepts_the_collector_path() -> None: + payload = { + "resourceSpans": [ + {"scopeSpans": [{"spans": [{"name": "collector.path"}]}]} + ] + } + + with Receiver() as receiver: + assert post_otlp(payload, receiver.collector_endpoint) == 200 + assert receiver.spans() == [{"name": "collector.path"}] + + def test_run_captures_output_and_kills_timed_out_process() -> None: completed = run( [ From 3eaadc8bdbf428a472db6779091bbce3d6d89015 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Sun, 4 Oct 2026 15:59:06 -0700 Subject: [PATCH 03/14] fix(harness): decode chunked OTLP bodies and record request metadata The Node OTLP exporter streams with Transfer-Encoding: chunked and sends no Content-Length. The Receiver read Content-Length only, so every Node export came back 400 with zero spans; two TH-8103 children had to put a de-chunking relay in front of it. Receiver now accepts both framings. It also records one entry per accepted export (path, lower-cased headers, flattened resource attributes) via requests(), so a contract test can assert X-Api-Key/X-Secret-Key and project_name through the shared harness instead of a private recorder. Verified: 6 harness tests pass, and the TanStack example's real Node exporter delivered 4 chunked exports (4 spans, collector path, both auth headers, project_name/project_type) with no relay. Refs: TH-8339, TH-8103 --- python/tests/harness/__init__.py | 61 ++++++++++++++++++++-- python/tests/harness/test_harness.py | 76 ++++++++++++++++++++++++++++ 2 files changed, 133 insertions(+), 4 deletions(-) diff --git a/python/tests/harness/__init__.py b/python/tests/harness/__init__.py index cb510ad10..cf79047f1 100644 --- a/python/tests/harness/__init__.py +++ b/python/tests/harness/__init__.py @@ -23,18 +23,36 @@ class Receiver: def __init__(self) -> None: self._spans: list[dict[str, Any]] = [] + self._requests: list[dict[str, Any]] = [] self._lock = threading.Lock() owner = self class Handler(BaseHTTPRequestHandler): + def _read_body(self) -> bytes: + # The Node OTLP exporter streams with Transfer-Encoding: chunked + # and sends no Content-Length, so both framings are accepted. + if "chunked" in self.headers.get("Transfer-Encoding", "").lower(): + chunks = [] + while True: + size_line = self.rfile.readline().split(b";", 1)[0].strip() + size = int(size_line, 16) + if size == 0: + while self.rfile.readline() not in (b"\r\n", b"\n", b""): + pass + return b"".join(chunks) + chunks.append(self.rfile.read(size)) + self.rfile.readline() + length = int(self.headers.get("Content-Length", "0")) + return self.rfile.read(length) + def do_POST(self) -> None: # noqa: N802 - required by BaseHTTPRequestHandler - if urlsplit(self.path).path not in ("/v1/traces", "/tracer/v1/traces"): + path = urlsplit(self.path).path + if path not in ("/v1/traces", "/tracer/v1/traces"): self.send_error(HTTPStatus.NOT_FOUND) return try: - length = int(self.headers.get("Content-Length", "0")) - body = self.rfile.read(length) + body = self._read_body() content_type = self.headers.get("Content-Type", "").lower() if content_type.startswith("application/json"): @@ -53,9 +71,15 @@ def do_POST(self) -> None: # noqa: N802 - required by BaseHTTPRequestHandler return spans: list[dict[str, Any]] = [] + resource_attributes: list[dict[str, Any]] = [] for resource_spans in request.get( "resourceSpans", request.get("resource_spans", []) ): + resource_attributes.append( + _flatten_attributes( + resource_spans.get("resource", {}).get("attributes", []) + ) + ) for scope_spans in resource_spans.get( "scopeSpans", resource_spans.get("scope_spans", []) ): @@ -66,6 +90,15 @@ def do_POST(self) -> None: # noqa: N802 - required by BaseHTTPRequestHandler with owner._lock: owner._spans.extend(copy.deepcopy(spans)) + owner._requests.append( + { + "path": path, + "headers": { + key.lower(): value for key, value in self.headers.items() + }, + "resource_attributes": resource_attributes, + } + ) self.send_response(HTTPStatus.OK) self.send_header("Content-Length", "0") self.end_headers() @@ -93,10 +126,30 @@ def spans(self) -> list[dict[str, Any]]: with self._lock: return copy.deepcopy(self._spans) + def requests(self) -> list[dict[str, Any]]: + """Return one record per accepted export: path, lower-cased headers, resource attributes.""" + with self._lock: + return copy.deepcopy(self._requests) + def clear(self) -> None: - """Remove every decoded span received so far.""" + """Remove every decoded span and request record received so far.""" with self._lock: self._spans.clear() + self._requests.clear() + + +def _flatten_attributes(attributes: Sequence[Mapping[str, Any]]) -> dict[str, Any]: + """Turn OTLP key/value attributes into a plain dict of scalar values.""" + flat: dict[str, Any] = {} + for attribute in attributes: + value = attribute.get("value", {}) + for kind in ("stringValue", "boolValue", "intValue", "doubleValue"): + if kind in value: + flat[attribute.get("key", "")] = value[kind] + break + else: + flat[attribute.get("key", "")] = value + return flat def run( diff --git a/python/tests/harness/test_harness.py b/python/tests/harness/test_harness.py index 6a32c2af8..bd61d5adc 100644 --- a/python/tests/harness/test_harness.py +++ b/python/tests/harness/test_harness.py @@ -50,6 +50,82 @@ def test_receiver_accepts_the_collector_path() -> None: assert receiver.spans() == [{"name": "collector.path"}] +def _post_chunked(url: str, body: bytes, headers: dict) -> int: + """Send a body with Transfer-Encoding: chunked, as the Node OTLP exporter does.""" + import http.client + from urllib.parse import urlsplit + + parts = urlsplit(url) + connection = http.client.HTTPConnection(parts.hostname, parts.port, timeout=5) + try: + chunks = [body[: len(body) // 2], body[len(body) // 2 :]] + connection.request( + "POST", + parts.path, + body=iter(chunks), + headers={**headers, "Transfer-Encoding": "chunked"}, + encode_chunked=True, + ) + return connection.getresponse().status + finally: + connection.close() + + +def test_receiver_decodes_a_chunked_body() -> None: + payload = { + "resourceSpans": [ + {"scopeSpans": [{"spans": [{"name": "chunked.body"}]}]} + ] + } + + with Receiver() as receiver: + status = _post_chunked( + receiver.collector_endpoint, + json.dumps(payload).encode("utf-8"), + {"Content-Type": "application/json"}, + ) + assert status == 200 + assert receiver.spans() == [{"name": "chunked.body"}] + + +def test_receiver_records_path_headers_and_resource_attributes() -> None: + payload = { + "resourceSpans": [ + { + "resource": { + "attributes": [ + {"key": "project_name", "value": {"stringValue": "harness"}}, + {"key": "project_type", "value": {"stringValue": "observe"}}, + ] + }, + "scopeSpans": [{"spans": [{"name": "with.resource"}]}], + } + ] + } + + with Receiver() as receiver: + status = _post_chunked( + receiver.collector_endpoint, + json.dumps(payload).encode("utf-8"), + { + "Content-Type": "application/json", + "X-Api-Key": "test-key", + "X-Secret-Key": "test-secret", + }, + ) + assert status == 200 + [request] = receiver.requests() + assert request["path"] == "/tracer/v1/traces" + assert request["headers"]["x-api-key"] == "test-key" + assert request["headers"]["x-secret-key"] == "test-secret" + assert "authorization" not in request["headers"] + assert request["resource_attributes"] == [ + {"project_name": "harness", "project_type": "observe"} + ] + receiver.clear() + assert receiver.requests() == [] + + def test_run_captures_output_and_kills_timed_out_process() -> None: completed = run( [ From 8a0b43c0a3bb2fcafd6448a3138e02592b68c08a Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Mon, 5 Oct 2026 09:14:58 -0700 Subject: [PATCH 04/14] test(examples): add a hand-built OpenInference OTLP fixture and golden Three spans in one trace, OTLP/JSON as fi-collector's HTTP handler decodes it (hex trace and span ids, int64 values as decimal strings): - retrieve: openinference.span.kind=RETRIEVER, input.value (the root) - ChatCompletion: openinference.span.kind=LLM, llm.model_name, llm.token_count.prompt, llm.token_count.completion (child of retrieve) - lookup_refund_policy: openinference.span.kind=TOOL (child of retrieve) The resource carries project_name, which fi-collector requires (pkg/auth/stamp.go:31-45), and project_type=observe. The keys are the ones openinference-semantic-conventions 0.1.41 defines and adapter.go / converter.go name; nothing imports openinference-instrumentation. openinference-spans.golden.json is the posted body's spans, verbatim, not a rendered trace. Co-Authored-By: Claude Opus 5.5 approved: Nikhil 2026-10-03 blanket Refs: TH-8336 --- .../fixtures/openinference-spans.golden.json | 44 +++++++++++++ .../fixtures/openinference-trace.otlp.json | 62 +++++++++++++++++++ 2 files changed, 106 insertions(+) create mode 100644 python/examples/openinference/tests/fixtures/openinference-spans.golden.json create mode 100644 python/examples/openinference/tests/fixtures/openinference-trace.otlp.json diff --git a/python/examples/openinference/tests/fixtures/openinference-spans.golden.json b/python/examples/openinference/tests/fixtures/openinference-spans.golden.json new file mode 100644 index 000000000..af92ba5c8 --- /dev/null +++ b/python/examples/openinference/tests/fixtures/openinference-spans.golden.json @@ -0,0 +1,44 @@ +[ + { + "traceId": "83360000000000000000000000000001", + "spanId": "8336000000000001", + "name": "retrieve", + "kind": 1, + "startTimeUnixNano": "1791115200000000000", + "endTimeUnixNano": "1791115201500000000", + "attributes": [ + {"key": "openinference.span.kind", "value": {"stringValue": "RETRIEVER"}}, + {"key": "input.value", "value": {"stringValue": "What is the refund window?"}} + ], + "status": {"code": 1} + }, + { + "traceId": "83360000000000000000000000000001", + "spanId": "8336000000000002", + "parentSpanId": "8336000000000001", + "name": "ChatCompletion", + "kind": 1, + "startTimeUnixNano": "1791115200300000000", + "endTimeUnixNano": "1791115201100000000", + "attributes": [ + {"key": "openinference.span.kind", "value": {"stringValue": "LLM"}}, + {"key": "llm.model_name", "value": {"stringValue": "gpt-4o-mini"}}, + {"key": "llm.token_count.prompt", "value": {"intValue": "120"}}, + {"key": "llm.token_count.completion", "value": {"intValue": "38"}} + ], + "status": {"code": 1} + }, + { + "traceId": "83360000000000000000000000000001", + "spanId": "8336000000000003", + "parentSpanId": "8336000000000001", + "name": "lookup_refund_policy", + "kind": 1, + "startTimeUnixNano": "1791115201200000000", + "endTimeUnixNano": "1791115201400000000", + "attributes": [ + {"key": "openinference.span.kind", "value": {"stringValue": "TOOL"}} + ], + "status": {"code": 1} + } +] diff --git a/python/examples/openinference/tests/fixtures/openinference-trace.otlp.json b/python/examples/openinference/tests/fixtures/openinference-trace.otlp.json new file mode 100644 index 000000000..3c35f3056 --- /dev/null +++ b/python/examples/openinference/tests/fixtures/openinference-trace.otlp.json @@ -0,0 +1,62 @@ +{ + "resourceSpans": [ + { + "resource": { + "attributes": [ + {"key": "service.name", "value": {"stringValue": "openinference-fixture"}}, + {"key": "project_name", "value": {"stringValue": "openinference-fixture-contract"}}, + {"key": "project_type", "value": {"stringValue": "observe"}} + ] + }, + "scopeSpans": [ + { + "scope": {"name": "openinference-fixture"}, + "spans": [ + { + "traceId": "83360000000000000000000000000001", + "spanId": "8336000000000001", + "name": "retrieve", + "kind": 1, + "startTimeUnixNano": "1791115200000000000", + "endTimeUnixNano": "1791115201500000000", + "attributes": [ + {"key": "openinference.span.kind", "value": {"stringValue": "RETRIEVER"}}, + {"key": "input.value", "value": {"stringValue": "What is the refund window?"}} + ], + "status": {"code": 1} + }, + { + "traceId": "83360000000000000000000000000001", + "spanId": "8336000000000002", + "parentSpanId": "8336000000000001", + "name": "ChatCompletion", + "kind": 1, + "startTimeUnixNano": "1791115200300000000", + "endTimeUnixNano": "1791115201100000000", + "attributes": [ + {"key": "openinference.span.kind", "value": {"stringValue": "LLM"}}, + {"key": "llm.model_name", "value": {"stringValue": "gpt-4o-mini"}}, + {"key": "llm.token_count.prompt", "value": {"intValue": "120"}}, + {"key": "llm.token_count.completion", "value": {"intValue": "38"}} + ], + "status": {"code": 1} + }, + { + "traceId": "83360000000000000000000000000001", + "spanId": "8336000000000003", + "parentSpanId": "8336000000000001", + "name": "lookup_refund_policy", + "kind": 1, + "startTimeUnixNano": "1791115201200000000", + "endTimeUnixNano": "1791115201400000000", + "attributes": [ + {"key": "openinference.span.kind", "value": {"stringValue": "TOOL"}} + ], + "status": {"code": 1} + } + ] + } + ] + } + ] +} From 3bfe3f5e18143dd8fbb8217d3d280d18c898e7be Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Mon, 5 Oct 2026 09:21:06 -0700 Subject: [PATCH 05/14] docs(examples): add the OpenInference fixture README States that the Django OTLP route is retired and is not proof (future-agi main 4af5338 otel_compat_urls.py:15-16: the only route is commented out), and what the fixture posts and why it is encoded as it is: hex ids, because fi-collector's pdata JSON decoder rejects base64 (server.go:438-442). Records, as source reading of fi-collector 4af5338, that the collector already reads every key the fixture sets: openinference.span.kind is the fourth kind key (converter.go:79-84, lower-cased and validated at :107-131), llm.model_name and llm.token_count.prompt/completion are the first aliases (adapter.go:279-305), and input.value is overflow (adapter.go:32-40) lifted into the input column (converter.go:343). Kind values outside the type list, including OpenInference's PROMPT and DECISION, are stored as unknown with no error path (converter.go:127-129, :407). Display is SF-1 and not claimed. Pins openinference-instrumentation 0.1.70 for the later round trip only: not installed, not imported, no license claimed. Tested Pythons are 3.11 and 3.13; 3.10 and 3.12 are stated as untested. Co-Authored-By: Claude Opus 5.5 approved: Nikhil 2026-10-03 blanket Refs: TH-8336 --- python/examples/openinference/README.md | 156 ++++++++++++++++++++++++ 1 file changed, 156 insertions(+) create mode 100644 python/examples/openinference/README.md diff --git a/python/examples/openinference/README.md b/python/examples/openinference/README.md new file mode 100644 index 000000000..a94a166af --- /dev/null +++ b/python/examples/openinference/README.md @@ -0,0 +1,156 @@ +# OpenInference → Future AGI (active-ingestion fixture) + +There is no traceAI package for OpenInference and this directory adds none. +It is a conformance fixture: one hand-built OTLP/JSON request with the +OpenInference keys fi-collector already reads, posted with the shared harness +and checked against a golden file. The backend's Python adapter +(`futureagi/tracer/utils/adapters/openinference.py`) is a semantic reference, +not the system under test, and nothing here imports it. + +Pin for the later round-trip step with a real instrumentor: +openinference-instrumentation 0.1.70 (`Requires-Python: <3.15,>=3.10`). It is +not installed here and the tests do not import it. No license is claimed for +it here. The fixture's keys are the ones its dependency, +`openinference-semantic-conventions` 0.1.41, defines. + +Alpha, fixture-validated. The tests pass on Python 3.11.12 and 3.13.15 (see +[Tests](#tests)); 3.10 and 3.12 were not tested. They post only to the +harness `Receiver` on 127.0.0.1: no fi-collector, no Arize account, no live +instrumentor. Sections marked "source reading" come from future-agi `main` +4af5338 and are not exercised by the always-on tests. + +## The Django route is retired and is not proof + +In future-agi `main` 4af5338, `futureagi/tracer/otel_compat_urls.py:15-16` +holds the old OTLP route only as a comment ("Migrated to fi-collector service +(June 2026)"), so its `urlpatterns` is empty. `tfc/urls.py:86` still mounts +the module under `api/public/otel/`, but it serves nothing. A response from +that path, or the existence of `openinference.py`, is not evidence that +OpenInference spans are ingested. This fixture does not uncomment the route, +post to it, or call the Python adapter. Ingestion is fi-collector's +`/v1/traces` and `/tracer/v1/traces` (`pkg/server/server.go:233-234`). + +## What is posted + +`tests/fixtures/openinference-trace.otlp.json`, three spans in one trace: + +| Span | Parent | Attributes | +|---|---|---| +| `retrieve` | none | `openinference.span.kind=RETRIEVER`, `input.value` (the query) | +| `ChatCompletion` | `retrieve` | `openinference.span.kind=LLM`, `llm.model_name=gpt-4o-mini`, `llm.token_count.prompt=120`, `llm.token_count.completion=38` | +| `lookup_refund_policy` | `retrieve` | `openinference.span.kind=TOOL` | + +The resource carries `service.name`, `project_name` and +`project_type=observe`. fi-collector fails a batch whose resource has no +`project_name` with HTTP 400 (`pkg/auth/stamp.go:31-45`, `server.go:461-464`). + +Kind values are posted exactly as OpenInference sets them, in upper case. The +fixture sets no other key fi-collector reads a kind from (`fi.span.kind`, +`gen_ai.span.kind`, `llm.request.type`, `gen_ai.operation.name`), so the kind +can come only from `openinference.span.kind`. + +The body is OTLP/JSON as fi-collector decodes it (pdata `UnmarshalJSON`, +`server.go:438-442`): hex trace and span ids, int64 values as decimal +strings, enums as integers. Base64 ids, which protobuf's `MessageToDict` (and +so the harness `Receiver`, for a protobuf export) produces, fail that decoder, +and the handler answers 400 (`server.go:440`). + +`tests/fixtures/openinference-spans.golden.json` is the posted body's spans, +verbatim. It is the golden for `compare()`, not a rendered trace. + +## What fi-collector does with it (source reading) + +| Fixture key | fi-collector 4af5338 | Stored as | +|---|---|---| +| `openinference.span.kind` | fourth kind key (`exporter/clickhouse25exporter/converter.go:79-84`); lower-cased and checked against the type list (`:107-131`) | `observation_type` `retriever`, `llm`, `tool` | +| `llm.model_name` | first model alias (`pkg/adapter/adapter.go:279-284`) | `model` | +| `llm.token_count.prompt`, `llm.token_count.completion` | first input and output token alias (`adapter.go:296-305`) | `prompt_tokens`, `completion_tokens`; `total_tokens` is their sum when no total key is set (`adapter.go:230-235`) | +| `input.value` | overflow prefix (`adapter.go:32-40`), also lifted into the `input` column (`converter.go:343`) | `attributes_extra` and `input` | + +So the collector's aliases cover kind, model and tokens for these keys, and +the retriever query stays readable without a new column. How the trace view +renders `observation_type` is shared processor work (SF-1), not this fixture. +OpenInference instrumentors also set `llm.provider` and `llm.system`; both are +provider aliases (`adapter.go:289-295`). The fixture sets neither. + +Unknown kinds: anything not in the collector's type list +(`converter.go:64-68`) after lower-casing is stored as `unknown` +(`:127-129`). That function has no error path and `spanToRow` returns no error +(`:407`), so a kind value cannot make the export fail. Of the 12 kinds in +`openinference-semantic-conventions` 0.1.41, `PROMPT` and `DECISION` are not +in that list, and `UNKNOWN` and an empty kind are stored as `unknown` too. None +of the handler's 4xx answers (method, read, size, content type, decode, usage +limit, missing `project_name`, scope, convert) depends on the value of a span +attribute (`server.go:402-482`). + +These rows were also checked once, offline and outside this repository's +tests, by running fi-collector 4af5338's own decoder and +`ConvertWithIdentities` on the fixture and on one-span variants with kinds +`PROMPT`, `DECISION`, `UNKNOWN`, `not-a-kind` and an empty kind. That run +started no server and had no auth, pricer or ClickHouse. + +## What this fixture does not do + +- Add a package, a private adapter, or a Go port of the Python adapter. +- Edit `openinference.py` or `otel_compat_urls.py`, reorder alias priority, + or add a column for `input.value`. +- Reproduce fi-collector auth (401 without keys), project stamping, storage + or the trace view. Those need a collector, which the harness does not start. + +## Tests + +`tests/test_openinference_fixture.py` posts the fixture with the harness +`post_otlp()` to the shared `Receiver` (`python/tests/harness`), which serves +`/v1/traces` and `/tracer/v1/traces` on 127.0.0.1 but does not authenticate, +resolve kinds or store anything, and checks the decoded spans with +`compare()` against the golden. So these tests assert what is posted, not +what is stored. They check: + +- the golden match (names, attributes, statuses, and ids, parentage and + times), with a control showing `compare()` fails when a kind value's case + changes; +- one trace with `retrieve` as the parent of the other two spans; +- each kind string as posted, and no other kind key on any span; +- the model and token keys and their types, and the query in `input.value`; +- hex ids, and the resource's `project_name`; +- that kind values `PROMPT`, `DECISION`, an empty kind and `not-a-kind` post + without error and arrive unchanged. The `Receiver` never reads a kind, so + this checks only the body; the collector side is the source reading above; +- that `post_otlp()` refuses non-loopback endpoints before opening a + connection (the control for the only network path), that no + `openinference` or `tracer` module was imported, and that the fixture holds + no key material. The harness sends no auth header and no key is used. + +From the repository root: + +```bash +env -u PYTHONPATH PYTHONPATH="python:python/tests" \ + uv run --no-project --python 3.11 \ + --with 'pytest==9.1.1' --with 'opentelemetry-sdk==1.45.0' \ + --with 'opentelemetry-exporter-otlp-proto-http==1.45.0' \ + --with 'opentelemetry-instrumentation==0.66b0' \ + --with 'requests==2.34.2' --with 'jsonschema==4.26.0' \ + pytest python/examples/openinference/tests -q -p no:cacheprovider \ + --noconftest -o addopts= -rfEs +``` + +For Python 3.13, replace `--python 3.11`. The test itself needs only pytest. +pytest imports `python/__init__.py` (fi_instrumentation) for any test under +`python/`, which is why the other packages are listed. A run takes about a +second. + +Two opt-in tests read fi-collector's Go source instead of copying its tables. +They check that the kind, model, token, `input.value` and `project_name` +facts above still hold, and that the fixture's kinds are in the collector's +type list and the unknown ones are not. Point `FI_COLLECTOR_SRC` at a +fi-collector checkout: + +```bash +mkdir -p /tmp/fi-main +git -C archive origin/main fi-collector | tar -x -C /tmp/fi-main +export FI_COLLECTOR_SRC=/tmp/fi-main/fi-collector # then run the command above +``` + +Without it they are skipped. They read the source; they do not run the +collector. No CI job runs this directory, so run them whenever fi-collector's +kind or alias code changes. From 9a16ad14ed365a2fd9559fa59cfca10504858255 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Mon, 5 Oct 2026 09:21:19 -0700 Subject: [PATCH 06/14] test(examples): contract-test the OpenInference fixture with the harness Posts fixtures/openinference-trace.otlp.json with post_otlp() to the shared harness Receiver and checks the decoded spans with compare() against the golden, plus exact equality (ids, parentage, times). Every always-on assertion is about the posted body, since the Receiver does not authenticate, resolve kinds or store anything: - one export to /v1/traces with no key headers; the resource carries project_name and project_type=observe; - one trace, retrieve the parent of ChatCompletion and lookup_refund_policy; ids are hex as fi-collector's JSON decoder needs; - kind strings posted verbatim (RETRIEVER, LLM, TOOL) and no other key the collector reads a kind from; a control shows compare() fails when a kind value's case changes; - the LLM span's model and intValue token keys, the retriever's query in input.value, the tool span's kind only; - kinds PROMPT, DECISION, "" and not-a-kind post without error and arrive unchanged (the docstring says the Receiver never reads kinds); - post_otlp() refuses non-loopback endpoints before urlopen, with a check that the patched urlopen is on the real path; no openinference or tracer module is imported; the fixture holds no key material. Two opt-in tests (FI_COLLECTOR_SRC) read fi-collector's Go tables (spanKindAttrKeys, knownObservationTypes, spanKindSynonyms, the model and token alias lists, overflowKeyPrefixes) rather than copying them. They pass against 4af5338 and fail against copies with the kind key removed or "prompt" added to the type list. Co-Authored-By: Claude Opus 5.5 approved: Nikhil 2026-10-03 blanket Refs: TH-8336 --- .../tests/test_openinference_fixture.py | 340 ++++++++++++++++++ 1 file changed, 340 insertions(+) create mode 100644 python/examples/openinference/tests/test_openinference_fixture.py diff --git a/python/examples/openinference/tests/test_openinference_fixture.py b/python/examples/openinference/tests/test_openinference_fixture.py new file mode 100644 index 000000000..465512478 --- /dev/null +++ b/python/examples/openinference/tests/test_openinference_fixture.py @@ -0,0 +1,340 @@ +"""Contract test for the OpenInference active-ingestion fixture (TH-8336). + +Posts the hand-built OTLP/JSON body in ``fixtures/openinference-trace.otlp.json`` +with the shared harness ``post_otlp()`` and checks what the harness +``Receiver`` decoded with ``compare()`` against +``fixtures/openinference-spans.golden.json``. The Receiver serves +``/v1/traces`` and ``/tracer/v1/traces`` on 127.0.0.1 like fi-collector's +HTTP mux, but it does not authenticate, stamp projects, resolve span kinds or +store anything. Every always-on assertion here is therefore about the posted +body (what a collector receives), not about a stored row or a rendered trace. + +Nothing imports ``openinference-instrumentation`` or the backend's Python +adapter (``tracer.utils.adapters.openinference``). The harness sends no auth +header, so no key, real or placeholder, is involved. + +Two opt-in tests read fi-collector's Go source (``FI_COLLECTOR_SRC``) to check +that the collector reads the keys this fixture sets. They read the source; they +do not run the collector. +""" + +from __future__ import annotations + +import copy +import json +import os +import re +import sys +from pathlib import Path +from typing import Any + +import pytest + +import harness +from harness import Receiver, compare, post_otlp + +TESTS_DIR = Path(__file__).resolve().parent +RECIPE_DIR = TESTS_DIR.parent +README = RECIPE_DIR / "README.md" +BODY_PATH = TESTS_DIR / "fixtures" / "openinference-trace.otlp.json" +GOLDEN = TESTS_DIR / "fixtures" / "openinference-spans.golden.json" + +PROJECT = "openinference-fixture-contract" +SERVICE_NAME = "openinference-fixture" +MODEL = "gpt-4o-mini" +PROMPT_TOKENS = "120" # OTLP/JSON encodes int64 as a decimal string +COMPLETION_TOKENS = "38" +QUERY = "What is the refund window?" + +RETRIEVER, LLM, TOOL = "retrieve", "ChatCompletion", "lookup_refund_policy" +# The exact strings posted; the data contract records them as set, not +# uppercased or lowercased by this test. +KINDS = {RETRIEVER: "RETRIEVER", LLM: "LLM", TOOL: "TOOL"} +KIND_KEY = "openinference.span.kind" +# The other keys fi-collector reads a span kind from, in its order +# (exporter/clickhouse25exporter/converter.go:79-90). The fixture sets none of +# them, so the collector can only take the kind from openinference.span.kind. +OTHER_KIND_KEYS = ("fi.span.kind", "gen_ai.span.kind", "llm.request.type", "gen_ai.operation.name") + +# Kind values fi-collector 4af5338 does not list as an observation type +# (converter.go:64-68). PROMPT and DECISION are real values in +# openinference-semantic-conventions 0.1.41; "" is an empty kind. +UNKNOWN_KINDS = ("PROMPT", "DECISION", "", "not-a-kind") + +HEX_TRACE_ID = re.compile(r"^[0-9a-f]{32}$") +HEX_SPAN_ID = re.compile(r"^[0-9a-f]{16}$") + + +def load_body() -> dict[str, Any]: + return json.loads(BODY_PATH.read_text(encoding="utf-8")) + + +def body_spans(body: dict[str, Any]) -> list[dict[str, Any]]: + (resource_spans,) = body["resourceSpans"] + (scope_spans,) = resource_spans["scopeSpans"] + return scope_spans["spans"] + + +def attributes(span: dict[str, Any]) -> dict[str, dict[str, Any]]: + """Attribute key -> OTLP AnyValue, typed value kept (e.g. ``{"intValue": "120"}``).""" + return {item["key"]: item["value"] for item in span.get("attributes", [])} + + +def by_name(spans: list[dict[str, Any]]) -> dict[str, dict[str, Any]]: + named = {span["name"]: span for span in spans} + assert len(named) == len(spans), sorted(span["name"] for span in spans) + return named + + +@pytest.fixture(scope="module") +def posted() -> dict[str, Any]: + """Post the fixture once; return the status and what the Receiver decoded.""" + with Receiver() as receiver: + status = post_otlp(load_body(), receiver.endpoint) + record = {"status": status, "spans": receiver.spans(), "requests": receiver.requests()} + # Every test below reads this record: fail here, not vacuously later. + assert record["status"] == 200 + assert record["requests"], "no export reached the receiver" + assert record["spans"], "the export carried no spans" + return record + + +# -------------------------------------------------------------------------- +# The posted body +# -------------------------------------------------------------------------- + + +def test_fixture_posts_once_to_v1_traces_without_keys(posted: dict[str, Any]) -> None: + (request,) = posted["requests"] + assert request["path"] == "/v1/traces" + assert request["headers"]["content-type"] == "application/json" + # The fixture is not authenticated: no key of any kind travels with it. + for header in ("x-api-key", "x-secret-key", "authorization"): + assert header not in request["headers"] + + +def test_posted_spans_match_the_golden(posted: dict[str, Any]) -> None: + compare(posted["spans"], GOLDEN) + # compare() checks names, attributes and statuses. The golden is the + # posted body, so ids, parentage, kinds and times must match too. + assert posted["spans"] == json.loads(GOLDEN.read_text(encoding="utf-8")) + assert posted["spans"] == body_spans(load_body()) + + +def test_golden_fails_when_a_kind_value_changes( + posted: dict[str, Any], capsys: pytest.CaptureFixture[str] +) -> None: + """Control: the golden pins the kind string exactly, case included.""" + spans = copy.deepcopy(posted["spans"]) + for span in spans: + for item in span["attributes"]: + if item["key"] == KIND_KEY: + item["value"] = {"stringValue": item["value"]["stringValue"].lower()} + with pytest.raises(AssertionError, match="spans do not match golden"): + compare(spans, GOLDEN) + assert '"stringValue": "llm"' in capsys.readouterr().out + + +def test_three_spans_in_one_trace_under_the_retriever(posted: dict[str, Any]) -> None: + spans = by_name(posted["spans"]) + assert set(spans) == set(KINDS) + assert len({span["traceId"] for span in spans.values()}) == 1 + assert len({span["spanId"] for span in spans.values()}) == 3 + root = spans[RETRIEVER] + assert not root.get("parentSpanId") + assert spans[LLM]["parentSpanId"] == root["spanId"] + assert spans[TOOL]["parentSpanId"] == root["spanId"] + + +def test_ids_are_hex_as_the_collector_json_decoder_needs(posted: dict[str, Any]) -> None: + """fi-collector decodes OTLP/JSON with pdata (pkg/server/server.go:439), + which reads trace and span ids as hex. Base64 ids are a decode error, + and the handler answers 400 (server.go:440).""" + for span in posted["spans"]: + assert HEX_TRACE_ID.match(span["traceId"]) and set(span["traceId"]) != {"0"} + assert HEX_SPAN_ID.match(span["spanId"]) and set(span["spanId"]) != {"0"} + if span.get("parentSpanId"): + assert HEX_SPAN_ID.match(span["parentSpanId"]) + + +def test_kind_values_are_posted_verbatim(posted: dict[str, Any]) -> None: + for name, span in by_name(posted["spans"]).items(): + attrs = attributes(span) + assert attrs[KIND_KEY] == {"stringValue": KINDS[name]}, name + assert not set(OTHER_KIND_KEYS) & set(attrs), name + + +def test_llm_span_carries_model_and_token_keys(posted: dict[str, Any]) -> None: + attrs = attributes(by_name(posted["spans"])[LLM]) + assert attrs == { + KIND_KEY: {"stringValue": "LLM"}, + "llm.model_name": {"stringValue": MODEL}, + # intValue, not stringValue: the collector puts ints in attrs_number + # (pkg/adapter/adapter.go:80-84). + "llm.token_count.prompt": {"intValue": PROMPT_TOKENS}, + "llm.token_count.completion": {"intValue": COMPLETION_TOKENS}, + } + + +def test_retriever_query_is_readable_in_input_value(posted: dict[str, Any]) -> None: + attrs = attributes(by_name(posted["spans"])[RETRIEVER]) + assert attrs == { + KIND_KEY: {"stringValue": "RETRIEVER"}, + "input.value": {"stringValue": QUERY}, + } + + +def test_tool_span_carries_only_the_kind(posted: dict[str, Any]) -> None: + assert attributes(by_name(posted["spans"])[TOOL]) == {KIND_KEY: {"stringValue": "TOOL"}} + + +def test_resource_carries_the_project(posted: dict[str, Any]) -> None: + """fi-collector fails a batch whose resource has no project_name + (pkg/auth/stamp.go:31-45; the handler answers 400, server.go:461-464).""" + (request,) = posted["requests"] + assert request["resource_attributes"] == [ + {"service.name": SERVICE_NAME, "project_name": PROJECT, "project_type": "observe"} + ] + + +@pytest.mark.parametrize("kind", UNKNOWN_KINDS) +def test_unknown_kind_value_is_posted_without_error(kind: str) -> None: + """The Receiver never reads a kind, so this shows only that the body + stays well-formed and carries the value unchanged. That fi-collector + stores such a span as ``unknown`` rather than rejecting it is read from + its source (converter.go:123-129; see the opt-in tests).""" + body = load_body() + tool = copy.deepcopy(by_name(body_spans(body))[TOOL]) + tool["attributes"] = [{"key": KIND_KEY, "value": {"stringValue": kind}}] + body["resourceSpans"][0]["scopeSpans"][0]["spans"] = [tool] + with Receiver() as receiver: + assert post_otlp(body, receiver.endpoint) == 200 + (span,) = receiver.spans() + assert attributes(span) == {KIND_KEY: {"stringValue": kind}} + + +def test_post_otlp_refuses_non_loopback_endpoints(monkeypatch: pytest.MonkeyPatch) -> None: + """Positive control for the only network path in this file: post_otlp() + refuses anything but http://127.0.0.1 before it opens a connection.""" + + def no_network(*_args: Any, **_kwargs: Any) -> None: + raise AssertionError("post_otlp opened a connection") + + monkeypatch.setattr(harness, "urlopen", no_network) + # The patch is on the real path: a loopback endpoint reaches it. + with pytest.raises(AssertionError, match="opened a connection"): + post_otlp(load_body(), "http://127.0.0.1:9/v1/traces") + for endpoint in ( + "http://192.0.2.1:4318/v1/traces", + "http://localhost:4318/v1/traces", + "https://127.0.0.1:4318/v1/traces", + ): + with pytest.raises(ValueError, match="loopback"): + post_otlp(load_body(), endpoint) + + +def test_nothing_imports_the_instrumentor_or_the_python_adapter(posted: dict[str, Any]) -> None: + assert posted["spans"] + loaded = [ + name + for name in sys.modules + if name in ("openinference", "tracer") or name.startswith(("openinference.", "tracer.")) + ] + assert loaded == [] + + +def test_fixture_contains_no_key_material() -> None: + text = BODY_PATH.read_text(encoding="utf-8") + GOLDEN.read_text(encoding="utf-8") + for pattern in (r"sk-", r"(?i)api[_-]?key", r"(?i)secret", r"(?i)bearer", r"(?i)password"): + assert not re.search(pattern, text), pattern + + +def test_readme_states_what_the_tests_check() -> None: + readme = README.read_text(encoding="utf-8") + for fact in ( + "openinference-instrumentation 0.1.70", + "otel_compat_urls.py", + "is not proof", + "post_otlp()", + "compare()", + BODY_PATH.name, + GOLDEN.name, + KIND_KEY, + "llm.model_name", + "llm.token_count.prompt", + "llm.token_count.completion", + "input.value", + "attributes_extra", + "project_name", + *UNKNOWN_KINDS[:2], + ): + assert fact in readme, fact + + +# -------------------------------------------------------------------------- +# Opt-in: read fi-collector's source instead of copying its tables. +# -------------------------------------------------------------------------- + + +def _collector_source() -> dict[str, str]: + root = os.environ.get("FI_COLLECTOR_SRC") + if not root: + pytest.skip("set FI_COLLECTOR_SRC to a fi-collector checkout") + files = { + "adapter": "pkg/adapter/adapter.go", + "converter": "exporter/clickhouse25exporter/converter.go", + "stamp": "pkg/auth/stamp.go", + } + return {name: (Path(root) / path).read_text(encoding="utf-8") for name, path in files.items()} + + +def _go_literal(source: str, name: str) -> str: + """The body of ``name = []string{...}`` or ``name = map[...]...{...}``, comments removed.""" + match = re.search( + r"\b" + re.escape(name) + r"\s*=\s*(?:\[\]string|map\[string\][\w{}]+?)\{\n(.*?)\n\s*\}\n", + source, + flags=re.DOTALL, + ) + assert match, name + return re.sub(r"//[^\n]*", "", match.group(1)) + + +def _go_strings(source: str, name: str) -> list[str]: + return re.findall(r'"([^"]*)"', _go_literal(source, name)) + + +def _go_map_keys(source: str, name: str) -> set[str]: + return set(re.findall(r'"([^"]*)"\s*:', _go_literal(source, name))) + + +def test_collector_reads_the_keys_the_fixture_sets() -> None: + source = _collector_source() + kind_keys = _go_strings(source["converter"], "spanKindAttrKeys") + assert KIND_KEY in kind_keys + operation_keys = _go_strings(source["converter"], "operationNameAttrKeys") + # Every key the collector would read a kind from, other than ours, is + # absent from the fixture, so the stored type comes from KIND_KEY alone. + assert set(kind_keys + operation_keys) - {KIND_KEY} == set(OTHER_KIND_KEYS) + # The fixture's model and token keys are the first alias of each list. + assert _go_strings(source["adapter"], "modelNameKeys")[0] == "llm.model_name" + assert _go_strings(source["adapter"], "inputTokenKeys")[0] == "llm.token_count.prompt" + assert _go_strings(source["adapter"], "outputTokenKeys")[0] == "llm.token_count.completion" + # input.value goes to the JSON overflow and is lifted into the input column. + assert "input.value" in _go_strings(source["adapter"], "overflowKeyPrefixes") + assert 'overflowAsString(overflow, "input.value")' in source["converter"] + assert 'getStrAttr(attrs, "project_name")' in source["stamp"] + + +def test_collector_type_list_has_the_fixture_kinds_but_not_the_unknown_ones() -> None: + """converter.go lowercases the raw kind, applies spanKindSynonyms and + stores ``unknown`` for anything not in knownObservationTypes. This reads + those two tables; it does not run resolveObservationType.""" + source = _collector_source() + assert "strings.ToLower(strings.TrimSpace(raw))" in source["converter"] + known = _go_map_keys(source["converter"], "knownObservationTypes") + synonyms = _go_map_keys(source["converter"], "spanKindSynonyms") + assert "unknown" in known + for kind in KINDS.values(): + assert kind.lower() in known, kind + for kind in UNKNOWN_KINDS: + assert kind.lower() not in known | synonyms, kind From 5be86d42c8e8146b88cc35b0ca7a516062daae56 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Mon, 5 Oct 2026 09:24:34 -0700 Subject: [PATCH 07/14] docs(examples): state the measured OpenInference test run time The four evidence runs on Python 3.11 and 3.13 took 2.7-3.0 s each, not about a second. Co-Authored-By: Claude Opus 5.5 approved: Nikhil 2026-10-03 blanket Refs: TH-8336 --- python/examples/openinference/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/examples/openinference/README.md b/python/examples/openinference/README.md index a94a166af..0a78f8972 100644 --- a/python/examples/openinference/README.md +++ b/python/examples/openinference/README.md @@ -136,8 +136,8 @@ env -u PYTHONPATH PYTHONPATH="python:python/tests" \ For Python 3.13, replace `--python 3.11`. The test itself needs only pytest. pytest imports `python/__init__.py` (fi_instrumentation) for any test under -`python/`, which is why the other packages are listed. A run takes about a -second. +`python/`, which is why the other packages are listed. A run takes a few +seconds. Two opt-in tests read fi-collector's Go source instead of copying its tables. They check that the kind, model, token, `input.value` and `project_name` From 7fd83eb7817a3bfa59ce5b298b942d19cf4bcdd8 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Mon, 5 Oct 2026 09:27:09 -0700 Subject: [PATCH 08/14] docs(examples): OpenInference fixture runs on Python 3.10-3.13 Coordinator exact-head run at 5be86d4 passed on 3.10, 3.11, 3.12 and 3.13 (18 passed + 2 skipped default; 20 passed with FI_COLLECTOR_SRC). approved: Nikhil 2026-10-03 blanket Refs: TH-8336 --- python/examples/openinference/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/examples/openinference/README.md b/python/examples/openinference/README.md index 0a78f8972..2f976de56 100644 --- a/python/examples/openinference/README.md +++ b/python/examples/openinference/README.md @@ -13,8 +13,8 @@ not installed here and the tests do not import it. No license is claimed for it here. The fixture's keys are the ones its dependency, `openinference-semantic-conventions` 0.1.41, defines. -Alpha, fixture-validated. The tests pass on Python 3.11.12 and 3.13.15 (see -[Tests](#tests)); 3.10 and 3.12 were not tested. They post only to the +Alpha, fixture-validated. The tests pass on Python 3.10, 3.11, 3.12 and 3.13 +(see [Tests](#tests)). They post only to the harness `Receiver` on 127.0.0.1: no fi-collector, no Arize account, no live instrumentor. Sections marked "source reading" come from future-agi `main` 4af5338 and are not exercised by the always-on tests. From ba6cc0881bc722775417dbb5f052f1b92fe01b85 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Mon, 5 Oct 2026 10:17:40 -0700 Subject: [PATCH 09/14] test(examples): scan the OpenInference example source for adapter imports The sys.modules check could not fail with the README command: neither openinference nor the backend's tracer package is installed. Add an ast scan of every .py file in the example and the harness for imports of either (import, from-import, import_module/__import__ of a literal), with a control, and keep the run-time check for environments that have them. The README says what each one guards. Co-Authored-By: Claude Opus 5.5 approved: Nikhil 2026-10-03 blanket Refs: TH-8336 --- python/examples/openinference/README.md | 11 ++-- .../tests/test_openinference_fixture.py | 50 ++++++++++++++++++- 2 files changed, 57 insertions(+), 4 deletions(-) diff --git a/python/examples/openinference/README.md b/python/examples/openinference/README.md index 2f976de56..9bf9fbf17 100644 --- a/python/examples/openinference/README.md +++ b/python/examples/openinference/README.md @@ -117,9 +117,14 @@ what is stored. They check: without error and arrive unchanged. The `Receiver` never reads a kind, so this checks only the body; the collector side is the source reading above; - that `post_otlp()` refuses non-loopback endpoints before opening a - connection (the control for the only network path), that no - `openinference` or `tracer` module was imported, and that the fixture holds - no key material. The harness sends no auth header and no key is used. + connection (the control for the only network path), and that the fixture + holds no key material. The harness sends no auth header and no key is used; +- that no `.py` file in this example, nor the harness, imports + `openinference` or `tracer`. This is read from the source with `ast`, with + a control, so it also catches an import that never runs. A second check + finds no such module loaded after a post. The command below installs + neither package, so only the source scan can fail here; the run-time check + is for an environment that has them. From the repository root: diff --git a/python/examples/openinference/tests/test_openinference_fixture.py b/python/examples/openinference/tests/test_openinference_fixture.py index 465512478..9f0f3bd8d 100644 --- a/python/examples/openinference/tests/test_openinference_fixture.py +++ b/python/examples/openinference/tests/test_openinference_fixture.py @@ -20,6 +20,7 @@ from __future__ import annotations +import ast import copy import json import os @@ -233,7 +234,54 @@ def no_network(*_args: Any, **_kwargs: Any) -> None: post_otlp(load_body(), endpoint) -def test_nothing_imports_the_instrumentor_or_the_python_adapter(posted: dict[str, Any]) -> None: +def _adapter_imports(source: str) -> list[str]: + """Modules under ``openinference`` or ``tracer`` that Python source imports, + read with ``ast``: ``import``, ``from ... import``, and + ``importlib.import_module()`` or ``__import__()`` of a string literal.""" + found = [] + for node in ast.walk(ast.parse(source)): + if isinstance(node, ast.Import): + names = [alias.name for alias in node.names] + elif isinstance(node, ast.ImportFrom) and node.level == 0: + names = [node.module or ""] + elif ( + isinstance(node, ast.Call) + and getattr(node.func, "id", getattr(node.func, "attr", None)) in ("import_module", "__import__") + and node.args + and isinstance(node.args[0], ast.Constant) + and isinstance(node.args[0].value, str) + ): + names = [node.args[0].value] + else: + continue + found += [name for name in names if name.split(".")[0] in ("openinference", "tracer")] + return found + + +def test_no_file_here_imports_the_instrumentor_or_the_python_adapter() -> None: + """Reads the source of every .py file in this example, and the harness. + This catches an import that never runs here: one inside a function or a + skipped test, or one of a package that is not installed.""" + # Control: the scan finds each import form it looks for, and only those. + assert _adapter_imports( + "import harness, opentelemetry\n" + "import openinference.instrumentation\n" + "from tracer.utils.adapters import openinference\n" + "def later():\n" + " importlib.import_module('openinference.semconv')\n" + ) == ["openinference.instrumentation", "tracer.utils.adapters", "openinference.semconv"] + files = [*sorted(RECIPE_DIR.rglob("*.py")), Path(harness.__file__).resolve()] + assert Path(__file__).resolve() in files + for path in files: + assert _adapter_imports(path.read_text(encoding="utf-8")) == [], path + + +def test_no_instrumentor_or_adapter_module_is_loaded(posted: dict[str, Any]) -> None: + """Run-time complement to the source scan above: no ``openinference`` or + ``tracer`` module is loaded after a post, including one imported + indirectly by a dependency. With the README command neither package is + installed, so this can fail only where one is (e.g. a round-trip run + with the instrumentor).""" assert posted["spans"] loaded = [ name From 46c6a31a8df1e039253750c75a78f17f822c9745 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Mon, 5 Oct 2026 10:17:53 -0700 Subject: [PATCH 10/14] test(examples): pin the fi-collector statements behind the OpenInference facts The opt-in source tests pinned key tables and literals, so edits that falsify README facts passed them (verifier V1: project_name fail-fast disabled, unknown kind returning "", observation_type not from the resolver, total_tokens derivation removed, kind key moved first). Read function bodies with comments removed and whitespace collapsed and match the statements that carry each fact: Split routing, DeriveHotKeys model/token/total, resolveObservationType's first-key, lower-case, synonym and unknown fallback, spanToRow's stored columns and single nil error return, the kind key order, and the project_name statements in handleHTTPTraces, StampResourceAttrs and ResolveProjectsForKey. Co-Authored-By: Claude Opus 5.5 approved: Nikhil 2026-10-03 blanket Refs: TH-8336 --- .../tests/test_openinference_fixture.py | 147 ++++++++++++++++-- 1 file changed, 131 insertions(+), 16 deletions(-) diff --git a/python/examples/openinference/tests/test_openinference_fixture.py b/python/examples/openinference/tests/test_openinference_fixture.py index 9f0f3bd8d..793755210 100644 --- a/python/examples/openinference/tests/test_openinference_fixture.py +++ b/python/examples/openinference/tests/test_openinference_fixture.py @@ -13,9 +13,9 @@ adapter (``tracer.utils.adapters.openinference``). The harness sends no auth header, so no key, real or placeholder, is involved. -Two opt-in tests read fi-collector's Go source (``FI_COLLECTOR_SRC``) to check -that the collector reads the keys this fixture sets. They read the source; they -do not run the collector. +Four opt-in tests read fi-collector's Go source (``FI_COLLECTOR_SRC``) and match +the tables and statements behind the README's source-reading facts. They read +the source; they do not run the collector. """ from __future__ import annotations @@ -332,10 +332,36 @@ def _collector_source() -> dict[str, str]: "adapter": "pkg/adapter/adapter.go", "converter": "exporter/clickhouse25exporter/converter.go", "stamp": "pkg/auth/stamp.go", + "auth": "pkg/auth/auth.go", + "server": "pkg/server/server.go", } return {name: (Path(root) / path).read_text(encoding="utf-8") for name, path in files.items()} +_GO_STRING_OR_COMMENT = re.compile(r'"(?:\\.|[^"\\\n])*"|`[^`]*`|//[^\n]*|/\*.*?\*/', flags=re.DOTALL) + + +def _go_code(source: str) -> str: + """Go source with comments removed and whitespace collapsed to one space, + so a commented-out or reflowed statement does not match as live code.""" + source = _GO_STRING_OR_COMMENT.sub(lambda m: " " if m.group().startswith("/") else m.group(), source) + return " ".join(source.split()) + + +def _go_func(source: str, name: str) -> str: + """The body of top-level function or method ``name``, as ``_go_code``. + gofmt puts a function's closing brace, and only that, in column 0.""" + match = re.search(r"^func (?:\([^)]*\) )?" + re.escape(name) + r"\(", source, flags=re.MULTILINE) + assert match, name + start = source.index(" {\n", match.end()) + 3 + return _go_code(source[start : source.index("\n}\n", start)]) + + +def _has_statements(code: str, *statements: str) -> None: + for statement in statements: + assert statement in code, statement + + def _go_literal(source: str, name: str) -> str: """The body of ``name = []string{...}`` or ``name = map[...]...{...}``, comments removed.""" match = re.search( @@ -358,27 +384,43 @@ def _go_map_keys(source: str, name: str) -> set[str]: def test_collector_reads_the_keys_the_fixture_sets() -> None: source = _collector_source() kind_keys = _go_strings(source["converter"], "spanKindAttrKeys") - assert KIND_KEY in kind_keys operation_keys = _go_strings(source["converter"], "operationNameAttrKeys") - # Every key the collector would read a kind from, other than ours, is - # absent from the fixture, so the stored type comes from KIND_KEY alone. - assert set(kind_keys + operation_keys) - {KIND_KEY} == set(OTHER_KIND_KEYS) - # The fixture's model and token keys are the first alias of each list. + # KIND_KEY is the fourth kind key. Every other key the collector would + # read a kind from is absent from the fixture, so the stored type comes + # from KIND_KEY alone. + assert kind_keys + operation_keys == [*OTHER_KIND_KEYS[:3], KIND_KEY, OTHER_KIND_KEYS[3]] + # The fixture's model and token keys are the first alias of each list, + # and it sets no total key, so total_tokens is derived. assert _go_strings(source["adapter"], "modelNameKeys")[0] == "llm.model_name" assert _go_strings(source["adapter"], "inputTokenKeys")[0] == "llm.token_count.prompt" assert _go_strings(source["adapter"], "outputTokenKeys")[0] == "llm.token_count.completion" - # input.value goes to the JSON overflow and is lifted into the input column. - assert "input.value" in _go_strings(source["adapter"], "overflowKeyPrefixes") - assert 'overflowAsString(overflow, "input.value")' in source["converter"] - assert 'getStrAttr(attrs, "project_name")' in source["stamp"] + llm_keys = set(attributes(by_name(body_spans(load_body()))[LLM])) + assert not llm_keys & set(_go_strings(source["adapter"], "totalTokenKeys")) + # input.value goes to the JSON overflow; the kind, model and token keys + # do not, so they stay readable as strings and numbers. + prefixes = _go_strings(source["adapter"], "overflowKeyPrefixes") + assert "input.value" in prefixes + for key in llm_keys: + assert not any(key.startswith(prefix) for prefix in prefixes), key + _has_statements( + _go_func(source["adapter"], "hasOverflowPrefix"), + "for _, p := range overflowKeyPrefixes { if strings.HasPrefix(key, p) { return true } }", + ) def test_collector_type_list_has_the_fixture_kinds_but_not_the_unknown_ones() -> None: - """converter.go lowercases the raw kind, applies spanKindSynonyms and - stores ``unknown`` for anything not in knownObservationTypes. This reads - those two tables; it does not run resolveObservationType.""" + """converter.go takes the first non-empty kind key, lowercases it, applies + spanKindSynonyms and returns ``unknown`` for anything not in + knownObservationTypes. This reads those tables and statements; it does + not run resolveObservationType.""" source = _collector_source() - assert "strings.ToLower(strings.TrimSpace(raw))" in source["converter"] + _has_statements( + _go_func(source["converter"], "resolveObservationType"), + 'raw := "" for _, k := range spanKindAttrKeys { if v := attrsStr[k]; v != "" { raw = v break } }', + "t := strings.ToLower(strings.TrimSpace(raw)) if syn, ok := spanKindSynonyms[t]; ok { t = syn }", + "if _, ok := knownObservationTypes[t]; !ok { return observationTypeUnknown } return t", + ) + assert 'observationTypeUnknown = "unknown"' in _go_code(source["converter"]) known = _go_map_keys(source["converter"], "knownObservationTypes") synonyms = _go_map_keys(source["converter"], "spanKindSynonyms") assert "unknown" in known @@ -386,3 +428,76 @@ def test_collector_type_list_has_the_fixture_kinds_but_not_the_unknown_ones() -> assert kind.lower() in known, kind for kind in UNKNOWN_KINDS: assert kind.lower() not in known | synonyms, kind + + +def test_collector_stores_the_kind_model_tokens_and_input() -> None: + """The statements between the span's attributes and the stored row: + Split routes strings and ints, DeriveHotKeys picks the model and tokens + (deriving the total), and spanToRow stores what they and the kind + resolver return. Matched as source text; nothing is run.""" + source = _collector_source() + _has_statements( + _go_func(source["adapter"], "Split"), + "if hasOverflowPrefix(k) { overflow[k] = otelValueToJSON(v) return true }", + "case pcommon.ValueTypeStr: attrsString[k] = v.Str()", + "n := v.Int() if n >= minSafeInt && n <= maxSafeInt { attrsNumber[k] = float64(n) }", + ) + _has_statements( + _go_func(source["adapter"], "DeriveHotKeys"), + "hk.Model = firstString(attrsString, modelNameKeys)", + "if v, ok := firstNumber(attrsString, attrsNumber, inputTokenKeys); ok { hk.PromptTokens = int32(v) }", + "if v, ok := firstNumber(attrsString, attrsNumber, outputTokenKeys); ok { hk.CompletionTokens = int32(v) }", + "if v, ok := firstNumber(attrsString, attrsNumber, totalTokenKeys); ok { hk.TotalTokens = int32(v) }" + " else if hk.PromptTokens+hk.CompletionTokens > 0 { hk.TotalTokens = hk.PromptTokens + hk.CompletionTokens }", + ) + row = _go_func(source["converter"], "spanToRow") + _has_statements( + row, + "adapter.Split(span.Attributes(), attrsStr, attrsNum, attrsBool, overflow)", + "hot := adapter.DeriveHotKeys(attrsStr, attrsNum)", + "observationType := resolveObservationType(attrsStr)", + 'input := overflowAsString(overflow, "input.value")', + '"observation_type": observationType,', + '"model": hot.Model,', + '"prompt_tokens": hot.PromptTokens,', + '"completion_tokens": hot.CompletionTokens,', + '"total_tokens": hot.TotalTokens,', + '"attributes_extra": overflow,', + '"input": input,', + ) + # One return, with a nil error: no kind value can fail the export. + assert row.count("return ") == row.count("return row, identity, nil") == 1 + + +def test_collector_project_name_rules() -> None: + """A missing project_name is a 400 only on an authenticated request; a + name not yet in the workspace is created; one that still does not + resolve is dropped and the request answered 200 (handleHTTPTraces, + StampResourceAttrs, ResolveProjectsForKey). Matched as source text.""" + source = _collector_source() + handler = _go_func(source["server"], "handleHTTPTraces") + assert handler.count("auth.StampResourceAttrs(") == 1 + # Stamping runs only inside the authenticated branch. Its error is the + # 400; a drop is only logged, and the handler goes on to its 200. + assert re.search( + r"if result := auth\.FromContext\(r\.Context\(\)\); result != nil \{[^}]*" + r"dropped, err := auth\.StampResourceAttrs\(.*?\) " + r'if err != nil \{ http\.Error\(w, "auth stamp: "\+err\.Error\(\), http\.StatusBadRequest\) return \} ' + r"if dropped > 0 \{ s\.log\.Warn\([^{}]*\) \}", + handler, + ) + assert handler.endswith("w.WriteHeader(http.StatusOK) _, _ = w.Write(out)") + _has_statements( + _go_func(source["stamp"], "StampResourceAttrs"), + "if result == nil { return 0, nil }", + 'pn := getStrAttr(attrs, "project_name") if pn == "" { missing = append(missing, i) continue }', + "if len(missing) > 0 { return 0, fmt.Errorf(", + 'if !ok || projectID == "" { unresolvable[projectName] = struct{}{} continue }', + "rss.RemoveIf(", + "return before - rss.Len(), nil", + ) + assert re.search( + r'return resolver\.GetOrCreateProject\(ctx, result\.OrgID, result\.WorkspaceID, name, "observe"\) \}\) ' + r'if err != nil \{ a\.log\.Warn\("project auto-create failed"[^{}]*\) continue \}', + _go_func(source["auth"], "ResolveProjectsForKey"), + ) From 5dfe7c503db3b35c9905864c8daa3eec2cd04a33 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Mon, 5 Oct 2026 10:18:10 -0700 Subject: [PATCH 11/14] docs(examples): qualify the OpenInference project_name 400 and state the drop The README said fi-collector fails a batch without project_name with a 400, unqualified. That holds only on an authenticated request: stamping is gated on auth.FromContext (server.go:458, stamp.go:19-21). It also left out that a project_name that is created on demand but still does not resolve is dropped with a 200 and a log warning, not rejected (auth.go:202-216, stamp.go:57-85, server.go:466-468). State both, and guard the wording with a README test; the opt-in test reads the lines. Co-Authored-By: Claude Opus 5.5 approved: Nikhil 2026-10-03 blanket Refs: TH-8336 --- python/examples/openinference/README.md | 11 +++++++-- .../tests/test_openinference_fixture.py | 24 +++++++++++++++++-- 2 files changed, 31 insertions(+), 4 deletions(-) diff --git a/python/examples/openinference/README.md b/python/examples/openinference/README.md index 9bf9fbf17..f3460611b 100644 --- a/python/examples/openinference/README.md +++ b/python/examples/openinference/README.md @@ -41,8 +41,15 @@ post to it, or call the Python adapter. Ingestion is fi-collector's | `lookup_refund_policy` | `retrieve` | `openinference.span.kind=TOOL` | The resource carries `service.name`, `project_name` and -`project_type=observe`. fi-collector fails a batch whose resource has no -`project_name` with HTTP 400 (`pkg/auth/stamp.go:31-45`, `server.go:461-464`). +`project_type=observe`. On an authenticated request, fi-collector fails a +batch whose resource has no `project_name` with HTTP 400 +(`pkg/auth/stamp.go:31-46`, `server.go:458-465`). A collector with auth +disabled skips this check (`server.go:458`, `stamp.go:19-21`). A +`project_name` that is not yet a project in the key's workspace is created +(`pkg/auth/auth.go:202-216`). One that still does not resolve, for example +because that create failed, is not rejected: the spans under that resource +are dropped and the request is still answered 200, with only a warning in +the collector log (`stamp.go:57-85`, `server.go:466-468`). Kind values are posted exactly as OpenInference sets them, in upper case. The fixture sets no other key fi-collector reads a kind from (`fi.span.kind`, diff --git a/python/examples/openinference/tests/test_openinference_fixture.py b/python/examples/openinference/tests/test_openinference_fixture.py index 793755210..45462abe2 100644 --- a/python/examples/openinference/tests/test_openinference_fixture.py +++ b/python/examples/openinference/tests/test_openinference_fixture.py @@ -190,8 +190,9 @@ def test_tool_span_carries_only_the_kind(posted: dict[str, Any]) -> None: def test_resource_carries_the_project(posted: dict[str, Any]) -> None: - """fi-collector fails a batch whose resource has no project_name - (pkg/auth/stamp.go:31-45; the handler answers 400, server.go:461-464).""" + """On an authenticated request, fi-collector fails a batch whose resource + has no project_name (pkg/auth/stamp.go:31-46; the handler answers 400, + server.go:458-465).""" (request,) = posted["requests"] assert request["resource_attributes"] == [ {"service.name": SERVICE_NAME, "project_name": PROJECT, "project_type": "observe"} @@ -319,6 +320,25 @@ def test_readme_states_what_the_tests_check() -> None: assert fact in readme, fact +def _readme_text() -> str: + """The README with line breaks and runs of spaces collapsed.""" + return " ".join(README.read_text(encoding="utf-8").split()) + + +def test_readme_qualifies_the_project_name_rules() -> None: + """The 400 for a missing project_name needs an authenticated request, and + a project_name that does not resolve is dropped with a 200, not rejected. + The opt-in test_collector_project_name_rules reads the source lines.""" + readme = _readme_text() + for fact in ( + "On an authenticated request, fi-collector fails a batch whose resource has no `project_name` with HTTP 400", + "A collector with auth disabled skips this check", + "is created", + "the request is still answered 200", + ): + assert fact in readme, fact + + # -------------------------------------------------------------------------- # Opt-in: read fi-collector's source instead of copying its tables. # -------------------------------------------------------------------------- From ea8859be850ced2f6b2b6fa80828d07c1986ad01 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Mon, 5 Oct 2026 10:18:32 -0700 Subject: [PATCH 12/14] docs(examples): say what the OpenInference opt-in tests check The README said the opt-in tests "check that the kind, model, token, input.value and project_name facts above still hold". They match source text, not behaviour. List the tables and statements they read, say how they match, and that a behaviour change outside those statements passes them while a behaviour-preserving refactor fails them. Guard the wording with a README test. Co-Authored-By: Claude Opus 5.5 approved: Nikhil 2026-10-03 blanket Refs: TH-8336 --- python/examples/openinference/README.md | 28 +++++++++++++------ .../tests/test_openinference_fixture.py | 12 ++++++++ 2 files changed, 32 insertions(+), 8 deletions(-) diff --git a/python/examples/openinference/README.md b/python/examples/openinference/README.md index f3460611b..e4436f03e 100644 --- a/python/examples/openinference/README.md +++ b/python/examples/openinference/README.md @@ -151,11 +151,24 @@ pytest imports `python/__init__.py` (fi_instrumentation) for any test under `python/`, which is why the other packages are listed. A run takes a few seconds. -Two opt-in tests read fi-collector's Go source instead of copying its tables. -They check that the kind, model, token, `input.value` and `project_name` -facts above still hold, and that the fixture's kinds are in the collector's -type list and the unknown ones are not. Point `FI_COLLECTOR_SRC` at a -fi-collector checkout: +Four opt-in tests read fi-collector's Go source instead of copying its +tables. They check that the source still has the tables and statements +behind the source-reading facts above: + +- the kind keys and their order, the type list and synonyms (the fixture's + kinds are in the list, the unknown ones are not), and the first model and + token aliases; +- the statements in `Split`, `DeriveHotKeys`, `resolveObservationType` and + `spanToRow` that route the fixture's keys, resolve the kind with its + `unknown` fallback, derive `total_tokens`, lift `input.value` and store + each in its column, and that `spanToRow` has one return, with a nil error; +- the `project_name` statements in `handleHTTPTraces`, `StampResourceAttrs` + and `ResolveProjectsForKey`. + +They match the source text, with comments removed and whitespace collapsed. +They do not run the collector: a behaviour change outside those statements +passes them, and a refactor that keeps the behaviour fails them. Point +`FI_COLLECTOR_SRC` at a fi-collector checkout: ```bash mkdir -p /tmp/fi-main @@ -163,6 +176,5 @@ git -C archive origin/main fi-collector | tar -x -C /tmp/f export FI_COLLECTOR_SRC=/tmp/fi-main/fi-collector # then run the command above ``` -Without it they are skipped. They read the source; they do not run the -collector. No CI job runs this directory, so run them whenever fi-collector's -kind or alias code changes. +Without it they are skipped. No CI job runs this directory, so run them +whenever fi-collector's kind, alias or project code changes. diff --git a/python/examples/openinference/tests/test_openinference_fixture.py b/python/examples/openinference/tests/test_openinference_fixture.py index 45462abe2..d2d114b02 100644 --- a/python/examples/openinference/tests/test_openinference_fixture.py +++ b/python/examples/openinference/tests/test_openinference_fixture.py @@ -339,6 +339,18 @@ def test_readme_qualifies_the_project_name_rules() -> None: assert fact in readme, fact +def test_readme_says_the_opt_in_tests_read_source_text() -> None: + """The opt-in tests match source statements; they do not show behaviour.""" + readme = _readme_text() + assert "facts above still hold" not in readme + for fact in ( + "They match the source text, with comments removed and whitespace collapsed", + "They do not run the collector", + "a refactor that keeps the behaviour fails them", + ): + assert fact in readme, fact + + # -------------------------------------------------------------------------- # Opt-in: read fi-collector's source instead of copying its tables. # -------------------------------------------------------------------------- From 020a8bd1bf2d2cdf06c5d2e186b805fe7379c80a Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Fri, 16 Oct 2026 19:56:43 -0700 Subject: [PATCH 13/14] test(examples): live OpenInference instrumentor run against a Future AGI collector (TH-8336) Real openinference-instrumentation 0.1.71/0.1.70 + OpenAI instrumentor, agent/retriever/tool/llm and two failure cases, marker-based privacy check. Static tests only in CI. Co-Authored-By: Claude Sonnet 5.5 --- python/examples/openinference-live/README.md | 19 ++++++ .../examples/openinference-live/live_e2e.py | 63 +++++++++++++++++++ .../openinference-live/requirements.txt | 5 ++ .../tests/test_live_script.py | 27 ++++++++ python/examples/openinference/README.md | 2 +- 5 files changed, 115 insertions(+), 1 deletion(-) create mode 100644 python/examples/openinference-live/README.md create mode 100644 python/examples/openinference-live/live_e2e.py create mode 100644 python/examples/openinference-live/requirements.txt create mode 100644 python/examples/openinference-live/tests/test_live_script.py diff --git a/python/examples/openinference-live/README.md b/python/examples/openinference-live/README.md new file mode 100644 index 000000000..ad4294b18 --- /dev/null +++ b/python/examples/openinference-live/README.md @@ -0,0 +1,19 @@ +# OpenInference live run (TH-8336 re-verification) + +`live_e2e.py` runs real OpenInference instrumentation (`openinference-instrumentation-openai` plus the `OITracer` tool/agent/retriever helpers) against a Future AGI collector. It is the real-instrumentor round trip that the fixture in `../openinference/` deliberately does not do. CI does not run it; `tests/` only checks the script statically. + +It makes one agent trace (retriever, two OpenAI chat calls with a tool call, one tool) and one failure trace (a tool that raises, and a chat call with an unknown model). A unique marker string is put in every piece of content so privacy can be checked. + +```bash +pip install -r requirements.txt +export FI_BASE_URL=http://localhost:14338 FI_API_KEY=... FI_SECRET_KEY=... +export OPENAI_API_KEY=... # OPENAI_BASE_URL optional (any OpenAI-compatible endpoint) +python live_e2e.py my-project MARKER123 +# content off: +OPENINFERENCE_HIDE_INPUTS=true OPENINFERENCE_HIDE_OUTPUTS=true \ +OPENINFERENCE_HIDE_INPUT_MESSAGES=true OPENINFERENCE_HIDE_OUTPUT_MESSAGES=true \ +OPENINFERENCE_HIDE_INPUT_TEXT=true OPENINFERENCE_HIDE_OUTPUT_TEXT=true \ +python live_e2e.py my-project-hidden MARKER456 +``` + +Result on 2026-10-16 (local Future AGI stack, openinference-instrumentation 0.1.71 and 0.1.70): agent, retriever, llm (model and tokens), tool, and two ERROR spans stored; with the hide variables the marker is absent from every stored column except the span status message of the two errors (the exception text is exported unmasked). See the docs page for the full table. diff --git a/python/examples/openinference-live/live_e2e.py b/python/examples/openinference-live/live_e2e.py new file mode 100644 index 000000000..3cbec667f --- /dev/null +++ b/python/examples/openinference-live/live_e2e.py @@ -0,0 +1,63 @@ +"""Real OpenInference instrumentors -> Future AGI (OTLP/HTTP, uncompressed). Not run in CI. +usage: python live_e2e.py PROJECT MARKER (hide flags: OPENINFERENCE_HIDE_* env)""" +import json, os, sys +from openai import OpenAI +from opentelemetry import trace +from opentelemetry.sdk.resources import Resource +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter, Compression +from openinference.instrumentation import OITracer, TraceConfig +from openinference.instrumentation.openai import OpenAIInstrumentor + +project, marker = sys.argv[1], sys.argv[2] +tp = TracerProvider(resource=Resource.create({"service.name": "th8336-live", "project_name": project, "project_type": "observe"})) +tp.add_span_processor(SimpleSpanProcessor(OTLPSpanExporter( + endpoint=os.environ["FI_BASE_URL"].rstrip("/") + "/v1/traces", + headers={"X-Api-Key": os.environ["FI_API_KEY"], "X-Secret-Key": os.environ["FI_SECRET_KEY"]}, + compression=Compression.NoCompression))) +OpenAIInstrumentor().instrument(tracer_provider=tp) +tracer = OITracer(tp.get_tracer("th8336"), TraceConfig()) +client = OpenAI(base_url=os.environ.get("OPENAI_BASE_URL", "https://api.openai.com/v1"), api_key=os.environ["OPENAI_API_KEY"]) +MODEL = os.environ.get("E2E_MODEL", "gpt-4o-mini") + +@tracer.tool +def lookup_refund_policy(order: str) -> str: + """Look up the refund policy for an order.""" + return f"policy for {order}: refunds within 30 days {marker}" + +@tracer.tool +def broken_tool(order: str) -> str: + """Always fails.""" + raise RuntimeError(f"upstream refused {marker}") + +tools = [{"type": "function", "function": {"name": "lookup_refund_policy", "description": "refund policy", + "parameters": {"type": "object", "properties": {"order": {"type": "string"}}, "required": ["order"]}}}] +q = f"What is the refund policy for order A-17? {marker}" +with tracer.start_as_current_span("support-agent", openinference_span_kind="agent") as root: + root.set_input(q) + with tracer.start_as_current_span("retrieve-docs", openinference_span_kind="retriever") as r: + r.set_input(q) + r.set_attribute("retrieval.documents.0.document.content", f"Refunds within 30 days {marker}") + msgs = [{"role": "user", "content": q}] + resp = client.chat.completions.create(model=MODEL, messages=msgs, tools=tools, + tool_choice={"type": "function", "function": {"name": "lookup_refund_policy"}}) + m = resp.choices[0].message + msgs.append(m.model_dump(exclude_none=True)) + for tc in m.tool_calls or []: + out = lookup_refund_policy(**json.loads(tc.function.arguments)) + msgs.append({"role": "tool", "tool_call_id": tc.id, "content": out}) + final = client.chat.completions.create(model=MODEL, messages=msgs, tools=tools) + root.set_output(final.choices[0].message.content or "") +# failure cases (separate trace): tool exception + upstream 4xx +with tracer.start_as_current_span("failing-agent", openinference_span_kind="agent") as fr: + try: + broken_tool("A-17") + except RuntimeError: + pass + try: + client.chat.completions.create(model="no-such-model-" + marker[:6], messages=[{"role": "user", "content": marker}]) + except Exception: + pass +tp.force_flush(); tp.shutdown() +print("done", trace.format_trace_id(root.get_span_context().trace_id)) diff --git a/python/examples/openinference-live/requirements.txt b/python/examples/openinference-live/requirements.txt new file mode 100644 index 000000000..468c5ef12 --- /dev/null +++ b/python/examples/openinference-live/requirements.txt @@ -0,0 +1,5 @@ +openinference-instrumentation==0.1.71 +openinference-instrumentation-openai==0.1.65 +openai==3.33.0 +opentelemetry-sdk==1.45.1 +opentelemetry-exporter-otlp-proto-http==1.45.1 diff --git a/python/examples/openinference-live/tests/test_live_script.py b/python/examples/openinference-live/tests/test_live_script.py new file mode 100644 index 000000000..5cb25fc44 --- /dev/null +++ b/python/examples/openinference-live/tests/test_live_script.py @@ -0,0 +1,27 @@ +"""Static checks for live_e2e.py. The script itself needs a Future AGI stack and an LLM key, so CI never runs it.""" +import ast +import re +from pathlib import Path + +SRC = (Path(__file__).resolve().parent.parent / "live_e2e.py").read_text() + + +def test_compiles(): + ast.parse(SRC) + + +def test_exports_uncompressed_with_project_and_key_headers_from_env(): + assert "Compression.NoCompression" in SRC + assert '"project_name": project' in SRC + for name in ("FI_BASE_URL", "FI_API_KEY", "FI_SECRET_KEY"): + assert f'os.environ["{name}"]' in SRC + + +def test_no_hardcoded_secret_or_local_path(): + assert not re.search(r"sk-[A-Za-z0-9]{10,}|/Users/|127\.0\.0\.1", SRC) + + +def test_has_success_tool_retriever_and_two_failure_cases(): + for needle in ('openinference_span_kind="agent"', 'openinference_span_kind="retriever"', + "@tracer.tool", "broken_tool", "no-such-model"): + assert needle in SRC diff --git a/python/examples/openinference/README.md b/python/examples/openinference/README.md index e4436f03e..b4f3696e2 100644 --- a/python/examples/openinference/README.md +++ b/python/examples/openinference/README.md @@ -13,7 +13,7 @@ not installed here and the tests do not import it. No license is claimed for it here. The fixture's keys are the ones its dependency, `openinference-semantic-conventions` 0.1.41, defines. -Alpha, fixture-validated. The tests pass on Python 3.10, 3.11, 3.12 and 3.13 +A real-instrumentor round trip against a live collector is in `../openinference-live/` (re-verified 2026-10-16). Alpha, fixture-validated. The tests pass on Python 3.10, 3.11, 3.12 and 3.13 (see [Tests](#tests)). They post only to the harness `Receiver` on 127.0.0.1: no fi-collector, no Arize account, no live instrumentor. Sections marked "source reading" come from future-agi `main` From 3c3be788884fd0a380f8c692f4203f263f18f480 Mon Sep 17 00:00:00 2001 From: Nikhil Pareek Date: Wed, 21 Oct 2026 17:13:52 -0700 Subject: [PATCH 14/14] test(openinference): match the project_name rules on current fi-collector (TH-8336) The opt-in source test failed against future-agi main b3737c4: handleHTTPTraces now routes simulation spans before stamping project_name, and writes its 200 through writeExportResponse. The behaviour the docs describe (400 without project_name, unresolved project dropped with a 200) is unchanged. The test now reads the authenticated branch by matching braces and follows the helper for the 200. It passes on 4af5338, main b3737c4 and dev 674f2f3, and fails when the stamp's 400 becomes a 200 or the stamp moves out of the authenticated branch. Co-Authored-By: Claude Opus 5.5 --- python/examples/openinference/README.md | 5 ++- .../tests/test_openinference_fixture.py | 32 ++++++++++++++++--- 2 files changed, 32 insertions(+), 5 deletions(-) diff --git a/python/examples/openinference/README.md b/python/examples/openinference/README.md index b4f3696e2..0d8ad7308 100644 --- a/python/examples/openinference/README.md +++ b/python/examples/openinference/README.md @@ -17,7 +17,10 @@ A real-instrumentor round trip against a live collector is in `../openinference- (see [Tests](#tests)). They post only to the harness `Receiver` on 127.0.0.1: no fi-collector, no Arize account, no live instrumentor. Sections marked "source reading" come from future-agi `main` -4af5338 and are not exercised by the always-on tests. +4af5338 and are not exercised by the always-on tests. The opt-in tests also +pass on `main` b3737c4 (2026-10-21), where `handleHTTPTraces` routes +simulation spans before stamping `project_name` and writes its 200 through +`writeExportResponse`; the `project_name` rules below are unchanged. ## The Django route is retired and is not proof diff --git a/python/examples/openinference/tests/test_openinference_fixture.py b/python/examples/openinference/tests/test_openinference_fixture.py index d2d114b02..c32470017 100644 --- a/python/examples/openinference/tests/test_openinference_fixture.py +++ b/python/examples/openinference/tests/test_openinference_fixture.py @@ -389,6 +389,23 @@ def _go_func(source: str, name: str) -> str: return _go_code(source[start : source.index("\n}\n", start)]) +def _go_block(code: str, opener: str) -> str: + """The body of the block that ``opener`` (ending in ``{``) opens in ``_go_code`` + text, braces matched. String literals are blanked before counting.""" + blanked = re.sub(r'"(?:\\.|[^"\\])*"|`[^`]*`', lambda m: "_" * len(m.group()), code) + assert opener in code, opener + start = code.index(opener) + len(opener) + depth = 1 + for i in range(start, len(blanked)): + if blanked[i] == "{": + depth += 1 + elif blanked[i] == "}": + depth -= 1 + if depth == 0: + return code[start:i].strip() + raise AssertionError("unclosed block: " + opener) + + def _has_statements(code: str, *statements: str) -> None: for statement in statements: assert statement in code, statement @@ -510,15 +527,22 @@ def test_collector_project_name_rules() -> None: handler = _go_func(source["server"], "handleHTTPTraces") assert handler.count("auth.StampResourceAttrs(") == 1 # Stamping runs only inside the authenticated branch. Its error is the - # 400; a drop is only logged, and the handler goes on to its 200. + # 400; a drop is only logged, and the handler goes on to its 200. Since + # future-agi main b3737c4 the branch first routes simulation (sandbox-key) + # spans, so the stamp is no longer the branch's first statement. + auth_branch = _go_block(handler, "if result := auth.FromContext(r.Context()); result != nil {") assert re.search( - r"if result := auth\.FromContext\(r\.Context\(\)\); result != nil \{[^}]*" r"dropped, err := auth\.StampResourceAttrs\(.*?\) " r'if err != nil \{ http\.Error\(w, "auth stamp: "\+err\.Error\(\), http\.StatusBadRequest\) return \} ' r"if dropped > 0 \{ s\.log\.Warn\([^{}]*\) \}", - handler, + auth_branch, ) - assert handler.endswith("w.WriteHeader(http.StatusOK) _, _ = w.Write(out)") + ok = "w.WriteHeader(http.StatusOK) _, _ = w.Write(out)" + if handler.endswith("writeExportResponse(w, ct, ptraceotlp.NewExportResponse())"): + # b3737c4 and later write the 200 through a helper. + assert _go_func(source["server"], "writeExportResponse").endswith(ok) + else: + assert handler.endswith(ok) _has_statements( _go_func(source["stamp"], "StampResourceAttrs"), "if result == nil { return 0, nil }",