diff --git a/python/examples/openllmetry/README.md b/python/examples/openllmetry/README.md new file mode 100644 index 000000000..851f770c1 --- /dev/null +++ b/python/examples/openllmetry/README.md @@ -0,0 +1,279 @@ +# OpenLLMetry (Traceloop SDK) → Future AGI (recipe) + +There is no traceAI package for OpenLLMetry. The Traceloop SDK already +exports OTLP/HTTP. This example points it at fi-collector with environment +variables, adds the Future AGI project as a resource attribute, and turns +content capture and metrics off. It adds no kind mapping of its own. Today +fi-collector stores the model call as an LLM span with model and tokens, and +the Traceloop decorator spans as `unknown` (see "What Future AGI stores"). + +Pinned: `traceloop-sdk` 0.62.4 (`Requires-Python: <4,>=3.10`). Its wheel +METADATA declares `License-Expression: Apache-2.0`; it has no Trove +classifiers and ships no license file. It is a dependency of this example +only, not of any traceAI package. 0.62.4 imports `httpx` and `requests` +without declaring them, so `requirements.txt` lists both. + +The contract test (see "Tests") passes on Python 3.10.17, 3.11.12, 3.12.10 +and 3.13.15 with `openai` 3.24.0 (pinned in `requirements.txt`), +`opentelemetry-sdk` and `opentelemetry-exporter-otlp-proto-http` 1.45.0 and +`opentelemetry-instrumentation-openai` 0.62.4, the versions uv resolved on +2026-10-05. + +## Run + +```bash +cd python/examples/openllmetry +pip install -r requirements.txt + +export FI_API_KEY="YOUR_API_KEY" +export FI_SECRET_KEY="YOUR_SECRET_KEY" +export TRACELOOP_BASE_URL="https://YOUR_FI_COLLECTOR_ORIGIN" # no path +export TRACELOOP_HEADERS="$(python -c 'import os; from urllib.parse import quote; print("x-api-key={0},x-secret-key={1}".format(quote(os.environ["FI_API_KEY"], safe=""), quote(os.environ["FI_SECRET_KEY"], safe="")))')" +export TRACELOOP_TRACE_CONTENT=false # the SDK default is true +export TRACELOOP_METRICS_ENABLED=false # fi-collector serves no /v1/metrics +export FI_PROJECT_NAME="my-chatbot" # resource attribute project_name +export OPENAI_API_KEY="YOUR_OPENAI_KEY" # optional: OPENAI_BASE_URL, OPENAI_MODEL +unset TRACELOOP_API_KEY # src/app.py refuses to start with it + +python src/app.py "What is the refund window?" +``` + +## The recipe + +`init_tracing()` in `src/app.py` is the whole integration: + +```python +Traceloop.init( + app_name="openllmetry-recipe", # service.name + resource_attributes={ + "project_name": os.environ["FI_PROJECT_NAME"], + "project_type": "observe", + }, + telemetry_enabled=False, +) +``` + +Everything else comes from the environment above, read by the SDK itself. +fi-collector rejects a batch whose resource has no `project_name` (HTTP 400) +and creates the project on first use. The rest of `src/app.py` is a +`@workflow` that calls a `@task` and an `@agent`; the agent calls a `@tool` +and the OpenAI Chat Completions API. + +### Endpoint + +Set `TRACELOOP_BASE_URL` to the collector origin with no path. With an +`http://` or `https://` URL the SDK uses OTLP/HTTP and posts to +`/v1/traces`, which fi-collector serves. Tested results with +0.62.4: + +| `TRACELOOP_BASE_URL` | Request path | +|---|---| +| `http://host:port` | `/v1/traces` | +| `http://host:port/` | `/v1/traces` | +| `http://host:port/v1/traces` | `/v1/traces` (0.62.4 does not append it twice) | +| `http://host:port/tracer` | `/tracer/v1/traces` | + +fi-collector also serves `/tracer/v1/traces`, the path traceAI's own +exporter appends to `FI_BASE_URL`. If your collector is only reachable under +that path, set the base URL to `/tracer`. These paths were tested +against a loopback receiver; no deployed endpoint was called. + +A base URL with no scheme makes the SDK switch to gRPC. This recipe does not +test gRPC. + +### Auth headers + +`TRACELOOP_HEADERS` carries the Future AGI keys. The SDK parses it with +OpenTelemetry's `parse_env_headers`: comma-separated `name=value` pairs, each +value percent-decoded, names lower-cased. The contract test proves that +`x-api-key=,x-secret-key=`, each value percent-encoded as in the +`export` line above, arrives as the `x-api-key` and `x-secret-key` request +headers that fi-collector reads. HTTP header names are case-insensitive. + +Percent-encode the values. The SDK splits on every comma, so an unencoded +comma cuts a value short (tested), and it drops, with only a logged warning, +any pair whose value has a space, `"`, `;` or `\`. That warning prints the +whole pair, value included, so an unencoded secret with one of those +characters ends up in your logs. + +Do not set `TRACELOOP_API_KEY`. That key is for Traceloop's own host. +Without `TRACELOOP_HEADERS` the SDK sends it as `Authorization: Bearer` on +every export. With them it is not sent with spans, but the SDK still uses it +for its image uploader (see "Privacy"). `src/app.py` exits if it is set. + +### Metrics + +The SDK exports metrics by default to `/v1/metrics` (tested). +fi-collector routes no `/v1/metrics`, so the recipe sets +`TRACELOOP_METRICS_ENABLED=false`. + +### Telemetry + +`telemetry_enabled=False` is passed. In 0.62.4 that parameter is accepted +and never read, and the package has no telemetry client. The contract test +runs the app with every non-loopback connection blocked and logged, and no +run logs an attempt. + +## What is traced + +With content off, `python src/app.py` exports five spans in one trace: + +| Span | `traceloop.span.kind` | Parent | Attributes | +|---|---|---|---| +| `support_request.workflow` | `workflow` | none | `traceloop.entity.name`, `traceloop.workflow.name` | +| `normalize_question.task` | `task` | workflow | same | +| `support_agent.agent` | `agent` | workflow | same, plus `gen_ai.agent.name` | +| `lookup_policy.tool` | `tool` | agent | same, plus `gen_ai.agent.name`, `gen_ai.tool.name` | +| `openai.chat` | none | agent | `gen_ai.operation.name=chat`, `gen_ai.provider.name`, `gen_ai.request.model`, `gen_ai.response.model`, `gen_ai.response.id`, `gen_ai.response.finish_reasons`, `gen_ai.usage.input_tokens`, `gen_ai.usage.output_tokens`, `gen_ai.usage.total_tokens`, `gen_ai.is_streaming`, `gen_ai.openai.api_base`, `gen_ai.agent.name`, `traceloop.workflow.name` | + +0.62.4 emits the `gen_ai.*` model and usage keys, not the older +`llm.request.model` or `llm.usage.prompt_tokens`. fi-collector's alias list +reads the `gen_ai.*` keys first. + +### What Future AGI stores + +fi-collector writes spans to ClickHouse itself. It reads a span's kind from +`fi.span.kind`, `gen_ai.span.kind`, `llm.request.type` or +`openinference.span.kind`, and falls back to `gen_ai.operation.name` +(`fi-collector/exporter/clickhouse25exporter/converter.go:79-131` at +future-agi `dev` ea12cc3). It does not read `traceloop.span.kind`. Running +this recipe against that collector (2026-10-11, `traceloop-sdk` 0.62.4) +stored: + +| Span | Stored `observation_type` | Model and tokens | `input` / `output` columns | +|---|---|---|---| +| `support_request.workflow`, `normalize_question.task` | `unknown` | none | empty | +| `support_agent.agent`, `lookup_policy.tool` | `unknown` | none | empty | +| `openai.chat` | `llm` (from `gen_ai.operation.name=chat`) | `model`, `prompt_tokens`, `completion_tokens` | empty | + +The tree (workflow > task, workflow > agent > tool and model call) and error +status are kept, and every `traceloop.*` key is kept as a span attribute. +A streamed call (`stream_options={"include_usage": True}`) is stored the same +way, with tokens. The `input` and `output` columns stay empty with content on +too: fi-collector fills them only from `input.value` and `output.value` +(`converter.go:343-344`), and 0.62.4 sets neither, so the text stays in +the span attributes listed under "Privacy". A model call made outside any +Traceloop decorator is stored as `llm` with its model and tokens as well. + +Showing the decorator spans as chain, agent and tool needs fi-collector to +read `traceloop.span.kind`. That is backend work (SF-1), not this recipe. + +The backend's Python OpenLLMetry adapter +(`futureagi/tracer/utils/adapters/openllmetry.py`) has that map, but it runs +only on the Django ingestion path, not on fi-collector's. Its map, quoted so +this page adds no second one: + +| `traceloop.span.kind` | Future AGI kind | +|---|---| +| `workflow` | CHAIN | +| `task` | CHAIN | +| `agent` | AGENT | +| `tool` | TOOL | +| `unknown` | UNKNOWN | + +In that adapter a span without `traceloop.span.kind` takes its kind from +`gen_ai.operation.name`: `chat` and `completion` are LLM, `embedding` and +`embeddings` EMBEDDING, `rerank` RERANKER, `execute_tool` TOOL. The opt-in +tests under "Tests" read both maps from that file, check the table above +against the first, and check the emitted span kinds and operation names +against both. The adapter recognises a span as OpenLLMetry only when it has +a `traceloop.*` key; a model call made outside any Traceloop decorator has +none (tested). + +The agent, tool and model-call spans carry both `gen_ai.*` and +`traceloop.*` keys (tested). + +## Privacy + +Content capture is on unless `TRACELOOP_TRACE_CONTENT` is set: the SDK +treats an unset variable as `true`. With content on, the model-call span +carries `gen_ai.input.messages` (system prompt and user question, including +anything your code put into the prompt) and `gen_ai.output.messages`, and +every decorated span carries `traceloop.entity.input` and +`traceloop.entity.output` (the function's arguments and return value as +JSON). The contract test finds the question, the answer and the tool output +in those keys with the variable unset, and none of those keys and none of +that text anywhere in the export with it set to `false`. + +Use `true` or `false` only. The Traceloop decorators treat only `true` as +on; the OpenAI instrumentor also treats `1`, `yes` and `on` as on. + +Not covered by the content switch (read in the 0.62.4 source, not tested): + +- When a decorated function raises, its span gets the exception message as + its status and an `exception` event with the message and stack trace. +- `gen_ai.openai.api_base` (the OpenAI base URL) is on every model-call span. + +With content on, a base64 image in a prompt makes the OpenAI instrumentor +upload it to `/v2/traces//spans//images` with +`Authorization: Bearer `. fi-collector has no such route. +Keep content off, or keep base64 images out of prompts. + +No Future AGI key, secret or OpenAI key appears in any exported span or +resource, or in the app's output (tested). + +## Notes + +- Traceloop uses a `BatchSpanProcessor` and registers an `atexit` flush, so + a short script exports on exit and a server exports in batches. +- With content on, 0.62.4 writes messages as `gen_ai.input.messages` and + `gen_ai.output.messages`. fi-collector keeps them as span attributes and + leaves the `input` and `output` columns empty (see "What Future AGI + stores"). The Django-path adapter (`openllmetry.py:114`, `:214-215`, + `:231`) builds input and output only from indexed `gen_ai.prompt..*` and + `gen_ai.completion..*` keys and then deletes every `gen_ai.*` key, so it + would drop them. Both are backend items (SF-1); the recipe keeps content + off. +- The contract test makes no call to fi-collector, Traceloop or OpenAI. Auth + rejection (401 without keys), project stamping and storage are + fi-collector behaviour the harness receiver does not reproduce; "What + Future AGI stores" comes from one manual run against a local fi-collector. + +## Tests + +`tests/test_openllmetry_recipe.py` runs `src/app.py` as written, in a +subprocess, with the real `traceloop-sdk` 0.62.4 instrumenting the real +`openai` client. `tests/_fake_openai.py` is a loopback fake of the Chat +Completions API; the spans go to the shared harness `Receiver` +(`python/tests/harness`), which serves `/v1/traces` and `/tracer/v1/traces` +on 127.0.0.1. `tests/_guarded_run.py` blocks and logs every non-loopback +connection; a positive control (`tests/_guard_probe.py`) proves it refuses +and logs IPv4, IPv6 and DNS attempts. All keys are placeholders. + +The tests check the request path, the auth headers, the resource +attributes, the span kinds and tree, content off and on, metrics off and +on, the API key guard, and that no key reaches the export. +`tests/bare_llm_call.py` is a fixture (a model call outside any decorator), +not part of the recipe. + +From the repository root, Python 3.11: + +```bash +env -u PYTHONPATH PYTHONPATH="python/examples/openllmetry:python:python/tests" \ + uv run --no-project --python 3.11 \ + --with pytest --with pytest-asyncio --with opentelemetry-api \ + --with opentelemetry-sdk --with opentelemetry-exporter-otlp-proto-http \ + --with requests --with jsonschema --with protobuf --with opentelemetry-proto \ + --with 'traceloop-sdk==0.62.4' --with 'openai==3.24.0' --with httpx \ + pytest python/examples/openllmetry/tests -q -p no:cacheprovider \ + --noconftest -o addopts= -rfEs +``` + +For Python 3.10, 3.12 or 3.13, replace `--python 3.11`. Without +`--with httpx` every run fails at `import traceloop.sdk`. + +Two tests are opt-in. They read `_TRACELOOP_KIND_MAP` and +`_OPERATION_KIND_MAP` from the backend's `openllmetry.py` and check the +emitted kinds and the kind table above against them. Point +`FI_BACKEND_OPENLLMETRY_PY` at a copy of that file: + +```bash +git -C show origin/main:futureagi/tracer/utils/adapters/openllmetry.py > /tmp/openllmetry.py +export FI_BACKEND_OPENLLMETRY_PY=/tmp/openllmetry.py # then run the command above +``` + +Without it they are skipped. No CI job runs this example, so run the pair +whenever the backend adapter or this recipe changes; otherwise the kind table +above can drift from the backend unnoticed. The tests start the app in a new process 10 +times, and importing and initialising the SDK and its instrumentors takes +most of each run, so a full run takes about 5 minutes. diff --git a/python/examples/openllmetry/requirements.txt b/python/examples/openllmetry/requirements.txt new file mode 100644 index 000000000..ed42b3b61 --- /dev/null +++ b/python/examples/openllmetry/requirements.txt @@ -0,0 +1,7 @@ +# Not a traceAI package: these are the example's own dependencies. +traceloop-sdk==0.62.4 +# traceloop-sdk 0.62.4 imports httpx and requests but declares neither, and +# openai 3.x depends on httpx2, not httpx. Without them the import fails. +httpx +requests +openai==3.24.0 # the version the contract test ran with; the instrumentor is pinned by traceloop-sdk diff --git a/python/examples/openllmetry/src/app.py b/python/examples/openllmetry/src/app.py new file mode 100644 index 000000000..924e67185 --- /dev/null +++ b/python/examples/openllmetry/src/app.py @@ -0,0 +1,99 @@ +"""Send OpenLLMetry (Traceloop SDK) traces to Future AGI. + +Recipe example, not a package. The Traceloop SDK already exports OTLP/HTTP; +this file only calls ``Traceloop.init`` with the resource attributes +fi-collector needs and wraps one model call in Traceloop's decorators. +Everything else (endpoint, auth headers, metrics, content) comes from the +environment variables listed in README.md. + + python src/app.py "What is the refund window?" +""" + +from __future__ import annotations + +import os +import sys + +from openai import OpenAI +from traceloop.sdk import Traceloop +from traceloop.sdk.decorators import agent, task, tool, workflow + +APP_NAME = "openllmetry-recipe" # exported as the service.name resource attribute +MODEL = os.environ.get("OPENAI_MODEL", "gpt-4o-mini") + +_POLICIES = { + "refund": "Refunds are accepted within 30 days of purchase.", + "shipping": "Orders ship within 2 business days.", +} + + +def init_tracing() -> None: + """Start Traceloop's exporter. Call once per process, before any model call. + + TRACELOOP_BASE_URL, TRACELOOP_HEADERS, TRACELOOP_METRICS_ENABLED and + TRACELOOP_TRACE_CONTENT are read by the SDK itself (see README.md). + """ + if os.environ.get("TRACELOOP_API_KEY"): + # That key is for Traceloop's own host. Without TRACELOOP_HEADERS the + # SDK sends it as a bearer token with every export; with them it is + # not sent with spans, but the SDK still hands it to its image + # uploader, which sends it as a bearer token to TRACELOOP_BASE_URL + # when content capture is on and a prompt carries a base64 image. + raise SystemExit("Unset TRACELOOP_API_KEY when exporting to Future AGI.") + Traceloop.init( + app_name=APP_NAME, + # fi-collector rejects (HTTP 400) a batch whose resource has no + # project_name, and creates the project on first use. + resource_attributes={ + "project_name": os.environ["FI_PROJECT_NAME"], + "project_type": "observe", + }, + # traceloop-sdk 0.62.4 accepts this flag and has no telemetry code + # behind it; it is passed so nothing depends on that. + telemetry_enabled=False, + ) + + +@tool(name="lookup_policy") +def lookup_policy(question: str) -> str: + topic = "refund" if "refund" in question.lower() else "shipping" + return _POLICIES[topic] + + +@agent(name="support_agent") +def support_agent(question: str) -> str: + facts = lookup_policy(question) + # OpenAI() reads OPENAI_API_KEY and OPENAI_BASE_URL. + response = OpenAI().chat.completions.create( + model=MODEL, + messages=[ + {"role": "system", "content": "Answer using only these facts: " + facts}, + {"role": "user", "content": question}, + ], + ) + return response.choices[0].message.content or "" + + +@task(name="normalize_question") +def normalize_question(question: str) -> str: + return " ".join(question.split()) + + +@workflow(name="support_request") +def support_request(question: str) -> str: + # Model calls made inside a @workflow carry traceloop.workflow.name and + # nest under the workflow span in one trace. + return support_agent(normalize_question(question)) + + +def main(argv: list[str]) -> int: + init_tracing() + question = argv[1] if len(argv) > 1 else "What is the refund window?" + print(support_request(question)) + # Traceloop registers an atexit flush, so a short script needs no more. + # A long-lived server keeps the default batch processor running. + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/python/examples/openllmetry/tests/_fake_openai.py b/python/examples/openllmetry/tests/_fake_openai.py new file mode 100644 index 000000000..e3426e180 --- /dev/null +++ b/python/examples/openllmetry/tests/_fake_openai.py @@ -0,0 +1,76 @@ +"""A loopback stand-in for the OpenAI Chat Completions API (non-streaming). + +Every POST to /v1/chat/completions gets one assistant message whose text is +``answer_text``, with fixed token usage, so the instrumentor records model +and usage attributes. Each request body and Authorization header is kept for +assertions. Listens on 127.0.0.1 only. +""" + +from __future__ import annotations + +import json +import threading +from http import HTTPStatus +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from typing import Any + +RESPONSE_MODEL = "gpt-4o-mini-2024-07-18" +USAGE = {"prompt_tokens": 11, "completion_tokens": 7, "total_tokens": 18} + + +class FakeOpenAI: + """Serve /v1/chat/completions on 127.0.0.1 and record each request.""" + + def __init__(self, answer_text: str) -> None: + self.requests: list[dict[str, Any]] = [] + self.authorizations: list[str | None] = [] + lock = threading.Lock() + owner = self + + class Handler(BaseHTTPRequestHandler): + def do_POST(self) -> None: # noqa: N802 - BaseHTTPRequestHandler API + if self.path.rstrip("/") != "/v1/chat/completions": + self.send_error(HTTPStatus.NOT_FOUND) + return + length = int(self.headers.get("Content-Length", "0")) + request = json.loads(self.rfile.read(length).decode("utf-8")) + with lock: + owner.requests.append(request) + owner.authorizations.append(self.headers.get("Authorization")) + body = json.dumps( + { + "id": "chatcmpl-fake", + "object": "chat.completion", + "created": 1_700_000_000, + "model": RESPONSE_MODEL, + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": answer_text}, + "finish_reason": "stop", + } + ], + "usage": USAGE, + } + ).encode("utf-8") + self.send_response(HTTPStatus.OK) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, format: str, *args: Any) -> None: # noqa: A002 + return + + self._server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + self.base_url = "http://127.0.0.1:{0}/v1".format(self._server.server_port) + self._thread = threading.Thread(target=self._server.serve_forever, daemon=True) + self._thread.start() + + def __enter__(self) -> "FakeOpenAI": + return self + + def __exit__(self, *_exc: Any) -> None: + self._server.shutdown() + self._server.server_close() + self._thread.join() diff --git a/python/examples/openllmetry/tests/_guard_probe.py b/python/examples/openllmetry/tests/_guard_probe.py new file mode 100644 index 000000000..5ebeff59d --- /dev/null +++ b/python/examples/openllmetry/tests/_guard_probe.py @@ -0,0 +1,26 @@ +"""A test-only probe: try three non-loopback connections under the loopback guard. + +The contract test runs this through ``_guarded_run.py`` to prove the guard +really refuses and logs a non-loopback attempt, so the ``guard_attempts == []`` +assertions elsewhere cannot pass because the guard silently stopped working. +The targets are reserved and never reachable: TEST-NET-1 (RFC 5737), the IPv6 +documentation prefix (RFC 3849) and the ``.invalid`` TLD (RFC 2606). +Not part of the recipe. +""" + +from __future__ import annotations + +import socket + +refused = [] +for label, attempt in ( + ("connect", lambda: socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect(("192.0.2.1", 80))), + ("connect_ex", lambda: socket.socket(socket.AF_INET6, socket.SOCK_STREAM).connect_ex(("2001:db8::1", 80, 0, 0))), + ("getaddrinfo", lambda: socket.getaddrinfo("example.invalid", 443)), +): + try: + attempt() + except OSError as error: + if "blocked by loopback guard" in str(error): + refused.append(label) +print("refused:", ",".join(refused)) diff --git a/python/examples/openllmetry/tests/_guarded_run.py b/python/examples/openllmetry/tests/_guarded_run.py new file mode 100644 index 000000000..a1d3d2e37 --- /dev/null +++ b/python/examples/openllmetry/tests/_guarded_run.py @@ -0,0 +1,81 @@ +"""Run a script with every non-loopback network connection blocked and logged. + +Usage: python _guarded_run.py [args...] + +Before the script starts, ``socket.socket.connect``/``connect_ex`` and +``socket.getaddrinfo`` refuse any host other than 127.0.0.1, ::1 or +localhost. Each refused attempt is appended as a JSON line to the file named +by ``LOOPBACK_GUARD_LOG``, so a test can assert that the SDK, its +instrumentors and the example tried to reach nothing else (no Traceloop host, +no telemetry endpoint, no real OpenAI). Unix sockets are not affected. +""" + +from __future__ import annotations + +import json +import os +import runpy +import socket +import sys + +_ALLOWED_HOSTS = {"127.0.0.1", "::1", "localhost"} +_LOG_PATH = os.environ.get("LOOPBACK_GUARD_LOG", "") + + +def _host(value: object) -> str: + if isinstance(value, bytes): + return value.decode("ascii", "replace") + return str(value) + + +def _refuse(kind: str, target: object) -> None: + if _LOG_PATH: + with open(_LOG_PATH, "a", encoding="utf-8") as log: + log.write(json.dumps({"kind": kind, "target": repr(target)}) + "\n") + + +_original_connect = socket.socket.connect +_original_connect_ex = socket.socket.connect_ex +_original_getaddrinfo = socket.getaddrinfo + + +def _is_blocked(sock: socket.socket, address: object) -> bool: + if sock.family not in (socket.AF_INET, socket.AF_INET6): + return False + host = address[0] if isinstance(address, tuple) else address + return _host(host) not in _ALLOWED_HOSTS + + +def _connect(self: socket.socket, address: object) -> None: + if _is_blocked(self, address): + _refuse("connect", address) + raise OSError("blocked by loopback guard: {0!r}".format(address)) + return _original_connect(self, address) + + +def _connect_ex(self: socket.socket, address: object) -> int: + if _is_blocked(self, address): + _refuse("connect_ex", address) + raise OSError("blocked by loopback guard: {0!r}".format(address)) + return _original_connect_ex(self, address) + + +def _getaddrinfo(host: object, *args: object, **kwargs: object): # type: ignore[no-untyped-def] + if host is not None and _host(host) not in _ALLOWED_HOSTS: + _refuse("getaddrinfo", host) + raise socket.gaierror("blocked by loopback guard: {0!r}".format(host)) + return _original_getaddrinfo(host, *args, **kwargs) + + +def main() -> None: + socket.socket.connect = _connect # type: ignore[method-assign] + socket.socket.connect_ex = _connect_ex # type: ignore[method-assign] + socket.getaddrinfo = _getaddrinfo # type: ignore[assignment] + script = sys.argv[1] + sys.argv = sys.argv[1:] + sys.path.insert(0, os.path.dirname(os.path.abspath(script))) + runpy.run_path(script, run_name="__main__") + + +if __name__ == "__main__": + main() diff --git a/python/examples/openllmetry/tests/bare_llm_call.py b/python/examples/openllmetry/tests/bare_llm_call.py new file mode 100644 index 000000000..13188ada4 --- /dev/null +++ b/python/examples/openllmetry/tests/bare_llm_call.py @@ -0,0 +1,28 @@ +"""A test-only fixture: one OpenAI call made outside any Traceloop decorator. + +It shows which attributes a bare model call carries, so the contract test +can check the README's claim that such a span has no ``traceloop.*`` key. +Not part of the recipe. +""" + +from __future__ import annotations + +import os +import sys + +from openai import OpenAI +from traceloop.sdk import Traceloop + +Traceloop.init( + app_name="openllmetry-recipe", + resource_attributes={ + "project_name": os.environ["FI_PROJECT_NAME"], + "project_type": "observe", + }, + telemetry_enabled=False, +) +response = OpenAI().chat.completions.create( + model="gpt-4o-mini", + messages=[{"role": "user", "content": sys.argv[1]}], +) +print(response.choices[0].message.content) diff --git a/python/examples/openllmetry/tests/test_openllmetry_recipe.py b/python/examples/openllmetry/tests/test_openllmetry_recipe.py new file mode 100644 index 000000000..9f48ef16a --- /dev/null +++ b/python/examples/openllmetry/tests/test_openllmetry_recipe.py @@ -0,0 +1,540 @@ +"""Contract test for the OpenLLMetry (Traceloop SDK) recipe. + +Runs ``src/app.py`` as written with the real ``traceloop-sdk`` 0.62.4 and a +real ``openai`` client, in a subprocess whose network is limited to +127.0.0.1 (``_guarded_run.py``). The model is a loopback fake of the OpenAI +Chat Completions API; spans go to the shared harness ``Receiver``, which +serves ``/v1/traces`` and ``/tracer/v1/traces`` like fi-collector's HTTP mux +but does not authenticate, stamp projects or store anything. + +All keys are placeholders. Nothing here contacts Traceloop, OpenAI or Future +AGI. +""" + +from __future__ import annotations + +import ast +import json +import os +import re +import sys +import threading +from http import HTTPStatus +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from typing import Any, Optional +from urllib.parse import quote, urlsplit + +import pytest + +from harness import Receiver, run + +TESTS_DIR = Path(__file__).resolve().parent +RECIPE_DIR = TESTS_DIR.parent +APP = RECIPE_DIR / "src" / "app.py" +README = RECIPE_DIR / "README.md" +BARE_CALL = TESTS_DIR / "bare_llm_call.py" +GUARD = TESTS_DIR / "_guarded_run.py" +GUARD_PROBE = TESTS_DIR / "_guard_probe.py" + +sys.path.insert(0, str(TESTS_DIR)) +from _fake_openai import RESPONSE_MODEL, USAGE, FakeOpenAI # noqa: E402 + +PROJECT = "openllmetry-recipe-contract" +SERVICE_NAME = "openllmetry-recipe" +FI_API_KEY = "fi-api-placeholder-0000" +# A comma and an equals sign: the recipe's percent-encoding must carry them. +FI_SECRET_KEY = "fi-secret-placeholder,part=0000" +OPENAI_KEY = "sk-openai-placeholder-0000" + +# Content markers: the question (input), the fake model's answer (output) +# and the tool's result. None may reach an export with content off. +QUESTION = "QMARK7c1e what is the refund window?" +ANSWER = "AMARK5d2b refunds are accepted for 30 days." +TOOL_OUTPUT = "Refunds are accepted within 30 days of purchase." +CONTENT_MARKERS = ("QMARK7c1e", "AMARK5d2b", TOOL_OUTPUT) +CONTENT_KEYS = ( + "gen_ai.input.messages", + "gen_ai.output.messages", + "traceloop.entity.input", + "traceloop.entity.output", +) + +SPAN_NAMES = { + "workflow": "support_request.workflow", + "task": "normalize_question.task", + "agent": "support_agent.agent", + "tool": "lookup_policy.tool", + "llm": "openai.chat", +} +# A hang guard, not a speed check: one run imports the SDK and its ~40 +# instrumentors, which took up to a minute on a loaded laptop. +RUN_TIMEOUT_SECONDS = 180 + + +def traceloop_headers(api_key: str, secret_key: str) -> str: + """Build TRACELOOP_HEADERS exactly as README.md tells users to.""" + return "x-api-key={0},x-secret-key={1}".format( + quote(api_key, safe=""), quote(secret_key, safe="") + ) + + +class _PathRecorder: + """Loopback catch-all that records every request's method and path. + + It answers /v1/traces and /tracer/v1/traces with 200 and anything else + with 404, as fi-collector's HTTP mux does (pkg/server/server.go). + """ + + def __init__(self) -> None: + self.calls: list[tuple[str, str]] = [] + lock = threading.Lock() + owner = self + + class Handler(BaseHTTPRequestHandler): + def do_POST(self) -> None: # noqa: N802 + path = urlsplit(self.path).path + self.rfile.read(int(self.headers.get("Content-Length", "0"))) + with lock: + owner.calls.append(("POST", path)) + status = ( + HTTPStatus.OK + if path in ("/v1/traces", "/tracer/v1/traces") + else HTTPStatus.NOT_FOUND + ) + self.send_response(status) + self.send_header("Content-Length", "0") + self.end_headers() + + def log_message(self, format: str, *args: Any) -> None: # noqa: A002 + return + + self._server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + self.origin = "http://127.0.0.1:{0}".format(self._server.server_port) + self._thread = threading.Thread(target=self._server.serve_forever, daemon=True) + self._thread.start() + + def __enter__(self) -> "_PathRecorder": + return self + + def __exit__(self, *_exc: Any) -> None: + self._server.shutdown() + self._server.server_close() + self._thread.join() + + def paths(self) -> set[str]: + return {path for _method, path in self.calls} + + +def _child_env( + home: Path, guard_log: Path, base_url: str, openai_base_url: str, **overrides: Optional[str] +) -> dict[str, str]: + """The recipe's environment, built from scratch (nothing inherited but PATH). + + Bytecode writing stays on so later runs reuse the SDK's compiled modules; + the scripts themselves run through runpy and leave no .pyc in the repo. + """ + env = { + "PATH": os.environ.get("PATH", ""), + "HOME": str(home), + "LOOPBACK_GUARD_LOG": str(guard_log), + "TRACELOOP_BASE_URL": base_url, + "TRACELOOP_HEADERS": traceloop_headers(FI_API_KEY, FI_SECRET_KEY), + "TRACELOOP_METRICS_ENABLED": "false", + "TRACELOOP_TRACE_CONTENT": "false", + "FI_PROJECT_NAME": PROJECT, + "OPENAI_API_KEY": OPENAI_KEY, + "OPENAI_BASE_URL": openai_base_url, + } + for key, value in overrides.items(): + if value is None: + env.pop(key, None) + else: + env[key] = value + return env + + +def _run_script( + tmp_path: Path, + script: Path, + args: list[str], + base_url_for: Any, + **overrides: Optional[str], +) -> dict[str, Any]: + """Run one script under the loopback guard; return everything the tests read.""" + tmp_path.mkdir(parents=True, exist_ok=True) + guard_log = tmp_path / "guard.jsonl" + with Receiver() as receiver, FakeOpenAI(ANSWER) as fake: + env = _child_env( + tmp_path, guard_log, base_url_for(receiver), fake.base_url, **overrides + ) + result = run( + [sys.executable, str(GUARD), str(script), *args], + env=env, + stdin=None, + timeout=RUN_TIMEOUT_SECONDS, + ) + record = { + "result": result, + "stdout": result.stdout.decode("utf-8", "replace"), + "stderr": result.stderr.decode("utf-8", "replace"), + "requests": receiver.requests(), + "spans": receiver.spans(), + "model_requests": list(fake.requests), + "model_authorizations": list(fake.authorizations), + } + record["guard_attempts"] = ( + [json.loads(line) for line in guard_log.read_text().splitlines()] + if guard_log.exists() + else [] + ) + return record + + +def _value(value: dict[str, Any]) -> Any: + for kind in ("stringValue", "boolValue", "doubleValue"): + if kind in value: + return value[kind] + if "intValue" in value: + return int(value["intValue"]) + if "arrayValue" in value: + return [_value(item) for item in value["arrayValue"].get("values", [])] + return value + + +def _attrs(span: dict[str, Any]) -> dict[str, Any]: + return {item["key"]: _value(item.get("value", {})) for item in span.get("attributes", [])} + + +def _by_name(spans: list[dict[str, Any]]) -> dict[str, dict[str, Any]]: + named = {span["name"]: span for span in spans} + assert len(named) == len(spans), "span names repeat: {0}".format( + sorted(span["name"] for span in spans) + ) + return named + + +def _export_dump(record: dict[str, Any]) -> str: + """Everything the export carried except HTTP headers: spans and resources.""" + return json.dumps( + { + "spans": record["spans"], + "resources": [r["resource_attributes"] for r in record["requests"]], + }, + sort_keys=True, + ) + + +def _assert_ran(record: dict[str, Any]) -> None: + result = record["result"] + assert not result.timed_out, record["stderr"] + assert result.returncode == 0, record["stderr"] + assert record["guard_attempts"] == [], record["guard_attempts"] + + +# -------------------------------------------------------------------------- +# The recipe as documented: content off, metrics off, headers percent-encoded. +# -------------------------------------------------------------------------- + + +@pytest.fixture(scope="module") +def recipe_run(tmp_path_factory: pytest.TempPathFactory) -> dict[str, Any]: + record = _run_script( + tmp_path_factory.mktemp("recipe"), + APP, + [QUESTION], + lambda receiver: receiver.origin, + ) + _assert_ran(record) + return record + + +def test_recipe_runs_with_loopback_network_only(recipe_run: dict[str, Any]) -> None: + assert recipe_run["guard_attempts"] == [] + assert ANSWER in recipe_run["stdout"] + # The model call reached the fake, with the OpenAI key the client was given. + assert len(recipe_run["model_requests"]) == 1 + assert recipe_run["model_authorizations"] == ["Bearer " + OPENAI_KEY] + + +def test_guard_refuses_and_logs_non_loopback_connections(tmp_path: Path) -> None: + """Positive control for every ``guard_attempts == []`` assertion in this file.""" + guard_log = tmp_path / "guard.jsonl" + result = run( + [sys.executable, str(GUARD), str(GUARD_PROBE)], + env={"PATH": os.environ.get("PATH", ""), "HOME": str(tmp_path), "LOOPBACK_GUARD_LOG": str(guard_log)}, + stdin=None, + timeout=60, + ) + assert not result.timed_out + assert result.returncode == 0, result.stderr.decode("utf-8", "replace") + assert "refused: connect,connect_ex,getaddrinfo" in result.stdout.decode("utf-8", "replace") + logged = [json.loads(line) for line in guard_log.read_text().splitlines()] + assert [entry["kind"] for entry in logged] == ["connect", "connect_ex", "getaddrinfo"] + assert "192.0.2.1" in logged[0]["target"] + assert "2001:db8::1" in logged[1]["target"] + assert "example.invalid" in logged[2]["target"] + + +def test_base_url_without_path_posts_to_v1_traces_once(recipe_run: dict[str, Any]) -> None: + assert recipe_run["requests"], "no export reached the receiver" + assert {request["path"] for request in recipe_run["requests"]} == {"/v1/traces"} + + +def test_traceloop_headers_carry_the_future_agi_keys(recipe_run: dict[str, Any]) -> None: + assert recipe_run["requests"], "no export reached the receiver" + for request in recipe_run["requests"]: + headers = request["headers"] + assert headers.get("x-api-key") == FI_API_KEY + assert headers.get("x-secret-key") == FI_SECRET_KEY + # No Traceloop bearer token and no OpenAI key on the export. + assert "authorization" not in headers + assert all(OPENAI_KEY not in value for value in headers.values()) + + +def test_resource_carries_the_project(recipe_run: dict[str, Any]) -> None: + assert recipe_run["requests"], "no export reached the receiver" + for request in recipe_run["requests"]: + assert request["resource_attributes"], "export without a resource" + for resource in request["resource_attributes"]: + assert resource.get("project_name") == PROJECT + assert resource.get("project_type") == "observe" + assert resource.get("service.name") == SERVICE_NAME + + +def test_traceloop_span_kinds_and_tree(recipe_run: dict[str, Any]) -> None: + spans = _by_name(recipe_run["spans"]) + assert set(spans) == set(SPAN_NAMES.values()) + for kind in ("workflow", "task", "agent", "tool"): + assert _attrs(spans[SPAN_NAMES[kind]]).get("traceloop.span.kind") == kind + + llm = _attrs(spans[SPAN_NAMES["llm"]]) + assert "traceloop.span.kind" not in llm + assert llm.get("gen_ai.operation.name") == "chat" + assert llm.get("gen_ai.provider.name") == "openai" + assert llm.get("gen_ai.request.model") == "gpt-4o-mini" + assert llm.get("gen_ai.response.model") == RESPONSE_MODEL + assert llm.get("gen_ai.usage.input_tokens") == USAGE["prompt_tokens"] + assert llm.get("gen_ai.usage.output_tokens") == USAGE["completion_tokens"] + assert llm.get("gen_ai.usage.total_tokens") == USAGE["total_tokens"] + # 0.62.4 does not emit the legacy llm.* model/usage keys. + assert "llm.request.model" not in llm + assert "llm.usage.prompt_tokens" not in llm + # Every span, the model call included, carries a traceloop.* key. + assert llm.get("traceloop.workflow.name") == "support_request" + for span in spans.values(): + assert any(key.startswith("traceloop.") for key in _attrs(span)), span["name"] + + # One trace: workflow > task, workflow > agent > tool, agent > model call. + assert len({span["traceId"] for span in spans.values()}) == 1 + workflow = spans[SPAN_NAMES["workflow"]] + agent_span = spans[SPAN_NAMES["agent"]] + assert not workflow.get("parentSpanId") + assert spans[SPAN_NAMES["task"]]["parentSpanId"] == workflow["spanId"] + assert agent_span["parentSpanId"] == workflow["spanId"] + assert spans[SPAN_NAMES["tool"]]["parentSpanId"] == agent_span["spanId"] + assert spans[SPAN_NAMES["llm"]]["parentSpanId"] == agent_span["spanId"] + + +def test_spans_that_carry_both_gen_ai_and_traceloop_keys(recipe_run: dict[str, Any]) -> None: + """Recorded, not resolved: one stored kind for these spans is SF-1's assertion.""" + both = set() + for name, span in _by_name(recipe_run["spans"]).items(): + keys = _attrs(span) + if any(k.startswith("gen_ai.") for k in keys) and any( + k.startswith("traceloop.") for k in keys + ): + both.add(name) + assert both == {SPAN_NAMES["agent"], SPAN_NAMES["tool"], SPAN_NAMES["llm"]} + + +def test_content_off_exports_no_content(recipe_run: dict[str, Any]) -> None: + dump = _export_dump(recipe_run) + for marker in CONTENT_MARKERS: + assert marker not in dump, marker + for span in recipe_run["spans"]: + assert not set(CONTENT_KEYS) & set(_attrs(span)), span["name"] + assert not span.get("events"), span["name"] + + +def test_no_secret_in_the_export_or_output(recipe_run: dict[str, Any]) -> None: + dump = _export_dump(recipe_run) + output = recipe_run["stdout"] + recipe_run["stderr"] + secrets = (FI_API_KEY, FI_SECRET_KEY, quote(FI_SECRET_KEY, safe=""), OPENAI_KEY) + for secret in secrets: + assert secret not in dump + assert secret not in output + + +# -------------------------------------------------------------------------- +# Controls and the documented edges. +# -------------------------------------------------------------------------- + + +def test_content_is_exported_when_trace_content_is_unset(tmp_path: Path) -> None: + """Traceloop's default is content ON. Control for the content-off test.""" + record = _run_script( + tmp_path, + APP, + [QUESTION], + lambda receiver: receiver.origin, + TRACELOOP_TRACE_CONTENT=None, + ) + _assert_ran(record) + spans = {name: _attrs(span) for name, span in _by_name(record["spans"]).items()} + llm = spans[SPAN_NAMES["llm"]] + assert "QMARK7c1e" in llm["gen_ai.input.messages"] + assert TOOL_OUTPUT in llm["gen_ai.input.messages"] # the system prompt + assert "AMARK5d2b" in llm["gen_ai.output.messages"] + assert "QMARK7c1e" in spans[SPAN_NAMES["workflow"]]["traceloop.entity.input"] + assert "AMARK5d2b" in spans[SPAN_NAMES["workflow"]]["traceloop.entity.output"] + assert TOOL_OUTPUT in spans[SPAN_NAMES["tool"]]["traceloop.entity.output"] + dump = _export_dump(record) + for secret in (FI_API_KEY, FI_SECRET_KEY, OPENAI_KEY): + assert secret not in dump + + +@pytest.mark.parametrize( + ("suffix", "expected_path"), + [ + ("/", "/v1/traces"), + # 0.62.4 does not double a suffix that is already there. + ("/v1/traces", "/v1/traces"), + # fi-collector also serves /tracer/v1/traces. + ("/tracer", "/tracer/v1/traces"), + ], +) +def test_sdk_appends_v1_traces_unless_already_present( + tmp_path: Path, suffix: str, expected_path: str +) -> None: + record = _run_script( + tmp_path, + APP, + [QUESTION], + lambda receiver: receiver.origin + suffix, + ) + _assert_ran(record) + assert {request["path"] for request in record["requests"]} == {expected_path} + assert {span["name"] for span in record["spans"]} == set(SPAN_NAMES.values()) + + +def test_unencoded_comma_in_a_header_value_truncates_it(tmp_path: Path) -> None: + """Why the recipe percent-encodes: the SDK splits TRACELOOP_HEADERS on commas.""" + raw = "x-api-key={0},x-secret-key={1}".format(FI_API_KEY, FI_SECRET_KEY) + record = _run_script( + tmp_path, + APP, + [QUESTION], + lambda receiver: receiver.origin, + TRACELOOP_HEADERS=raw, + ) + _assert_ran(record) + assert record["requests"] + for request in record["requests"]: + assert request["headers"].get("x-api-key") == FI_API_KEY + assert request["headers"].get("x-secret-key") == FI_SECRET_KEY.split(",")[0] + + +def test_app_refuses_a_traceloop_api_key(tmp_path: Path) -> None: + record = _run_script( + tmp_path, + APP, + [QUESTION], + lambda receiver: receiver.origin, + TRACELOOP_API_KEY="tl-placeholder-0000", + ) + assert not record["result"].timed_out + assert record["result"].returncode != 0 + assert "Unset TRACELOOP_API_KEY" in record["stderr"] + assert record["requests"] == [] + assert record["model_requests"] == [] + assert record["guard_attempts"] == [] + + +def test_model_call_outside_a_workflow_has_no_traceloop_key(tmp_path: Path) -> None: + record = _run_script( + tmp_path, + BARE_CALL, + [QUESTION], + lambda receiver: receiver.origin, + ) + _assert_ran(record) + spans = _by_name(record["spans"]) + assert set(spans) == {SPAN_NAMES["llm"]} + attrs = _attrs(spans[SPAN_NAMES["llm"]]) + assert attrs.get("gen_ai.operation.name") == "chat" + assert not [key for key in attrs if key.startswith("traceloop.")] + + +@pytest.mark.parametrize("metrics_enabled", [None, "false"]) +def test_metrics_go_to_v1_metrics_unless_disabled( + tmp_path: Path, metrics_enabled: Optional[str] +) -> None: + """fi-collector routes no /v1/metrics; the recipe turns metrics off.""" + tmp_path.mkdir(parents=True, exist_ok=True) + guard_log = tmp_path / "guard.jsonl" + with _PathRecorder() as recorder, FakeOpenAI(ANSWER) as fake: + env = _child_env( + tmp_path, + guard_log, + recorder.origin, + fake.base_url, + TRACELOOP_METRICS_ENABLED=metrics_enabled, + ) + result = run( + [sys.executable, str(GUARD), str(APP), QUESTION], + env=env, + stdin=None, + timeout=RUN_TIMEOUT_SECONDS, + ) + paths = recorder.paths() + assert not result.timed_out + assert result.returncode == 0, result.stderr.decode("utf-8", "replace") + assert not guard_log.exists() or guard_log.read_text() == "" + assert "/v1/traces" in paths + if metrics_enabled is None: + assert "/v1/metrics" in paths + else: + assert "/v1/metrics" not in paths + + +# -------------------------------------------------------------------------- +# Opt-in: read the backend's kind map instead of copying it. +# -------------------------------------------------------------------------- + + +def _backend_maps() -> tuple[dict[str, str], dict[str, str]]: + path = os.environ.get("FI_BACKEND_OPENLLMETRY_PY") + if not path: + pytest.skip( + "set FI_BACKEND_OPENLLMETRY_PY to futureagi/tracer/utils/adapters/openllmetry.py" + ) + tree = ast.parse(Path(path).read_text(encoding="utf-8")) + maps: dict[str, dict[str, str]] = {} + for node in tree.body: + if isinstance(node, ast.Assign) and len(node.targets) == 1: + target = node.targets[0] + if isinstance(target, ast.Name) and target.id in ( + "_TRACELOOP_KIND_MAP", + "_OPERATION_KIND_MAP", + ): + maps[target.id] = ast.literal_eval(node.value) + return maps["_TRACELOOP_KIND_MAP"], maps["_OPERATION_KIND_MAP"] + + +def test_emitted_kinds_are_keys_of_the_backend_map(recipe_run: dict[str, Any]) -> None: + kind_map, operation_map = _backend_maps() + attrs = [_attrs(span) for span in recipe_run["spans"]] + kinds = {a["traceloop.span.kind"] for a in attrs if "traceloop.span.kind" in a} + operations = {a["gen_ai.operation.name"] for a in attrs if "gen_ai.operation.name" in a} + assert kinds == {"workflow", "task", "agent", "tool"} + assert kinds <= set(kind_map) + assert operations == {"chat"} + assert operations <= set(operation_map) + + +def test_readme_kind_table_matches_the_backend_map() -> None: + kind_map, _operation_map = _backend_maps() + text = README.read_text(encoding="utf-8") + rows = dict(re.findall(r"^\| `([a-z_]+)` \| ([A-Z_]+) \|$", text, flags=re.MULTILINE)) + assert rows == kind_map diff --git a/python/tests/harness/__init__.py b/python/tests/harness/__init__.py new file mode 100644 index 000000000..cf79047f1 --- /dev/null +++ b/python/tests/harness/__init__.py @@ -0,0 +1,238 @@ +"""Small, loopback-only utilities for OTLP contract tests.""" + +from __future__ import annotations + +import copy +import difflib +import json +import subprocess +import threading +from http import HTTPStatus +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from typing import Any, Mapping, Optional, Sequence, Union +from urllib.parse import urlsplit +from urllib.request import Request, urlopen + + +class Receiver: + """An in-memory OTLP/HTTP receiver for tests. + + The receiver only listens on loopback and never forwards received spans. + """ + + def __init__(self) -> None: + self._spans: list[dict[str, Any]] = [] + self._requests: list[dict[str, Any]] = [] + self._lock = threading.Lock() + owner = self + + class Handler(BaseHTTPRequestHandler): + def _read_body(self) -> bytes: + # The Node OTLP exporter streams with Transfer-Encoding: chunked + # and sends no Content-Length, so both framings are accepted. + if "chunked" in self.headers.get("Transfer-Encoding", "").lower(): + chunks = [] + while True: + size_line = self.rfile.readline().split(b";", 1)[0].strip() + size = int(size_line, 16) + if size == 0: + while self.rfile.readline() not in (b"\r\n", b"\n", b""): + pass + return b"".join(chunks) + chunks.append(self.rfile.read(size)) + self.rfile.readline() + length = int(self.headers.get("Content-Length", "0")) + return self.rfile.read(length) + + def do_POST(self) -> None: # noqa: N802 - required by BaseHTTPRequestHandler + path = urlsplit(self.path).path + if path not in ("/v1/traces", "/tracer/v1/traces"): + self.send_error(HTTPStatus.NOT_FOUND) + return + + try: + body = self._read_body() + content_type = self.headers.get("Content-Type", "").lower() + + if content_type.startswith("application/json"): + request = json.loads(body.decode("utf-8")) + elif content_type.startswith(("application/x-protobuf", "application/protobuf")): + from google.protobuf.json_format import MessageToDict + from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ( + ExportTraceServiceRequest, + ) + + protobuf_request = ExportTraceServiceRequest() + protobuf_request.ParseFromString(body) + request = MessageToDict(protobuf_request) + else: + self.send_error(HTTPStatus.UNSUPPORTED_MEDIA_TYPE) + return + + spans: list[dict[str, Any]] = [] + resource_attributes: list[dict[str, Any]] = [] + for resource_spans in request.get( + "resourceSpans", request.get("resource_spans", []) + ): + resource_attributes.append( + _flatten_attributes( + resource_spans.get("resource", {}).get("attributes", []) + ) + ) + for scope_spans in resource_spans.get( + "scopeSpans", resource_spans.get("scope_spans", []) + ): + spans.extend(scope_spans.get("spans", [])) + except (ImportError, TypeError, UnicodeDecodeError, ValueError) as error: + self.send_error(HTTPStatus.BAD_REQUEST, str(error)) + return + + with owner._lock: + owner._spans.extend(copy.deepcopy(spans)) + owner._requests.append( + { + "path": path, + "headers": { + key.lower(): value for key, value in self.headers.items() + }, + "resource_attributes": resource_attributes, + } + ) + self.send_response(HTTPStatus.OK) + self.send_header("Content-Length", "0") + self.end_headers() + + def log_message(self, _format: str, *args: Any) -> None: + return + + self._server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + self.origin = "http://127.0.0.1:{0}".format(self._server.server_port) + self.endpoint = "{0}/v1/traces".format(self.origin) + self.collector_endpoint = "{0}/tracer/v1/traces".format(self.origin) + self._thread = threading.Thread(target=self._server.serve_forever, daemon=True) + self._thread.start() + + def __enter__(self) -> "Receiver": + return self + + def __exit__(self, _type: Any, _value: Any, _traceback: Any) -> None: + self._server.shutdown() + self._server.server_close() + self._thread.join() + + def spans(self) -> list[dict[str, Any]]: + """Return a copy of the decoded spans received so far.""" + with self._lock: + return copy.deepcopy(self._spans) + + def requests(self) -> list[dict[str, Any]]: + """Return one record per accepted export: path, lower-cased headers, resource attributes.""" + with self._lock: + return copy.deepcopy(self._requests) + + def clear(self) -> None: + """Remove every decoded span and request record received so far.""" + with self._lock: + self._spans.clear() + self._requests.clear() + + +def _flatten_attributes(attributes: Sequence[Mapping[str, Any]]) -> dict[str, Any]: + """Turn OTLP key/value attributes into a plain dict of scalar values.""" + flat: dict[str, Any] = {} + for attribute in attributes: + value = attribute.get("value", {}) + for kind in ("stringValue", "boolValue", "intValue", "doubleValue"): + if kind in value: + flat[attribute.get("key", "")] = value[kind] + break + else: + flat[attribute.get("key", "")] = value + return flat + + +def run( + argv: Sequence[str], + env: Mapping[str, str], + stdin: Optional[bytes], + timeout: float, +) -> subprocess.CompletedProcess[bytes]: + """Run one process and return its captured output, including after a timeout.""" + process = subprocess.Popen( + argv, + env=dict(env), + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + timed_out = False + try: + stdout, stderr = process.communicate(input=stdin, timeout=timeout) + except subprocess.TimeoutExpired: + timed_out = True + process.kill() + stdout, stderr = process.communicate() + + result = subprocess.CompletedProcess(argv, process.returncode, stdout, stderr) + result.timed_out = timed_out + return result + + +def post_otlp(spans: dict[str, Any], endpoint: str) -> int: + """Post one JSON-encoded OTLP request to a loopback receiver.""" + parsed = urlsplit(endpoint) + if parsed.scheme != "http" or parsed.hostname != "127.0.0.1": + raise ValueError("post_otlp only sends to a loopback HTTP endpoint") + + request = Request( + endpoint, + data=json.dumps(spans).encode("utf-8"), + headers={"Content-Type": "application/json"}, + method="POST", + ) + with urlopen(request, timeout=5) as response: + return response.status + + +def compare(actual: Sequence[dict[str, Any]], golden_path: Union[str, Path]) -> None: + """Fail with a diff unless span names, attributes, and statuses match a golden.""" + with Path(golden_path).open(encoding="utf-8") as golden_file: + expected = json.load(golden_file) + + def canonical(spans: Sequence[dict[str, Any]]) -> list[dict[str, Any]]: + normalized = [] + for span in spans: + attributes = span.get("attributes", []) + if isinstance(attributes, dict): + attributes = [ + {"key": key, "value": value} for key, value in attributes.items() + ] + normalized.append( + { + "name": span.get("name", ""), + "attributes": sorted( + attributes, + key=lambda attribute: json.dumps( + attribute, sort_keys=True, separators=(",", ":") + ), + ), + "status": span.get("status", {}), + } + ) + return normalized + + actual_normalized = canonical(actual) + expected_normalized = canonical(expected) + if actual_normalized != expected_normalized: + diff = "\n".join( + difflib.unified_diff( + json.dumps(expected_normalized, indent=2, sort_keys=True).splitlines(), + json.dumps(actual_normalized, indent=2, sort_keys=True).splitlines(), + fromfile=str(golden_path), + tofile="actual", + lineterm="", + ) + ) + print(diff) + raise AssertionError("spans do not match golden") diff --git a/python/tests/harness/test_harness.py b/python/tests/harness/test_harness.py new file mode 100644 index 000000000..bd61d5adc --- /dev/null +++ b/python/tests/harness/test_harness.py @@ -0,0 +1,189 @@ +"""Contract tests for the shared loopback-only harness.""" + +import json +import sys + +import pytest + +from tests.harness import Receiver, compare, post_otlp, run + + +def test_receiver_stores_posted_span_and_can_clear() -> None: + payload = { + "resourceSpans": [ + { + "scopeSpans": [ + { + "spans": [ + { + "traceId": "trace-id", + "spanId": "span-id", + "name": "harness.receiver", + "attributes": [ + {"key": "component", "value": {"stringValue": "test"}} + ], + "status": {"code": 1}, + } + ] + } + ] + } + ] + } + + with Receiver() as receiver: + assert post_otlp(payload, receiver.endpoint) == 200 + assert receiver.spans() == payload["resourceSpans"][0]["scopeSpans"][0]["spans"] + receiver.clear() + assert receiver.spans() == [] + + +def test_receiver_accepts_the_collector_path() -> None: + payload = { + "resourceSpans": [ + {"scopeSpans": [{"spans": [{"name": "collector.path"}]}]} + ] + } + + with Receiver() as receiver: + assert post_otlp(payload, receiver.collector_endpoint) == 200 + assert receiver.spans() == [{"name": "collector.path"}] + + +def _post_chunked(url: str, body: bytes, headers: dict) -> int: + """Send a body with Transfer-Encoding: chunked, as the Node OTLP exporter does.""" + import http.client + from urllib.parse import urlsplit + + parts = urlsplit(url) + connection = http.client.HTTPConnection(parts.hostname, parts.port, timeout=5) + try: + chunks = [body[: len(body) // 2], body[len(body) // 2 :]] + connection.request( + "POST", + parts.path, + body=iter(chunks), + headers={**headers, "Transfer-Encoding": "chunked"}, + encode_chunked=True, + ) + return connection.getresponse().status + finally: + connection.close() + + +def test_receiver_decodes_a_chunked_body() -> None: + payload = { + "resourceSpans": [ + {"scopeSpans": [{"spans": [{"name": "chunked.body"}]}]} + ] + } + + with Receiver() as receiver: + status = _post_chunked( + receiver.collector_endpoint, + json.dumps(payload).encode("utf-8"), + {"Content-Type": "application/json"}, + ) + assert status == 200 + assert receiver.spans() == [{"name": "chunked.body"}] + + +def test_receiver_records_path_headers_and_resource_attributes() -> None: + payload = { + "resourceSpans": [ + { + "resource": { + "attributes": [ + {"key": "project_name", "value": {"stringValue": "harness"}}, + {"key": "project_type", "value": {"stringValue": "observe"}}, + ] + }, + "scopeSpans": [{"spans": [{"name": "with.resource"}]}], + } + ] + } + + with Receiver() as receiver: + status = _post_chunked( + receiver.collector_endpoint, + json.dumps(payload).encode("utf-8"), + { + "Content-Type": "application/json", + "X-Api-Key": "test-key", + "X-Secret-Key": "test-secret", + }, + ) + assert status == 200 + [request] = receiver.requests() + assert request["path"] == "/tracer/v1/traces" + assert request["headers"]["x-api-key"] == "test-key" + assert request["headers"]["x-secret-key"] == "test-secret" + assert "authorization" not in request["headers"] + assert request["resource_attributes"] == [ + {"project_name": "harness", "project_type": "observe"} + ] + receiver.clear() + assert receiver.requests() == [] + + +def test_run_captures_output_and_kills_timed_out_process() -> None: + completed = run( + [ + sys.executable, + "-c", + "import sys; print('stdout'); sys.stderr.write('stderr\\n')", + ], + env={}, + stdin=b"", + timeout=1, + ) + assert completed.returncode == 0 + assert completed.stdout == b"stdout\n" + assert completed.stderr == b"stderr\n" + assert completed.timed_out is False + + timed_out = run( + [sys.executable, "-c", "import time; time.sleep(30)"], + env={}, + stdin=b"", + timeout=0.1, + ) + assert timed_out.timed_out is True + assert timed_out.returncode != 0 + + +def test_compare_accepts_normalized_match_and_prints_mismatch_diff(tmp_path, capsys) -> None: + golden = [ + { + "traceId": "golden-trace", + "spanId": "golden-span", + "startTimeUnixNano": "1", + "endTimeUnixNano": "2", + "name": "harness.compare", + "attributes": [ + {"key": "alpha", "value": {"stringValue": "a"}}, + {"key": "beta", "value": {"intValue": "2"}}, + ], + "status": {"code": 1}, + } + ] + golden_path = tmp_path / "spans.golden.json" + golden_path.write_text(json.dumps(golden), encoding="utf-8") + + actual = [ + { + "traceId": "other-trace", + "spanId": "other-span", + "startTimeUnixNano": "3", + "endTimeUnixNano": "4", + "name": "harness.compare", + "attributes": list(reversed(golden[0]["attributes"])), + "status": {"code": 1}, + } + ] + compare(actual, golden_path) + + actual[0]["name"] = "harness.mismatch" + with pytest.raises(AssertionError, match="spans do not match golden"): + compare(actual, golden_path) + assert "---" in capsys.readouterr().out