diff --git a/python/examples/gradio/README.md b/python/examples/gradio/README.md new file mode 100644 index 000000000..3528e1281 --- /dev/null +++ b/python/examples/gradio/README.md @@ -0,0 +1,393 @@ +# Gradio chat → Future AGI (recipe) + +There is no `traceai-gradio` package. Do not install one. Gradio emits no +GenAI spans: no Python module in the `gradio` 6.29.1 wheel imports +OpenTelemetry. The span you see comes from `traceAI-openai` (import +`traceai_openai`) instrumenting the OpenAI client that the chat function +calls. With that instrumentor removed, the same Gradio turn exports zero +spans (tested). + +Pinned: `gradio==6.29.1`, the latest release on PyPI on 2026-10-05 +(`Requires-Python: >=3.10`). Its wheel METADATA declares +`License-Expression: Apache-2.0` and ships a `LICENSE` file. It is a +dependency of this example only, not of any traceAI package. `openai` is +pinned to 3.24.0, the version the tests ran with. + +The tests (see [Tests](#tests)) pass on Python 3.10, 3.11, 3.12 and 3.13 +against loopback fakes only: no model key, no live Future AGI project. One +opt-in test also installs `requirements.txt` from PyPI. +`fi-instrumentation-otel` and `traceAI-openai` declare Python <3.14. +Statements marked "source reading" come from the pinned code and are not +exercised by the tests. + +## Run + +```bash +cd python/examples/gradio +pip install -r requirements.txt + +export FI_API_KEY="YOUR_API_KEY" # sent as the X-Api-Key header +export FI_SECRET_KEY="YOUR_SECRET_KEY" # sent as the X-Secret-Key header +export FI_PROJECT_NAME="my-chatbot" +export OPENAI_API_KEY="YOUR_OPENAI_KEY" # optional: OPENAI_BASE_URL, OPENAI_MODEL + +python src/app.py +``` + +Open the URL Gradio prints and chat. Each turn is one LLM span in the +`my-chatbot` project. `OPENAI_BASE_URL` points the client at any +OpenAI-compatible host; the tests use a loopback one. + +## The recipe + +[`src/app.py`](src/app.py), in full apart from imports, `MODEL`, docstrings +and comments: + +```python +def init_tracing(trace_content: bool = False): + provider = register( + project_name=os.environ["FI_PROJECT_NAME"], + project_type=ProjectType.OBSERVE, + verbose=False, + ) + if trace_content: + config = TraceConfig() + else: + config = TraceConfig(hide_inputs=True, hide_outputs=True) + OpenAIInstrumentor().instrument(tracer_provider=provider, config=config) + return provider + + +client = OpenAI() # reads OPENAI_API_KEY and OPENAI_BASE_URL + + +def _text(content) -> str: + if isinstance(content, str): + return content + return "".join(part["text"] for part in content if part.get("type") == "text") + + +_SESSION_KEY = globals().get("_SESSION_KEY") or secrets.token_bytes(32) + + +def _session_id(request: gr.Request | None) -> str | None: + if request is None or not request.session_hash: + return None + digest = hmac.new(_SESSION_KEY, request.session_hash.encode("utf-8"), hashlib.sha256) + return digest.hexdigest()[:32] + + +def predict(message: str, history: list, request: gr.Request = None) -> str: + messages = [{"role": m["role"], "content": _text(m["content"])} for m in history] + messages.append({"role": "user", "content": message}) + session_id = _session_id(request) + with using_session(session_id) if session_id else nullcontext(): + response = client.chat.completions.create(model=MODEL, messages=messages) + return response.choices[0].message.content or "" + + +demo = gr.ChatInterface(predict, analytics_enabled=False) + +if __name__ == "__main__": + init_tracing() + demo.launch() +``` + +- **`register()` at startup.** It exports OTLP/HTTP protobuf to + `/tracer/v1/traces` (default origin + `https://api.futureagi.com`) with the `X-Api-Key` and `X-Secret-Key` + headers, and stamps every batch with the resource attributes + `project_name` and `project_type=observe`. fi-collector rejects a batch + whose resource has no `project_name` (source reading). Spans are batched; + `register()` flushes the batch on normal exit (tested: with a 10-minute + batch delay every span still arrived) and on SIGINT/SIGTERM, so Ctrl+C on + `python src/app.py` flushes too (checked in a live run: every turn was + exported after SIGINT). +- **Content off.** traceAI records prompts and answers by default + (`TraceConfig`'s `hide_inputs` and `hide_outputs` default to `False`). The + recipe passes `hide_inputs=True, hide_outputs=True`. See [Privacy](#privacy). +- **Session.** Gradio passes a `gr.Request` to any chat function that + declares one. Its `session_hash` identifies one browser session (one page + load), not a user, and it is not a Future AGI user id. It is also a Gradio + session key: any client that sends it can read that session's live state + back from the running app, for example through + `/gradio_api/deep_link?session_hash=...`. If the app sets `auth`, the + client has to be logged in, but as any user, not the session's (source + reading of Gradio 6.29.1's `routes.py`, `deep_link` and `login_check`). + So the recipe deliberately does not export it. + `_session_id` exports an HMAC-SHA256 of it instead, keyed with 32 random + bytes drawn once per process, cut to 32 hex characters. Passed to + `using_session`, that id goes on every turn of the session as `session.id` + and `gen_ai.conversation.id`: the same within one session, different + across sessions, and not the hash, which appears nowhere in the export + (tested). Without a session (`predict` called with no request, or a + request with no hash), turns carry no session id. +- **Why a keyed digest.** Gradio's browser client makes the hash with + `Math.random().toString(36)` (in the 6.29.1 frontend bundle), so it has at + most 53 random bits, one JavaScript double. A plain SHA-256 of it could be + reversed by trying every value, for all sessions in one pass. Without the + key, which never leaves the process, the id cannot be computed from a + hash. The cost: each process has its own key, so the same hash gets + another id after a restart (tested). Gradio's reload mode re-runs the file + in the same namespace; `globals().get` keeps the first key, so open + sessions keep their id (tested). +- **History.** Gradio 6 passes each history message's `content` as a list of + parts (`[{"type": "text", "text": ...}]`). `_text` turns it back into the + plain string the Chat Completions API takes (tested with Gradio's own + history round trip). +- **`analytics_enabled=False`** turns off Gradio's own usage analytics. + Without it, building the app contacts `api.gradio.app` (tested). This is + unrelated to tracing. +- **Order.** The client is created at import, before `init_tracing()`. That + works: the instrumentor wraps `OpenAI.request` on the class, so existing + clients are traced too (tested). +- **No `print`.** Nothing prints or logs the user's message (tested). + +## What is traced + +One span per chat turn, from `traceai_openai`: + +| Span | Parent | Attributes | +|---|---|---| +| `ChatCompletion` | none: each turn is its own trace | `gen_ai.span.kind=LLM`, `gen_ai.provider.name=openai`, `gen_ai.request.model`, `gen_ai.request.parameters`, `gen_ai.usage.input_tokens`, `gen_ai.usage.output_tokens`, `gen_ai.usage.total_tokens`, `gen_ai.tool.definitions` (`null` here), `input.value` and `output.value` (both `__REDACTED__`), and, with a Gradio session, `session.id` and `gen_ai.conversation.id` (both the session digest, not Gradio's session hash) | + +- fi-collector reads the span kind from `gen_ai.span.kind` and stores this + span as `llm`. It promotes `gen_ai.request.model`, `gen_ai.provider.name` + and the `gen_ai.usage.*` token counts, and reads `session.id` for the + session. The tests use the harness receiver, not fi-collector; a live run + against a local Future AGI stack (see [Out of scope](#out-of-scope)) + stored each turn as one `llm` trace with model and token counts, and + Observe's Sessions view showed the two turns of one browser session as + one session and a second browser session as another. +- `gen_ai.request.model` holds the model the response names + (`gpt-4o-mini-2024-07-18` from the test fake), not the requested + `gpt-4o-mini`; the requested one is in `gen_ai.request.parameters`. +- `openai` is the OpenAI instrumentor's provider label, not Gradio's. It was + `openai` for the loopback test host too. +- **A failed model call** still gives one `ChatCompletion` span, with status + `ERROR`, exported like the others (by the exit flush in the test), with + `input.value` `__REDACTED__` and no message text, and `predict` raises the + OpenAI error (`openai.InternalServerError` for a 500). Tested with the + model host answering HTTP 500 and the client's retries turned off + (`max_retries=0`). With the default retries, a 400 for an unknown model, + which the client does not retry, also gave one `ERROR` span in the live + run; a retried 5xx is not tested. + +## Privacy + +With the recipe as written, no message text reaches Future AGI: +`input.value` and `output.value` are `__REDACTED__`, and there are no +`gen_ai.input.messages.*` or `gen_ai.output.messages.*` keys. The test puts +markers in the user's message and the model's answer and finds neither in +the export. + +To turn content on, call `init_tracing(trace_content=True)`. **Warning:** the +span then carries the whole conversation that `predict` sends, on every turn: +`gen_ai.input.messages..message.role` and `.content` for each history +message and the new one, `gen_ai.output.messages.0.message.*` and +`output.value` for the answer, and `input.value`, which holds the text of the +request's first message (on later turns, an earlier user message, not the +latest). Tested. With content on, traceAI's `FI_HIDE_INPUTS` and +`FI_HIDE_OUTPUTS` environment variables apply again (source reading, +`fi_instrumentation/instrumentation/config.py`). + +Not covered by the switch: when the model call raises, the span records the +exception type, message and stack trace as an `exception` event, and the +type and message in the status description. For an HTTP error the message +includes the error body the model host returned (tested with a 500 whose +body holds no message text; a host that echoes the prompt in its error body +would put it in the span). + +No Future AGI key, secret or OpenAI key appears in any exported span or +resource, or in the app's output (tested). The keys travel only as request +headers: the Future AGI keys to fi-collector, the OpenAI key to the model +host. Gradio's session hash is not exported either (see Session above; +tested). + +## Gradio's thread pool and a turn span + +The recipe adds no span of its own, so the OpenAI instrumentor is the only +source. If you want one span per turn around the model call, wrap `predict` +with traceAI's manual API: + +```python +from fi_instrumentation import FITracer + +tracer = FITracer(provider.get_tracer(__name__)) # provider = init_tracing() + + +def predict_in_a_turn(message, history, request: gr.Request = None): + with tracer.start_as_current_span("chat_turn", fi_span_kind="chain"): + return predict(message, history, request) +``` + +Does the LLM span stay under it when Gradio runs the function? Tested +through Gradio 6.29.1's own event dispatch (the ChatInterface submit event, +called with `Blocks.call_function`; no server): + +- **Plain function: yes.** Gradio runs a sync chat function in an anyio + worker thread (`gradio/chat_interface.py:940`), and anyio 4.15.1 runs it + in a copy of the caller's context (`anyio/_backends/_asyncio.py:2698`, + `:1100`). A turn span opened inside the function, or around the dispatch + on the event loop, is the LLM span's parent, in one trace. No context + bridge is needed, and none is added. +- **Generator function: no, after the first `yield`.** Gradio steps a sync + generator one item per worker call (`gradio/utils.py:897-910`), each in a + fresh copy of the event loop's context. A span opened inside the + generator is current only until the first `yield`. A model call made + after it becomes a root span in another trace, and OpenTelemetry logs + `Failed to detach context` when the `with` block closes. The recipe does + not stream. If you turn `predict` into a generator, make the model call + before the first `yield` or do not wrap the generator in a turn span. + traceAI has no context-bridge helper for this case. + +Streaming responses (`stream=True`) are the OpenAI instrumentor's existing +behaviour (`python/frameworks/openai/traceai_openai/_stream.py`); this recipe +does not test them. + +## Gradio reload mode + +`gradio src/app.py` (reload mode) does not restart the process when you +save. Gradio 6.29.1 runs the file again in the same process, on a watcher +thread (`gradio/utils.py`, `watchfn`): once at startup, before any save, and +again after each save. So `init_tracing()` runs twice per launch even if you +never save, and once more per save: + +- Each of those runs' `register()` builds another provider with its own + batch queue and logs `Failed to register signal handlers: ... This is + expected when running in non-main threads.` Its + `OpenAIInstrumentor().instrument()` is ignored with the warning + `Attempting to instrument while already instrumented`. So both warnings + appear once at startup and once per save. Spans keep going to the first + provider, with the first `TraceConfig`; the other providers export + nothing. +- **Edits to the tracing setup do not take effect until you restart.** In + the tested replay, an edit that turned content on was saved and reloaded, + and the next turn still exported no content. +- In that replay no span was dropped: the turns before and after the reload + were exported at exit by the first provider. +- The session key survives the re-runs, so an open session keeps its id + (tested). + +A batch is lost only if the process ends without the exit flush. +`register()` flushes on normal exit, SIGINT and SIGTERM. SIGKILL, or the +SIGHUP a closed terminal sends, skips it (source reading of +`fi_instrumentation/otel.py`, `setup_signal_handlers`; not tested). The +startup run leaves one idle provider from the start, and each save one +more, until the process exits (that they stay until exit is source +reading). + +The replay re-runs `src/app.py` the way `watchfn` does: once before the +first turn, as at startup, and once between the two turns, as after a save. +The tests count both warnings twice. It is not a `gradio src/app.py` run: +the tests open no server port. + +## Troubleshooting + +- **No span.** `init_tracing()` did not run. Here it runs under + `if __name__ == "__main__":`; if you import `demo` from another module + (for example to mount it in FastAPI), call `init_tracing()` there. Gradio + will not emit the span for you. +- **A span labelled Gradio.** There isn't one. The span comes from + `traceai_openai`. +- **Turns not grouped by session.** `predict` ran without a `gr.Request`, + so no session id was set. +- **`session.id` is not Gradio's session hash.** By design: it is a keyed + digest of the hash (see Session above). +- **Startup warnings in reload mode.** `Failed to register signal handlers` + and `Attempting to instrument while already instrumented` come from + Gradio's startup re-run of the file. In the tested replay the spans still + went to the first provider; see [Gradio reload mode](#gradio-reload-mode). +- **A tracing change did nothing after a save.** Reload mode keeps the + first instrumentation. Restart the process. + +## Out of scope + +`gradio[mcp]`, Gradio's queue metrics and Spaces hosting. The tests do not +exercise Gradio's HTTP and queue layer (they call its event dispatch +directly), streaming, model errors with the client's default retries, or a +real fi-collector (authentication, project stamping and storage). A separate +live run on 2026-10-11 covered the HTTP and queue layer and fi-collector: +`python src/app.py` as in [Run](#run), chatted with from a headless +browser, a real model behind an OpenAI-compatible host, and a local Future +AGI stack built from future-agi `dev`. Streaming stays untested. + +## Tests + +`tests/test_gradio_recipe.py` has two parts, plus one opt-in test. None +opens a browser or a Gradio server port. + +- **In-process.** `src/app.py` is imported and `register` is swapped for a + provider with an `InMemorySpanExporter`; the recipe's own `init_tracing()` + installs the instrumentor. The tests call `predict` directly and through + Gradio's event dispatch: one LLM span per turn, the session digest (equal + to the HMAC the recipe describes, shared within a session, different + across sessions, never the hash) only when the turn has a Gradio session, + zero spans with the instrumentor removed (with that provider also set as + the global one), Gradio's history format, and the turn-span results + above. +- **Contract.** `tests/drive_turns.py` runs two turns with the real + `register()` in a subprocess. `tests/_guarded_run.py` blocks and logs + every non-loopback connection; a positive control + (`tests/_guard_probe.py`) proves it refuses and logs IPv4, IPv6 and DNS + attempts. The model is a loopback fake of the Chat Completions API + (`tests/_fake_openai.py`), and spans go to the shared harness `Receiver` + (`python/tests/harness`), which serves `/v1/traces` and + `/tracer/v1/traces` like fi-collector but does not authenticate or store + anything. The tests check the request path, both key headers, the + resource (`project_name`, `project_type=observe`), the exit flush, the + span attributes, content off and (as a control) on, that no key and no + session hash reaches the export or the output, a new session key per + process, that the prompt is never printed, a model error (the fake + answers HTTP 500), the reload replay and the Gradio analytics probe + (`tests/_analytics_probe.py`). +- **Opt-in: the published releases.** With `GRADIO_RECIPE_PUBLISHED_PINS=1`, + one more test creates a fresh virtualenv, runs + `pip install -r requirements.txt` from PyPI, checks that + `fi_instrumentation` and `traceai_openai` load from that virtualenv at + `fi-instrumentation-otel` 1.1.0 and `traceAI-openai` 0.1.10, and runs the + same two turns into the `Receiver`: one export at `/tracer/v1/traces` with + both keys, two LLM spans with one session digest, content off, no key or + session hash in the export. It needs network access to PyPI and is + skipped otherwise. + +All keys are placeholders. From the repository root, Python 3.11: + +```bash +env -u PYTHONPATH PYTHONDONTWRITEBYTECODE=1 \ + PYTHONPATH="python:python/frameworks/openai:python/tests" \ + uv run --no-project --python 3.11 \ + --with 'pytest==9.1.1' --with 'gradio==6.29.1' --with 'anyio==4.15.1' \ + --with 'openai==3.24.0' --with 'opentelemetry-api==1.45.0' \ + --with 'opentelemetry-sdk==1.45.0' \ + --with 'opentelemetry-exporter-otlp-proto-http==1.45.0' \ + --with 'opentelemetry-proto==1.45.0' \ + --with 'opentelemetry-instrumentation==0.66b0' --with 'wrapt==1.17.3' \ + --with 'requests==2.34.2' --with 'jsonschema==4.26.0' \ + --with 'pydantic==2.13.5' --with 'protobuf==7.36.2' \ + pytest python/examples/gradio/tests -q -p no:cacheprovider \ + --noconftest -o addopts= -rfEs +``` + +For Python 3.13, replace `--python 3.11`. A full run takes under a minute. +The thread-pool results depend on anyio, so it is pinned with Gradio. + +In that command, `register()` and the instrumentor come from this +repository's `python/fi_instrumentation` and `python/frameworks/openai`, not +from the releases pinned in `requirements.txt`. The published +`traceAI-openai` 0.1.10 wheel's `traceai_openai` package is byte-identical to +`python/frameworks/openai/traceai_openai`. The published +`fi-instrumentation-otel` 1.1.0 differs from `python/fi_instrumentation` +only in shutdown handling (log calls instead of `print`, a lock, and its +exporters also calling the base `shutdown()`) and one extra provider enum +value. + +The opt-in test runs the published releases instead. Add +`GRADIO_RECIPE_PUBLISHED_PINS=1` after `env -u PYTHONPATH` in the command +above. On 2026-10-05 it passed on Python 3.10, 3.11, 3.12 and 3.13; pip +resolved `wrapt` 1.17.3, `anyio` 4.15.1 and OpenTelemetry 1.45.0 (API, SDK, +OTLP/HTTP exporter) with `opentelemetry-instrumentation` 0.66b0. Run it +again when `requirements.txt` changes. + +No CI job runs this example. Re-run the command above when Gradio, anyio or +the OpenAI instrumentor changes. diff --git a/python/examples/gradio/requirements.txt b/python/examples/gradio/requirements.txt new file mode 100644 index 000000000..b8d08fe18 --- /dev/null +++ b/python/examples/gradio/requirements.txt @@ -0,0 +1,9 @@ +# Not a traceAI package: these are the example's own dependencies. +gradio==6.29.1 +openai==3.24.0 # the version the tests ran with +# register() and the OpenAI instrumentor. The main tests run against this +# repository's python/fi_instrumentation and python/frameworks/openai at the +# same version numbers; an opt-in test installs this file from PyPI (see +# README.md, "Tests"). +fi-instrumentation-otel==1.1.0 +traceAI-openai==0.1.10 diff --git a/python/examples/gradio/src/app.py b/python/examples/gradio/src/app.py new file mode 100644 index 000000000..e3ee50e6a --- /dev/null +++ b/python/examples/gradio/src/app.py @@ -0,0 +1,97 @@ +"""Trace a Gradio chat app with traceAI's OpenAI instrumentor. + +Recipe example, not a package. Gradio emits no GenAI spans. The one span per +chat turn comes from ``traceai_openai`` instrumenting the OpenAI client that +``predict`` calls. This file calls ``register()`` once at startup, instruments +the OpenAI client with content capture off, and passes a keyed digest of +Gradio's session hash to ``using_session`` so the turns of one browser session +share a session id. + + python src/app.py + +FI_API_KEY, FI_SECRET_KEY, FI_PROJECT_NAME and (optionally) FI_BASE_URL are +read from the environment; so are OPENAI_API_KEY, OPENAI_BASE_URL and +OPENAI_MODEL. See README.md. +""" + +from __future__ import annotations + +import hashlib +import hmac +import os +import secrets +from contextlib import nullcontext + +import gradio as gr +from fi_instrumentation import TraceConfig, register, using_session +from fi_instrumentation.fi_types import ProjectType +from openai import OpenAI +from traceai_openai import OpenAIInstrumentor + +MODEL = os.environ.get("OPENAI_MODEL", "gpt-4o-mini") + + +def init_tracing(trace_content: bool = False): + """Register the Future AGI provider and instrument OpenAI. Call once, before the first turn.""" + provider = register( + project_name=os.environ["FI_PROJECT_NAME"], + project_type=ProjectType.OBSERVE, + verbose=False, + ) + # traceAI records prompts and answers unless told not to: TraceConfig's + # defaults are hide_inputs=False and hide_outputs=False. + if trace_content: + config = TraceConfig() + else: + config = TraceConfig(hide_inputs=True, hide_outputs=True) + OpenAIInstrumentor().instrument(tracer_provider=provider, config=config) + return provider + + +# OpenAI() reads OPENAI_API_KEY and OPENAI_BASE_URL. +client = OpenAI() + + +def _text(content) -> str: + # Gradio 6 passes each history message's content as a list of parts. + if isinstance(content, str): + return content + return "".join(part["text"] for part in content if part.get("type") == "text") + + +# Gradio's session hash identifies one browser session (one page load), not a +# user. It is also the key to that session's live state in the running app, +# so it is not exported. The span gets a digest keyed with a random +# per-process key that never leaves the process. Gradio's reload mode re-runs +# this file in the same namespace; keeping the key keeps the ids of open +# sessions. +_SESSION_KEY = globals().get("_SESSION_KEY") or secrets.token_bytes(32) + + +def _session_id(request: gr.Request | None) -> str | None: + if request is None or not request.session_hash: + return None + digest = hmac.new(_SESSION_KEY, request.session_hash.encode("utf-8"), hashlib.sha256) + return digest.hexdigest()[:32] + + +def predict(message: str, history: list, request: gr.Request = None) -> str: + messages = [{"role": m["role"], "content": _text(m["content"])} for m in history] + messages.append({"role": "user", "content": message}) + # With a Gradio session, every turn of it carries the same session.id; + # without one, each turn is an unrelated trace. + session_id = _session_id(request) + with using_session(session_id) if session_id else nullcontext(): + response = client.chat.completions.create(model=MODEL, messages=messages) + return response.choices[0].message.content or "" + + +# analytics_enabled=False turns off Gradio's own usage analytics, which +# otherwise contact api.gradio.app when the app is built. It is unrelated to +# tracing. +demo = gr.ChatInterface(predict, analytics_enabled=False) + +if __name__ == "__main__": + init_tracing() + # register() flushes the batch on normal exit and on SIGINT/SIGTERM. + demo.launch() diff --git a/python/examples/gradio/tests/_analytics_probe.py b/python/examples/gradio/tests/_analytics_probe.py new file mode 100644 index 000000000..1018754f9 --- /dev/null +++ b/python/examples/gradio/tests/_analytics_probe.py @@ -0,0 +1,15 @@ +"""A test-only probe: build a Gradio app with Gradio's default analytics setting. + +Run under ``_guarded_run.py`` with ``GRADIO_ANALYTICS_ENABLED`` unset, the +guard log shows whether building the app tries to reach Gradio's analytics +host. That is why ``src/app.py`` passes ``analytics_enabled=False``. +Not part of the recipe. +""" + +from __future__ import annotations + +import gradio as gr + +gr.ChatInterface(lambda message, history: message) +# Gradio sends its analytics from background threads; the interpreter waits +# for them before exiting. diff --git a/python/examples/gradio/tests/_fake_openai.py b/python/examples/gradio/tests/_fake_openai.py new file mode 100644 index 000000000..312c45683 --- /dev/null +++ b/python/examples/gradio/tests/_fake_openai.py @@ -0,0 +1,86 @@ +"""A loopback stand-in for the OpenAI Chat Completions API (non-streaming). + +Every POST to /v1/chat/completions gets one assistant message whose text is +``answer_text``, with fixed token usage, so the instrumentor records model +and usage attributes. With ``status`` other than 200 it answers that status +and an OpenAI-style error body instead, with no message text in it. Each +request body and Authorization header is kept for assertions. Listens on +127.0.0.1 only. +""" + +from __future__ import annotations + +import json +import threading +from http import HTTPStatus +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from typing import Any + +RESPONSE_MODEL = "gpt-4o-mini-2024-07-18" +USAGE = {"prompt_tokens": 11, "completion_tokens": 7, "total_tokens": 18} + + +class FakeOpenAI: + """Serve /v1/chat/completions on 127.0.0.1 and record each request.""" + + def __init__(self, answer_text: str, status: int = HTTPStatus.OK) -> None: + self.requests: list[dict[str, Any]] = [] + self.authorizations: list[str | None] = [] + lock = threading.Lock() + owner = self + + class Handler(BaseHTTPRequestHandler): + def do_POST(self) -> None: # noqa: N802 - BaseHTTPRequestHandler API + if self.path.rstrip("/") != "/v1/chat/completions": + self.send_error(HTTPStatus.NOT_FOUND) + return + length = int(self.headers.get("Content-Length", "0")) + request = json.loads(self.rfile.read(length).decode("utf-8")) + with lock: + owner.requests.append(request) + owner.authorizations.append(self.headers.get("Authorization")) + if status != HTTPStatus.OK: + error = {"message": "fixture failure", "type": "server_error", "code": None} + self._send_json(status, {"error": error}) + return + self._send_json( + HTTPStatus.OK, + { + "id": "chatcmpl-fake", + "object": "chat.completion", + "created": 1_700_000_000, + "model": RESPONSE_MODEL, + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": answer_text}, + "finish_reason": "stop", + } + ], + "usage": USAGE, + }, + ) + + def _send_json(self, code: int, payload: dict[str, Any]) -> None: + body = json.dumps(payload).encode("utf-8") + self.send_response(code) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, format: str, *args: Any) -> None: # noqa: A002 + return + + self._server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + self.base_url = "http://127.0.0.1:{0}/v1".format(self._server.server_port) + self._thread = threading.Thread(target=self._server.serve_forever, daemon=True) + self._thread.start() + + def __enter__(self) -> "FakeOpenAI": + return self + + def __exit__(self, *_exc: Any) -> None: + self._server.shutdown() + self._server.server_close() + self._thread.join() diff --git a/python/examples/gradio/tests/_guard_probe.py b/python/examples/gradio/tests/_guard_probe.py new file mode 100644 index 000000000..5ebeff59d --- /dev/null +++ b/python/examples/gradio/tests/_guard_probe.py @@ -0,0 +1,26 @@ +"""A test-only probe: try three non-loopback connections under the loopback guard. + +The contract test runs this through ``_guarded_run.py`` to prove the guard +really refuses and logs a non-loopback attempt, so the ``guard_attempts == []`` +assertions elsewhere cannot pass because the guard silently stopped working. +The targets are reserved and never reachable: TEST-NET-1 (RFC 5737), the IPv6 +documentation prefix (RFC 3849) and the ``.invalid`` TLD (RFC 2606). +Not part of the recipe. +""" + +from __future__ import annotations + +import socket + +refused = [] +for label, attempt in ( + ("connect", lambda: socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect(("192.0.2.1", 80))), + ("connect_ex", lambda: socket.socket(socket.AF_INET6, socket.SOCK_STREAM).connect_ex(("2001:db8::1", 80, 0, 0))), + ("getaddrinfo", lambda: socket.getaddrinfo("example.invalid", 443)), +): + try: + attempt() + except OSError as error: + if "blocked by loopback guard" in str(error): + refused.append(label) +print("refused:", ",".join(refused)) diff --git a/python/examples/gradio/tests/_guarded_run.py b/python/examples/gradio/tests/_guarded_run.py new file mode 100644 index 000000000..82f1d5cc1 --- /dev/null +++ b/python/examples/gradio/tests/_guarded_run.py @@ -0,0 +1,81 @@ +"""Run a script with every non-loopback network connection blocked and logged. + +Usage: python _guarded_run.py [args...] + +Before the script starts, ``socket.socket.connect``/``connect_ex`` and +``socket.getaddrinfo`` refuse any host other than 127.0.0.1, ::1 or +localhost. Each refused attempt is appended as a JSON line to the file named +by ``LOOPBACK_GUARD_LOG``, so a test can assert that Gradio, traceAI and the +example tried to reach nothing else (no Gradio analytics host, no real +OpenAI, no Future AGI). Unix sockets are not affected. +""" + +from __future__ import annotations + +import json +import os +import runpy +import socket +import sys + +_ALLOWED_HOSTS = {"127.0.0.1", "::1", "localhost"} +_LOG_PATH = os.environ.get("LOOPBACK_GUARD_LOG", "") + + +def _host(value: object) -> str: + if isinstance(value, bytes): + return value.decode("ascii", "replace") + return str(value) + + +def _refuse(kind: str, target: object) -> None: + if _LOG_PATH: + with open(_LOG_PATH, "a", encoding="utf-8") as log: + log.write(json.dumps({"kind": kind, "target": repr(target)}) + "\n") + + +_original_connect = socket.socket.connect +_original_connect_ex = socket.socket.connect_ex +_original_getaddrinfo = socket.getaddrinfo + + +def _is_blocked(sock: socket.socket, address: object) -> bool: + if sock.family not in (socket.AF_INET, socket.AF_INET6): + return False + host = address[0] if isinstance(address, tuple) else address + return _host(host) not in _ALLOWED_HOSTS + + +def _connect(self: socket.socket, address: object) -> None: + if _is_blocked(self, address): + _refuse("connect", address) + raise OSError("blocked by loopback guard: {0!r}".format(address)) + return _original_connect(self, address) + + +def _connect_ex(self: socket.socket, address: object) -> int: + if _is_blocked(self, address): + _refuse("connect_ex", address) + raise OSError("blocked by loopback guard: {0!r}".format(address)) + return _original_connect_ex(self, address) + + +def _getaddrinfo(host: object, *args: object, **kwargs: object): # type: ignore[no-untyped-def] + if host is not None and _host(host) not in _ALLOWED_HOSTS: + _refuse("getaddrinfo", host) + raise socket.gaierror("blocked by loopback guard: {0!r}".format(host)) + return _original_getaddrinfo(host, *args, **kwargs) + + +def main() -> None: + socket.socket.connect = _connect # type: ignore[method-assign] + socket.socket.connect_ex = _connect_ex # type: ignore[method-assign] + socket.getaddrinfo = _getaddrinfo # type: ignore[assignment] + script = sys.argv[1] + sys.argv = sys.argv[1:] + sys.path.insert(0, os.path.dirname(os.path.abspath(script))) + runpy.run_path(script, run_name="__main__") + + +if __name__ == "__main__": + main() diff --git a/python/examples/gradio/tests/drive_turns.py b/python/examples/gradio/tests/drive_turns.py new file mode 100644 index 000000000..5384313c1 --- /dev/null +++ b/python/examples/gradio/tests/drive_turns.py @@ -0,0 +1,115 @@ +"""Scenario script: run the recipe's ``predict`` for chat turns, then exit. + + python drive_turns.py [--content] [--reload] [--model-error] + +Loads ``src/app.py`` as a module, so its ``demo.launch()`` does not run and no +port is opened, then calls ``init_tracing()`` as the app's ``__main__`` block +does and calls ``predict`` twice with one Gradio session hash. Spans leave the +process only through ``register()``'s exit flush (the test sets a long batch +delay). Prints one JSON line with counts only, never the prompt. + +``--content`` calls ``init_tracing(trace_content=True)`` instead. + +``--reload`` replays Gradio 6.29.1's reload mode. Its watcher thread +(``gradio/utils.py`` ``watchfn``) runs the app's source again in the same +process: once at startup, before any save, and once after each save. Each +run is an ``exec`` of the file into the same module namespace with +``__name__ == "__main__"``, on a non-main thread marked as the reload +thread, so the app's ``__main__`` block runs again and ``demo.launch()`` +returns at once (``Blocks.launch``'s reload-thread check). This script does +the startup run before the first turn and a save run between the two turns. +The saved source carries one edit, ``init_tracing()`` -> +``init_tracing(trace_content=True)``, as if the developer had turned content +on and saved. The second turn then uses the re-created ``predict``. This +replays the reload's effect on tracing; it is not a ``gradio app.py`` run. + +``--model-error`` expects the model host to fail (the test's fake answers +HTTP 500). It turns the client's retries off and runs one turn, which must +raise ``openai.InternalServerError``. + +Prints one JSON line: the turn count and the ``project_version_id`` of the +provider ``init_tracing()`` returned first (a fresh UUID per ``register()`` +call), never the prompt. + +Not part of the recipe. +""" + +from __future__ import annotations + +import importlib.util +import json +import sys +import threading +from pathlib import Path + +import gradio as gr +import openai + +APP = Path(__file__).resolve().parents[1] / "src" / "app.py" +SESSION_HASH = "drive-session-1" + + +def load_app(): + spec = importlib.util.spec_from_file_location("gradio_recipe_app", APP) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def rerun_like_gradio(module, source: str) -> None: + """Run ``source`` into ``module`` the way Gradio's reload watcher does.""" + + def rerun() -> None: + from gradio.cli.commands.reload import reload_thread + + reload_thread.running_reload = True + module.__dict__["__name__"] = "__main__" + exec(compile(source, str(APP), "exec"), module.__dict__) + + thread = threading.Thread(target=rerun) + thread.start() + thread.join() + + +def main(argv: list[str]) -> int: + message = argv[1] + app = load_app() + provider = app.init_tracing(trace_content=True) if "--content" in argv else app.init_tracing() + request = gr.Request(session_hash=SESSION_HASH) + report = {"first_provider": provider.resource.attributes["project_version_id"]} + + if "--model-error" in argv: + app.client = app.client.with_options(max_retries=0) + try: + app.predict(message, [], request) + except openai.InternalServerError as error: + report.update(turns=1, error=type(error).__name__, status_code=error.status_code) + print(json.dumps(report)) + return 0 + raise AssertionError("the model call did not fail") + + source = APP.read_text(encoding="utf-8") + assert source.count("init_tracing()") == 1, "expected one init_tracing() call in app.py" + if "--reload" in argv: + # Gradio's watcher runs the file once at startup, before any save. + rerun_like_gradio(app, source) + + first = app.predict(message, [], request) + history = [ + {"role": "user", "content": message}, + {"role": "assistant", "content": first}, + ] + if "--reload" in argv: + # A save that turned content on. + rerun_like_gradio( + app, source.replace("init_tracing()", "init_tracing(trace_content=True)") + ) + second = app.predict(message, history, request) + + report.update(turns=2, answered=[bool(first), bool(second)]) + print(json.dumps(report)) + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/python/examples/gradio/tests/test_gradio_recipe.py b/python/examples/gradio/tests/test_gradio_recipe.py new file mode 100644 index 000000000..06b4118f3 --- /dev/null +++ b/python/examples/gradio/tests/test_gradio_recipe.py @@ -0,0 +1,844 @@ +"""Tests for the Gradio tracing recipe (``src/app.py``). + +Two parts, no browser and no Gradio server port: + +- In-process: the recipe's ``predict`` is called directly, and through + Gradio 6.29.1's own event dispatch (``Blocks.call_function`` on the + ChatInterface submit event, which runs a sync ``predict`` in an anyio + worker thread). ``register`` is swapped for a provider with an + ``InMemorySpanExporter``; the recipe's own ``init_tracing`` still installs + the instrumentor with its ``TraceConfig``. +- Contract: ``tests/drive_turns.py`` runs the recipe with the real + ``register()`` in a subprocess whose network is limited to 127.0.0.1 + (``_guarded_run.py``). The model is a loopback fake of the OpenAI Chat + Completions API; spans go to the shared harness ``Receiver``, which serves + ``/v1/traces`` and ``/tracer/v1/traces`` like fi-collector's HTTP mux but + does not authenticate, stamp projects or store anything. + +All keys are placeholders. Nothing here contacts OpenAI, Gradio's analytics +host or Future AGI. One opt-in test (``GRADIO_RECIPE_PUBLISHED_PINS=1``) +installs ``requirements.txt`` from PyPI into a fresh virtualenv; it is +skipped otherwise. +""" + +from __future__ import annotations + +import asyncio +import hashlib +import hmac +import importlib.util +import json +import logging +import os +import re +import subprocess +import sys +import threading +from http import HTTPStatus +from pathlib import Path +from types import SimpleNamespace +from typing import Any, Optional + +import gradio as gr +import pytest +from opentelemetry import trace as trace_api +from opentelemetry.sdk.trace import SpanProcessor, TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter + +from fi_instrumentation import FITracer +from harness import Receiver, run +from traceai_openai import OpenAIInstrumentor + +TESTS_DIR = Path(__file__).resolve().parent +RECIPE_DIR = TESTS_DIR.parent +PYTHON_DIR = RECIPE_DIR.parents[1] +APP = RECIPE_DIR / "src" / "app.py" +README = RECIPE_DIR / "README.md" +REQUIREMENTS = RECIPE_DIR / "requirements.txt" +DRIVER = TESTS_DIR / "drive_turns.py" +GUARD = TESTS_DIR / "_guarded_run.py" +GUARD_PROBE = TESTS_DIR / "_guard_probe.py" +ANALYTICS_PROBE = TESTS_DIR / "_analytics_probe.py" + +sys.path.insert(0, str(TESTS_DIR)) +from _fake_openai import RESPONSE_MODEL, USAGE, FakeOpenAI # noqa: E402 + +PROJECT = "gradio-recipe-contract" +FI_API_KEY = "fi-api-placeholder-0000" +FI_SECRET_KEY = "fi-secret-placeholder-0000" +OPENAI_KEY = "sk-openai-placeholder-0000" +DRIVER_SESSION = "drive-session-1" # the session hash drive_turns.py uses + +# Content markers: the user's message and the fake model's answer. +QUESTION = "QMARK3f9a what is the refund window?" +ANSWER = "AMARK8b1c refunds are accepted for 30 days." +CONTENT_MARKERS = ("QMARK3f9a", "AMARK8b1c") +CONTENT_KEY_PREFIXES = ("gen_ai.input.messages", "gen_ai.output.messages") +REDACTED = "__REDACTED__" + +LLM_SPAN = "ChatCompletion" +SESSION_KEYS = ("session.id", "gen_ai.conversation.id") +# What the recipe exports instead of a session hash: a 128-bit hex digest. +SESSION_ID = re.compile(r"[0-9a-f]{32}") +# A hang guard, not a speed check: one run imports Gradio, openai and traceAI. +RUN_TIMEOUT_SECONDS = 180 + + +# -------------------------------------------------------------------------- +# In-process: predict() with an in-memory exporter +# -------------------------------------------------------------------------- + + +class _ThreadRecorder(SpanProcessor): + """Remember which thread started each span.""" + + def __init__(self) -> None: + self.threads: dict[int, int] = {} + + def on_start(self, span: Any, parent_context: Any = None) -> None: + self.threads[span.context.span_id] = threading.get_ident() + + +@pytest.fixture(scope="module") +def recipe() -> Any: + exporter = InMemorySpanExporter() + threads = _ThreadRecorder() + provider = TracerProvider() + provider.add_span_processor(threads) + provider.add_span_processor(SimpleSpanProcessor(exporter)) + # Global as well, so a span made through the global OpenTelemetry API by + # anything else in the process (Gradio included) would be exported too. + trace_api.set_tracer_provider(provider) + assert trace_api.get_tracer_provider() is provider + + with FakeOpenAI(ANSWER) as fake, pytest.MonkeyPatch.context() as patch: + patch.setenv("OPENAI_API_KEY", OPENAI_KEY) + patch.setenv("OPENAI_BASE_URL", fake.base_url) + patch.setenv("FI_PROJECT_NAME", PROJECT) + spec = importlib.util.spec_from_file_location("gradio_recipe_app", APP) + app = importlib.util.module_from_spec(spec) + spec.loader.exec_module(app) + register_calls: list[dict[str, Any]] = [] + + def in_memory_register(**kwargs: Any) -> TracerProvider: + register_calls.append(kwargs) + return provider + + patch.setattr(app, "register", in_memory_register) + app.init_tracing() + yield SimpleNamespace( + app=app, + exporter=exporter, + threads=threads.threads, + fake=fake, + provider=provider, + register_calls=register_calls, + tracer=FITracer(provider.get_tracer("gradio-recipe-test")), + ) + OpenAIInstrumentor().uninstrument() + + +@pytest.fixture(autouse=True) +def _fresh(request: pytest.FixtureRequest) -> None: + if "recipe" in request.fixturenames: + state = request.getfixturevalue("recipe") + state.exporter.clear() + state.fake.requests.clear() + state.fake.authorizations.clear() + + +def _attrs(span: Any) -> dict[str, Any]: + return dict(span.attributes or {}) + + +def _session_digest(app: Any, session_hash: str) -> str: + """The session id the recipe should export for a Gradio session hash.""" + digest = hmac.new(app._SESSION_KEY, session_hash.encode("utf-8"), hashlib.sha256) + return digest.hexdigest()[:32] + + +def _submit_event(demo: gr.Blocks) -> Any: + """The ChatInterface's textbox-submit event, which calls predict.""" + (event,) = [fn for fn in demo.fns.values() if fn.api_name == "_submit_fn"] + return event + + +async def _gradio_turn( + demo: gr.Blocks, message: str, history: list, session_hash: str +) -> dict[str, Any]: + """One turn through Gradio's own dispatch, as the queue would run it.""" + return await demo.call_function( + _submit_event(demo), + [message, history], + requests=gr.Request(session_hash=session_hash), + ) + + +def test_each_turn_exports_exactly_one_llm_span(recipe: Any) -> None: + session = gr.Request(session_hash="tab-1") + for turn in range(1, 4): + assert recipe.app.predict(QUESTION, [], session) == ANSWER + assert len(recipe.exporter.get_finished_spans()) == turn + spans = recipe.exporter.get_finished_spans() + assert len(recipe.fake.requests) == 3 + for span in spans: + assert span.name == LLM_SPAN + assert _attrs(span)["gen_ai.span.kind"] == "LLM" + assert span.parent is None + # With no span of the app's own around it, each turn is its own trace. + assert len({span.context.trace_id for span in spans}) == 3 + + +def test_the_session_id_is_a_keyed_digest_of_the_session_hash(recipe: Any) -> None: + """Gradio's session hash is the key to that session's live state: not exported.""" + recipe.app.predict(QUESTION, [], gr.Request(session_hash="tab-1")) + (span,) = recipe.exporter.get_finished_spans() + attrs = _attrs(span) + # A per-process 32-byte key, so the digest cannot be recomputed from a + # guessed session hash without it. + assert isinstance(recipe.app._SESSION_KEY, bytes) + assert len(recipe.app._SESSION_KEY) == 32 + for key in SESSION_KEYS: + assert attrs[key] == _session_digest(recipe.app, "tab-1") + assert SESSION_ID.fullmatch(attrs[key]) + assert attrs[key] != "tab-1" + assert attrs[key] != hashlib.sha256(b"tab-1").hexdigest()[:32] + assert "tab-1" not in json.dumps(attrs) + + +def test_two_turns_in_one_gradio_session_share_its_session_id(recipe: Any) -> None: + session = gr.Request(session_hash="tab-1") + recipe.app.predict(QUESTION, [], session) + recipe.app.predict(QUESTION, [], session) + recipe.app.predict(QUESTION, [], gr.Request(session_hash="tab-2")) + first, second, other = (_attrs(span) for span in recipe.exporter.get_finished_spans()) + for key in SESSION_KEYS: + assert first[key] == second[key] == _session_digest(recipe.app, "tab-1") + assert other[key] == _session_digest(recipe.app, "tab-2") + assert first[key] != other[key] + for attrs in (first, second, other): + dump = json.dumps(attrs) + assert "tab-1" not in dump + assert "tab-2" not in dump + + +def test_turns_without_a_gradio_session_carry_no_session_id(recipe: Any) -> None: + recipe.app.predict(QUESTION, []) + recipe.app.predict(QUESTION, [], None) + recipe.app.predict(QUESTION, [], gr.Request()) # a request without a session hash + spans = recipe.exporter.get_finished_spans() + assert len(spans) == 3 + for span in spans: + assert not set(SESSION_KEYS) & set(_attrs(span)), span.name + + +def test_gradio_is_not_the_span_source(recipe: Any) -> None: + """With the OpenAI instrumentor removed, a full Gradio turn exports nothing.""" + assert trace_api.get_tracer_provider() is recipe.provider + OpenAIInstrumentor().uninstrument() + try: + recipe.app.predict(QUESTION, [], gr.Request(session_hash="tab-1")) + asyncio.run(_gradio_turn(recipe.app.demo, QUESTION, [], "tab-1")) + asyncio.run(_gradio_turn(recipe.app.demo, QUESTION, [], "tab-1")) + assert len(recipe.fake.requests) == 3 # the model was called each time + assert recipe.exporter.get_finished_spans() == () + finally: + recipe.app.init_tracing() + # Control: the same exporter receives the span once the instrumentor is back. + recipe.app.predict(QUESTION, [], gr.Request(session_hash="tab-1")) + assert [span.name for span in recipe.exporter.get_finished_spans()] == [LLM_SPAN] + + +def test_gradio_dispatch_runs_predict_on_a_worker_thread_with_the_session( + recipe: Any, +) -> None: + async def turn() -> tuple[int, dict[str, Any]]: + return threading.get_ident(), await _gradio_turn(recipe.app.demo, QUESTION, [], "tab-9") + + loop_thread, result = asyncio.run(turn()) + response, _history = result["prediction"] + assert response == ANSWER + (span,) = recipe.exporter.get_finished_spans() + # Gradio ran the sync predict off the event loop's thread ... + assert recipe.threads[span.context.span_id] != loop_thread + # ... and injected gr.Request, whose session hash reached the span as a digest. + assert _attrs(span)["session.id"] == _session_digest(recipe.app, "tab-9") + + +def test_gradio_history_reaches_the_model_as_plain_text(recipe: Any) -> None: + """Gradio 6 hands predict each history message as a list of content parts.""" + demo = recipe.app.demo + first = asyncio.run(_gradio_turn(demo, "first question", [], "tab-1")) + _response, history = first["prediction"] + # What the browser sends back on the next turn: postprocess, then preprocess. + history = demo.chatbot.preprocess(demo.chatbot.postprocess(history)) + assert history[0]["content"] == [{"type": "text", "text": "first question"}] + asyncio.run(_gradio_turn(demo, "second question", history, "tab-1")) + assert recipe.fake.requests[1]["messages"] == [ + {"role": "user", "content": "first question"}, + {"role": "assistant", "content": ANSWER}, + {"role": "user", "content": "second question"}, + ] + assert len(recipe.exporter.get_finished_spans()) == 2 + + +# -------------------------------------------------------------------------- +# In-process: does the LLM span stay under a turn span across Gradio's +# thread pool? +# -------------------------------------------------------------------------- + + +def test_turn_span_opened_before_gradio_dispatch_parents_the_llm_span(recipe: Any) -> None: + async def turn() -> int: + with recipe.tracer.start_as_current_span("chat_turn", fi_span_kind="chain"): + await _gradio_turn(recipe.app.demo, QUESTION, [], "tab-1") + return threading.get_ident() + + loop_thread = asyncio.run(turn()) + spans = {span.name: span for span in recipe.exporter.get_finished_spans()} + assert set(spans) == {"chat_turn", LLM_SPAN} + turn_span, llm = spans["chat_turn"], spans[LLM_SPAN] + assert recipe.threads[turn_span.context.span_id] == loop_thread + assert recipe.threads[llm.context.span_id] != loop_thread + assert llm.parent.span_id == turn_span.context.span_id + assert llm.context.trace_id == turn_span.context.trace_id + + +def test_turn_span_inside_predict_parents_the_llm_span(recipe: Any) -> None: + tracer = recipe.tracer + + def traced_predict(message: str, history: list, request: gr.Request = None) -> str: + with tracer.start_as_current_span("chat_turn", fi_span_kind="chain"): + return recipe.app.predict(message, history, request) + + demo = gr.ChatInterface(traced_predict, analytics_enabled=False) + + async def turn() -> int: + await _gradio_turn(demo, QUESTION, [], "tab-1") + return threading.get_ident() + + loop_thread = asyncio.run(turn()) + spans = {span.name: span for span in recipe.exporter.get_finished_spans()} + turn_span, llm = spans["chat_turn"], spans[LLM_SPAN] + assert _attrs(turn_span)["gen_ai.span.kind"] == "CHAIN" + assert recipe.threads[llm.context.span_id] != loop_thread + assert llm.parent.span_id == turn_span.context.span_id + assert _attrs(llm)["session.id"] == _session_digest(recipe.app, "tab-1") + + +def test_sync_generator_turn_span_detaches_after_the_first_yield( + recipe: Any, caplog: pytest.LogCaptureFixture +) -> None: + """Known limit: Gradio steps a sync generator in a fresh worker call per yield.""" + tracer = recipe.tracer + + def streaming_predict(message: str, history: list, request: gr.Request = None): + with tracer.start_as_current_span("chat_turn", fi_span_kind="chain"): + yield recipe.app.predict(message, history, request) + yield recipe.app.predict(message, history, request) + + demo = gr.ChatInterface(streaming_predict, analytics_enabled=False) + + async def turn() -> None: + result = await _gradio_turn(demo, QUESTION, [], "tab-1") + while result["is_generating"]: + result = await demo.call_function( + _submit_event(demo), + [QUESTION, []], + iterator=result["iterator"], + requests=gr.Request(session_hash="tab-1"), + ) + + with caplog.at_level(logging.ERROR, logger="opentelemetry.context"): + asyncio.run(turn()) + spans = recipe.exporter.get_finished_spans() + (turn_span,) = [span for span in spans if span.name == "chat_turn"] + first, second = [span for span in spans if span.name == LLM_SPAN] + # Started before the first yield: a child of the turn span. + assert first.parent.span_id == turn_span.context.span_id + # Started after it: a root span in a separate trace. + assert second.parent is None + assert second.context.trace_id != turn_span.context.trace_id + # Closing the turn span in a later step cannot restore the context. + assert "Failed to detach context" in caplog.text + + +# -------------------------------------------------------------------------- +# Contract: the recipe through register() into the harness Receiver +# -------------------------------------------------------------------------- + + +def _base_env(home: Path, guard_log: Path) -> dict[str, str]: + """Built from scratch: nothing inherited but PATH, no .pyc left in the repo.""" + return { + "PATH": os.environ.get("PATH", ""), + "HOME": str(home), + "PYTHONDONTWRITEBYTECODE": "1", + # This repository's register() and OpenAI instrumentor. + "PYTHONPATH": os.pathsep.join( + [str(PYTHON_DIR), str(PYTHON_DIR / "frameworks" / "openai")] + ), + "LOOPBACK_GUARD_LOG": str(guard_log), + } + + +def _child_env(home: Path, guard_log: Path, **overrides: Optional[str]) -> dict[str, str]: + """The recipe's environment.""" + env = { + **_base_env(home, guard_log), + "FI_API_KEY": FI_API_KEY, + "FI_SECRET_KEY": FI_SECRET_KEY, + "FI_PROJECT_NAME": PROJECT, + "OPENAI_API_KEY": OPENAI_KEY, + # Longer than any run, so only register()'s exit flush can export. + "OTEL_BSP_SCHEDULE_DELAY": "600000", + } + for key, value in overrides.items(): + if value is None: + env.pop(key, None) + else: + env[key] = value + return env + + +def _guard_attempts(guard_log: Path) -> list[dict[str, Any]]: + if not guard_log.exists(): + return [] + return [json.loads(line) for line in guard_log.read_text().splitlines()] + + +def _run_driver( + tmp_path: Path, + *flags: str, + status: int = HTTPStatus.OK, + python: str = sys.executable, + **overrides: Optional[str], +) -> dict[str, Any]: + """Run drive_turns.py under the loopback guard; return everything the tests read. + + ``status`` is what the fake model host answers; ``overrides`` change the + child's environment (``None`` removes a variable). + """ + tmp_path.mkdir(parents=True, exist_ok=True) + guard_log = tmp_path / "guard.jsonl" + with Receiver() as receiver, FakeOpenAI(ANSWER, status=status) as fake: + env = _child_env( + tmp_path, + guard_log, + FI_BASE_URL=receiver.origin, + OPENAI_BASE_URL=fake.base_url, + **overrides, + ) + result = run( + [python, str(GUARD), str(DRIVER), QUESTION, *flags], + env=env, + stdin=None, + timeout=RUN_TIMEOUT_SECONDS, + ) + record = { + "result": result, + "stdout": result.stdout.decode("utf-8", "replace"), + "stderr": result.stderr.decode("utf-8", "replace"), + "requests": receiver.requests(), + "spans": receiver.spans(), + "model_requests": list(fake.requests), + "model_authorizations": list(fake.authorizations), + } + record["guard_attempts"] = _guard_attempts(guard_log) + return record + + +def _assert_ran(record: dict[str, Any]) -> None: + result = record["result"] + assert not result.timed_out, record["stderr"] + assert result.returncode == 0, record["stderr"] + assert record["guard_attempts"] == [], record["guard_attempts"] + assert record["requests"], "no export reached the receiver" + assert record["spans"], "the exports carried no spans" + + +def _report(record: dict[str, Any]) -> dict[str, Any]: + lines = [line for line in record["stdout"].splitlines() if line.startswith("{")] + return json.loads(lines[-1]) + + +def _value(value: dict[str, Any]) -> Any: + for kind in ("stringValue", "boolValue", "doubleValue"): + if kind in value: + return value[kind] + if "intValue" in value: + return int(value["intValue"]) + return value + + +def _otlp_attrs(span: dict[str, Any]) -> dict[str, Any]: + return {item["key"]: _value(item.get("value", {})) for item in span.get("attributes", [])} + + +def _export_dump(record: dict[str, Any]) -> str: + """Everything the export carried except HTTP headers: spans and resources.""" + return json.dumps( + { + "spans": record["spans"], + "resources": [r["resource_attributes"] for r in record["requests"]], + }, + sort_keys=True, + ) + + +@pytest.fixture(scope="module") +def recipe_run(tmp_path_factory: pytest.TempPathFactory) -> dict[str, Any]: + record = _run_driver(tmp_path_factory.mktemp("recipe")) + _assert_ran(record) + return record + + +@pytest.fixture(scope="module") +def content_run(tmp_path_factory: pytest.TempPathFactory) -> dict[str, Any]: + """The same two turns with init_tracing(trace_content=True), in another process.""" + record = _run_driver(tmp_path_factory.mktemp("content"), "--content") + _assert_ran(record) + return record + + +def test_recipe_runs_with_loopback_network_only(recipe_run: dict[str, Any]) -> None: + assert recipe_run["guard_attempts"] == [] + assert _report(recipe_run)["turns"] == 2 + # Both turns reached the fake, with the OpenAI key the client was given. + assert len(recipe_run["model_requests"]) == 2 + assert recipe_run["model_authorizations"] == ["Bearer " + OPENAI_KEY] * 2 + + +def test_guard_refuses_and_logs_non_loopback_connections(tmp_path: Path) -> None: + """Positive control for every ``guard_attempts == []`` assertion in this file.""" + guard_log = tmp_path / "guard.jsonl" + result = run( + [sys.executable, str(GUARD), str(GUARD_PROBE)], + env=_base_env(tmp_path, guard_log), + stdin=None, + timeout=60, + ) + assert not result.timed_out + assert result.returncode == 0, result.stderr.decode("utf-8", "replace") + assert "refused: connect,connect_ex,getaddrinfo" in result.stdout.decode("utf-8", "replace") + logged = _guard_attempts(guard_log) + assert [entry["kind"] for entry in logged] == ["connect", "connect_ex", "getaddrinfo"] + assert "192.0.2.1" in logged[0]["target"] + assert "2001:db8::1" in logged[1]["target"] + assert "example.invalid" in logged[2]["target"] + + +def test_export_goes_to_the_collector_path_with_both_keys(recipe_run: dict[str, Any]) -> None: + assert recipe_run["requests"], "no export reached the receiver" + for request in recipe_run["requests"]: + headers = request["headers"] + assert request["path"] == "/tracer/v1/traces" + assert headers.get("x-api-key") == FI_API_KEY + assert headers.get("x-secret-key") == FI_SECRET_KEY + assert headers.get("content-type") == "application/x-protobuf" + assert "authorization" not in headers + assert all(OPENAI_KEY not in value for value in headers.values()) + + +def test_resource_is_an_observe_project(recipe_run: dict[str, Any]) -> None: + resources = [ + resource for request in recipe_run["requests"] for resource in request["resource_attributes"] + ] + assert resources, "export without a resource" + for resource in resources: + assert resource.get("project_name") == PROJECT + assert resource.get("project_type") == "observe" + + +def test_the_batch_leaves_through_the_exit_flush(recipe_run: dict[str, Any]) -> None: + # The batch delay is 10 minutes, so the one export is register()'s flush + # at interpreter exit. + assert len(recipe_run["requests"]) == 1 + assert len(recipe_run["spans"]) == 2 + + +def test_one_llm_span_per_turn_with_the_session(recipe_run: dict[str, Any]) -> None: + spans = recipe_run["spans"] + assert len(spans) == len(recipe_run["model_requests"]) == 2 + assert len({span["traceId"] for span in spans}) == 2 + for span in spans: + attrs = _otlp_attrs(span) + assert span["name"] == LLM_SPAN + assert not span.get("parentSpanId") + assert span["status"].get("code") == "STATUS_CODE_OK" + assert attrs["gen_ai.span.kind"] == "LLM" + assert attrs["gen_ai.provider.name"] == "openai" + # traceai_openai records the model the response names. + assert attrs["gen_ai.request.model"] == RESPONSE_MODEL + assert attrs["gen_ai.usage.input_tokens"] == USAGE["prompt_tokens"] + assert attrs["gen_ai.usage.output_tokens"] == USAGE["completion_tokens"] + assert attrs["gen_ai.usage.total_tokens"] == USAGE["total_tokens"] + # Both turns of the one Gradio session carry one id, on both keys, and it + # is a digest, not the session hash. + (session_id,) = {_otlp_attrs(span)[key] for span in spans for key in SESSION_KEYS} + assert SESSION_ID.fullmatch(session_id) + assert session_id != DRIVER_SESSION + + +def test_the_raw_session_hash_is_not_exported(recipe_run: dict[str, Any]) -> None: + assert DRIVER_SESSION not in _export_dump(recipe_run) + assert DRIVER_SESSION not in recipe_run["stdout"] + recipe_run["stderr"] + + +def test_each_process_has_its_own_session_key( + recipe_run: dict[str, Any], content_run: dict[str, Any] +) -> None: + """Two runs, one session hash: two ids, so the id cannot be precomputed.""" + ids = [ + {_otlp_attrs(span)["session.id"] for span in record["spans"]} + for record in (recipe_run, content_run) + ] + assert all(len(run_ids) == 1 for run_ids in ids), ids + assert ids[0] != ids[1] + + +def test_content_is_off_by_default(recipe_run: dict[str, Any]) -> None: + dump = _export_dump(recipe_run) + for marker in CONTENT_MARKERS: + assert marker not in dump, marker + for span in recipe_run["spans"]: + attrs = _otlp_attrs(span) + assert not [k for k in attrs if k.startswith(CONTENT_KEY_PREFIXES)], span["name"] + assert attrs["input.value"] == REDACTED + assert attrs["output.value"] == REDACTED + assert "input.mime_type" not in attrs + assert not span.get("events") + + +def test_no_key_in_the_export_or_output(recipe_run: dict[str, Any]) -> None: + dump = _export_dump(recipe_run) + output = recipe_run["stdout"] + recipe_run["stderr"] + for secret in (FI_API_KEY, FI_SECRET_KEY, OPENAI_KEY): + assert secret not in dump + assert secret not in output + + +def test_the_prompt_is_never_printed(recipe_run: dict[str, Any]) -> None: + output = recipe_run["stdout"] + recipe_run["stderr"] + for marker in CONTENT_MARKERS: + assert marker not in output + + +def test_content_is_exported_when_the_recipe_turns_it_on(content_run: dict[str, Any]) -> None: + """Control for the content-off test: init_tracing(trace_content=True).""" + record = content_run + first, second = (_otlp_attrs(span) for span in record["spans"]) + assert first["gen_ai.input.messages.0.message.content"] == QUESTION + assert first["gen_ai.output.messages.0.message.content"] == ANSWER + assert first["output.value"] == ANSWER + assert second["gen_ai.input.messages.1.message.content"] == ANSWER + assert second["gen_ai.input.messages.2.message.content"] == QUESTION + # input.value holds the first message of the request, for every turn. + assert first["input.value"] == second["input.value"] == QUESTION + dump = _export_dump(record) + for secret in (FI_API_KEY, FI_SECRET_KEY, OPENAI_KEY): + assert secret not in dump + for marker in CONTENT_MARKERS: + assert marker not in record["stdout"] + record["stderr"] + + +def test_reload_keeps_the_first_provider_and_ignores_tracing_edits(tmp_path: Path) -> None: + """Gradio reload mode, replayed: nothing is dropped, and edits do not apply.""" + record = _run_driver(tmp_path, "--reload") + _assert_ran(record) + # Both turns, before and after the reload, were exported at exit ... + assert len(record["spans"]) == 2 + # ... by the provider the first init_tracing() created. + first_provider = _report(record)["first_provider"] + for request in record["requests"]: + for resource in request["resource_attributes"]: + assert resource["project_version_id"] == first_provider + # init_tracing() ran on the watcher thread twice, at startup and after the + # save. Each run's register() made another provider, which exported + # nothing, and each run logged two warnings. + stderr = record["stderr"] + assert stderr.count("Attempting to instrument while already instrumented") == 2 + assert stderr.count("Failed to register signal handlers") == 2 + # The reloaded source turned content on; the instrumentor ignored it. + for marker in CONTENT_MARKERS: + assert marker not in _export_dump(record) + # The session key survived both runs of the file: one id for both turns. + first, second = (_otlp_attrs(span) for span in record["spans"]) + assert first["session.id"] == second["session.id"] + + +def test_a_model_error_is_exported_as_an_error_span_without_content(tmp_path: Path) -> None: + """The model host answers HTTP 500; the client's retries are off.""" + record = _run_driver(tmp_path, "--model-error", status=HTTPStatus.INTERNAL_SERVER_ERROR) + _assert_ran(record) + report = _report(record) + assert (report["error"], report["status_code"]) == ("InternalServerError", 500) + assert len(record["model_requests"]) == 1 + # The failed turn's span left through the exit flush ... + assert len(record["requests"]) == 1 + (span,) = record["spans"] + attrs = _otlp_attrs(span) + assert span["name"] == LLM_SPAN + assert attrs["gen_ai.span.kind"] == "LLM" + # ... with status ERROR (StatusCode.ERROR, as OTLP names it) ... + assert span["status"]["code"] == "STATUS_CODE_ERROR" + # ... and no message text. + for marker in CONTENT_MARKERS: + assert marker not in _export_dump(record) + assert not [k for k in attrs if k.startswith(CONTENT_KEY_PREFIXES)] + assert attrs["input.value"] == REDACTED + # What it does carry, whatever TraceConfig says: the exception, including + # the error body the model host returned. + (event,) = span["events"] + assert event["name"] == "exception" + exception = _otlp_attrs(event) + assert exception["exception.type"] == "openai.InternalServerError" + assert "fixture failure" in exception["exception.message"] + assert "exception.stacktrace" in exception + assert "fixture failure" in span["status"]["message"] + + +def test_building_a_gradio_app_contacts_gradio_analytics_unless_turned_off( + tmp_path: Path, +) -> None: + """Why src/app.py passes analytics_enabled=False (the recipe run logs nothing).""" + guard_log = tmp_path / "guard.jsonl" + result = run( + [sys.executable, str(GUARD), str(ANALYTICS_PROBE)], + env=_base_env(tmp_path, guard_log), # GRADIO_ANALYTICS_ENABLED unset + stdin=None, + timeout=RUN_TIMEOUT_SECONDS, + ) + assert not result.timed_out + assert result.returncode == 0, result.stderr.decode("utf-8", "replace") + targets = {entry["target"] for entry in _guard_attempts(guard_log)} + assert "'api.gradio.app'" in targets + + +@pytest.mark.parametrize("content", [False, True], ids=["app-default", "explicit-optin"]) +def test_the_driver_uses_the_app_default_unless_content_is_requested(monkeypatch, content): + """The content-off contract run must exercise init_tracing()'s default.""" + spec = importlib.util.spec_from_file_location("gradio_recipe_driver_v1", DRIVER) + assert spec is not None and spec.loader is not None + driver = importlib.util.module_from_spec(spec) + spec.loader.exec_module(driver) + calls = [] + + def init_tracing(*args, **kwargs): + calls.append((args, kwargs)) + return SimpleNamespace( + resource=SimpleNamespace(attributes={"project_version_id": "fixture-provider"}) + ) + + app = SimpleNamespace(init_tracing=init_tracing, predict=lambda *args: "fixture answer") + monkeypatch.setattr(driver, "load_app", lambda: app) + argv = ["drive_turns.py", "fixture question"] + (["--content"] if content else []) + assert driver.main(argv) == 0 + assert calls == [((), {"trace_content": True} if content else {})] + + +def test_readme_shows_the_recipe_as_written() -> None: + readme = README.read_text(encoding="utf-8") + app = APP.read_text(encoding="utf-8") + for line in ( + "def init_tracing(trace_content: bool = False):", + "project_type=ProjectType.OBSERVE,", + "config = TraceConfig(hide_inputs=True, hide_outputs=True)", + "OpenAIInstrumentor().instrument(tracer_provider=provider, config=config)", + '_SESSION_KEY = globals().get("_SESSION_KEY") or secrets.token_bytes(32)', + 'digest = hmac.new(_SESSION_KEY, request.session_hash.encode("utf-8"), hashlib.sha256)', + "session_id = _session_id(request)", + "with using_session(session_id) if session_id else nullcontext():", + "demo = gr.ChatInterface(predict, analytics_enabled=False)", + ): + assert line in app, line + assert line in readme, line + for fact in ("gradio==6.29.1", "/tracer/v1/traces", "project_type=observe"): + assert fact in readme, fact + + +# -------------------------------------------------------------------------- +# Opt-in: requirements.txt from PyPI, not this repository's source +# -------------------------------------------------------------------------- + +PUBLISHED_PINS = os.environ.get("GRADIO_RECIPE_PUBLISHED_PINS") == "1" +INSTALL_TIMEOUT_SECONDS = 900 +# Run with ``python -I`` in the fresh virtualenv: which traceAI it imports. +INSTALLED_PROBE = """ +import json +from importlib import metadata + +import fi_instrumentation +import traceai_openai + +names = ["fi-instrumentation-otel", "traceAI-openai", "gradio", "openai", "anyio", + "wrapt", "opentelemetry-api", "opentelemetry-sdk", + "opentelemetry-exporter-otlp-proto-http", "opentelemetry-instrumentation"] +print(json.dumps({ + "versions": {name: metadata.version(name) for name in names}, + "files": {"fi_instrumentation": fi_instrumentation.__file__, + "traceai_openai": traceai_openai.__file__}, +})) +""" + + +@pytest.mark.skipif( + not PUBLISHED_PINS, + reason="opt-in: GRADIO_RECIPE_PUBLISHED_PINS=1 installs requirements.txt from PyPI", +) +def test_requirements_txt_from_pypi_traces_into_the_receiver(tmp_path: Path) -> None: + """A fresh virtualenv with ``pip install -r requirements.txt``, then the contract run.""" + venv = tmp_path / "venv" + # PATH and HOME only (pip's cache and config); no PYTHONPATH. + install_env = {"PATH": os.environ.get("PATH", ""), "HOME": os.environ.get("HOME", "")} + subprocess.run( + [sys.executable, "-m", "venv", str(venv)], env=install_env, cwd=tmp_path, check=True + ) + python = str(venv / "bin" / "python") + install = subprocess.run( + [python, "-m", "pip", "install", "--disable-pip-version-check", "--no-input", "-q"] + + ["-r", str(REQUIREMENTS)], + env=install_env, + cwd=tmp_path, + capture_output=True, + timeout=INSTALL_TIMEOUT_SECONDS, + ) + assert install.returncode == 0, install.stderr.decode("utf-8", "replace") + probe = subprocess.run( + [python, "-I", "-c", INSTALLED_PROBE], + env=install_env, + cwd=tmp_path, + capture_output=True, + timeout=RUN_TIMEOUT_SECONDS, + ) + assert probe.returncode == 0, probe.stderr.decode("utf-8", "replace") + installed = json.loads(probe.stdout) + print(json.dumps(installed, indent=1, sort_keys=True)) # shown with -rP + assert installed["versions"]["fi-instrumentation-otel"] == "1.1.0" + assert installed["versions"]["traceAI-openai"] == "0.1.10" + for module_file in installed["files"].values(): + assert Path(module_file).resolve().is_relative_to(venv.resolve()), module_file + + record = _run_driver(tmp_path / "run", python=python, PYTHONPATH=None) + _assert_ran(record) + assert len(record["requests"]) == 1 + (request,) = record["requests"] + assert request["path"] == "/tracer/v1/traces" + assert request["headers"].get("x-api-key") == FI_API_KEY + assert request["headers"].get("x-secret-key") == FI_SECRET_KEY + for resource in request["resource_attributes"]: + assert resource.get("project_name") == PROJECT + assert resource.get("project_type") == "observe" + spans = record["spans"] + assert [span["name"] for span in spans] == [LLM_SPAN, LLM_SPAN] + (session_id,) = {_otlp_attrs(span)[key] for span in spans for key in SESSION_KEYS} + assert SESSION_ID.fullmatch(session_id) + for span in spans: + attrs = _otlp_attrs(span) + assert attrs["gen_ai.span.kind"] == "LLM" + assert attrs["gen_ai.usage.total_tokens"] == USAGE["total_tokens"] + assert attrs["input.value"] == attrs["output.value"] == REDACTED + dump = _export_dump(record) + for text in (*CONTENT_MARKERS, DRIVER_SESSION, FI_API_KEY, FI_SECRET_KEY, OPENAI_KEY): + assert text not in dump, text diff --git a/python/tests/harness/__init__.py b/python/tests/harness/__init__.py new file mode 100644 index 000000000..cf79047f1 --- /dev/null +++ b/python/tests/harness/__init__.py @@ -0,0 +1,238 @@ +"""Small, loopback-only utilities for OTLP contract tests.""" + +from __future__ import annotations + +import copy +import difflib +import json +import subprocess +import threading +from http import HTTPStatus +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from typing import Any, Mapping, Optional, Sequence, Union +from urllib.parse import urlsplit +from urllib.request import Request, urlopen + + +class Receiver: + """An in-memory OTLP/HTTP receiver for tests. + + The receiver only listens on loopback and never forwards received spans. + """ + + def __init__(self) -> None: + self._spans: list[dict[str, Any]] = [] + self._requests: list[dict[str, Any]] = [] + self._lock = threading.Lock() + owner = self + + class Handler(BaseHTTPRequestHandler): + def _read_body(self) -> bytes: + # The Node OTLP exporter streams with Transfer-Encoding: chunked + # and sends no Content-Length, so both framings are accepted. + if "chunked" in self.headers.get("Transfer-Encoding", "").lower(): + chunks = [] + while True: + size_line = self.rfile.readline().split(b";", 1)[0].strip() + size = int(size_line, 16) + if size == 0: + while self.rfile.readline() not in (b"\r\n", b"\n", b""): + pass + return b"".join(chunks) + chunks.append(self.rfile.read(size)) + self.rfile.readline() + length = int(self.headers.get("Content-Length", "0")) + return self.rfile.read(length) + + def do_POST(self) -> None: # noqa: N802 - required by BaseHTTPRequestHandler + path = urlsplit(self.path).path + if path not in ("/v1/traces", "/tracer/v1/traces"): + self.send_error(HTTPStatus.NOT_FOUND) + return + + try: + body = self._read_body() + content_type = self.headers.get("Content-Type", "").lower() + + if content_type.startswith("application/json"): + request = json.loads(body.decode("utf-8")) + elif content_type.startswith(("application/x-protobuf", "application/protobuf")): + from google.protobuf.json_format import MessageToDict + from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ( + ExportTraceServiceRequest, + ) + + protobuf_request = ExportTraceServiceRequest() + protobuf_request.ParseFromString(body) + request = MessageToDict(protobuf_request) + else: + self.send_error(HTTPStatus.UNSUPPORTED_MEDIA_TYPE) + return + + spans: list[dict[str, Any]] = [] + resource_attributes: list[dict[str, Any]] = [] + for resource_spans in request.get( + "resourceSpans", request.get("resource_spans", []) + ): + resource_attributes.append( + _flatten_attributes( + resource_spans.get("resource", {}).get("attributes", []) + ) + ) + for scope_spans in resource_spans.get( + "scopeSpans", resource_spans.get("scope_spans", []) + ): + spans.extend(scope_spans.get("spans", [])) + except (ImportError, TypeError, UnicodeDecodeError, ValueError) as error: + self.send_error(HTTPStatus.BAD_REQUEST, str(error)) + return + + with owner._lock: + owner._spans.extend(copy.deepcopy(spans)) + owner._requests.append( + { + "path": path, + "headers": { + key.lower(): value for key, value in self.headers.items() + }, + "resource_attributes": resource_attributes, + } + ) + self.send_response(HTTPStatus.OK) + self.send_header("Content-Length", "0") + self.end_headers() + + def log_message(self, _format: str, *args: Any) -> None: + return + + self._server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + self.origin = "http://127.0.0.1:{0}".format(self._server.server_port) + self.endpoint = "{0}/v1/traces".format(self.origin) + self.collector_endpoint = "{0}/tracer/v1/traces".format(self.origin) + self._thread = threading.Thread(target=self._server.serve_forever, daemon=True) + self._thread.start() + + def __enter__(self) -> "Receiver": + return self + + def __exit__(self, _type: Any, _value: Any, _traceback: Any) -> None: + self._server.shutdown() + self._server.server_close() + self._thread.join() + + def spans(self) -> list[dict[str, Any]]: + """Return a copy of the decoded spans received so far.""" + with self._lock: + return copy.deepcopy(self._spans) + + def requests(self) -> list[dict[str, Any]]: + """Return one record per accepted export: path, lower-cased headers, resource attributes.""" + with self._lock: + return copy.deepcopy(self._requests) + + def clear(self) -> None: + """Remove every decoded span and request record received so far.""" + with self._lock: + self._spans.clear() + self._requests.clear() + + +def _flatten_attributes(attributes: Sequence[Mapping[str, Any]]) -> dict[str, Any]: + """Turn OTLP key/value attributes into a plain dict of scalar values.""" + flat: dict[str, Any] = {} + for attribute in attributes: + value = attribute.get("value", {}) + for kind in ("stringValue", "boolValue", "intValue", "doubleValue"): + if kind in value: + flat[attribute.get("key", "")] = value[kind] + break + else: + flat[attribute.get("key", "")] = value + return flat + + +def run( + argv: Sequence[str], + env: Mapping[str, str], + stdin: Optional[bytes], + timeout: float, +) -> subprocess.CompletedProcess[bytes]: + """Run one process and return its captured output, including after a timeout.""" + process = subprocess.Popen( + argv, + env=dict(env), + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + timed_out = False + try: + stdout, stderr = process.communicate(input=stdin, timeout=timeout) + except subprocess.TimeoutExpired: + timed_out = True + process.kill() + stdout, stderr = process.communicate() + + result = subprocess.CompletedProcess(argv, process.returncode, stdout, stderr) + result.timed_out = timed_out + return result + + +def post_otlp(spans: dict[str, Any], endpoint: str) -> int: + """Post one JSON-encoded OTLP request to a loopback receiver.""" + parsed = urlsplit(endpoint) + if parsed.scheme != "http" or parsed.hostname != "127.0.0.1": + raise ValueError("post_otlp only sends to a loopback HTTP endpoint") + + request = Request( + endpoint, + data=json.dumps(spans).encode("utf-8"), + headers={"Content-Type": "application/json"}, + method="POST", + ) + with urlopen(request, timeout=5) as response: + return response.status + + +def compare(actual: Sequence[dict[str, Any]], golden_path: Union[str, Path]) -> None: + """Fail with a diff unless span names, attributes, and statuses match a golden.""" + with Path(golden_path).open(encoding="utf-8") as golden_file: + expected = json.load(golden_file) + + def canonical(spans: Sequence[dict[str, Any]]) -> list[dict[str, Any]]: + normalized = [] + for span in spans: + attributes = span.get("attributes", []) + if isinstance(attributes, dict): + attributes = [ + {"key": key, "value": value} for key, value in attributes.items() + ] + normalized.append( + { + "name": span.get("name", ""), + "attributes": sorted( + attributes, + key=lambda attribute: json.dumps( + attribute, sort_keys=True, separators=(",", ":") + ), + ), + "status": span.get("status", {}), + } + ) + return normalized + + actual_normalized = canonical(actual) + expected_normalized = canonical(expected) + if actual_normalized != expected_normalized: + diff = "\n".join( + difflib.unified_diff( + json.dumps(expected_normalized, indent=2, sort_keys=True).splitlines(), + json.dumps(actual_normalized, indent=2, sort_keys=True).splitlines(), + fromfile=str(golden_path), + tofile="actual", + lineterm="", + ) + ) + print(diff) + raise AssertionError("spans do not match golden") diff --git a/python/tests/harness/test_harness.py b/python/tests/harness/test_harness.py new file mode 100644 index 000000000..bd61d5adc --- /dev/null +++ b/python/tests/harness/test_harness.py @@ -0,0 +1,189 @@ +"""Contract tests for the shared loopback-only harness.""" + +import json +import sys + +import pytest + +from tests.harness import Receiver, compare, post_otlp, run + + +def test_receiver_stores_posted_span_and_can_clear() -> None: + payload = { + "resourceSpans": [ + { + "scopeSpans": [ + { + "spans": [ + { + "traceId": "trace-id", + "spanId": "span-id", + "name": "harness.receiver", + "attributes": [ + {"key": "component", "value": {"stringValue": "test"}} + ], + "status": {"code": 1}, + } + ] + } + ] + } + ] + } + + with Receiver() as receiver: + assert post_otlp(payload, receiver.endpoint) == 200 + assert receiver.spans() == payload["resourceSpans"][0]["scopeSpans"][0]["spans"] + receiver.clear() + assert receiver.spans() == [] + + +def test_receiver_accepts_the_collector_path() -> None: + payload = { + "resourceSpans": [ + {"scopeSpans": [{"spans": [{"name": "collector.path"}]}]} + ] + } + + with Receiver() as receiver: + assert post_otlp(payload, receiver.collector_endpoint) == 200 + assert receiver.spans() == [{"name": "collector.path"}] + + +def _post_chunked(url: str, body: bytes, headers: dict) -> int: + """Send a body with Transfer-Encoding: chunked, as the Node OTLP exporter does.""" + import http.client + from urllib.parse import urlsplit + + parts = urlsplit(url) + connection = http.client.HTTPConnection(parts.hostname, parts.port, timeout=5) + try: + chunks = [body[: len(body) // 2], body[len(body) // 2 :]] + connection.request( + "POST", + parts.path, + body=iter(chunks), + headers={**headers, "Transfer-Encoding": "chunked"}, + encode_chunked=True, + ) + return connection.getresponse().status + finally: + connection.close() + + +def test_receiver_decodes_a_chunked_body() -> None: + payload = { + "resourceSpans": [ + {"scopeSpans": [{"spans": [{"name": "chunked.body"}]}]} + ] + } + + with Receiver() as receiver: + status = _post_chunked( + receiver.collector_endpoint, + json.dumps(payload).encode("utf-8"), + {"Content-Type": "application/json"}, + ) + assert status == 200 + assert receiver.spans() == [{"name": "chunked.body"}] + + +def test_receiver_records_path_headers_and_resource_attributes() -> None: + payload = { + "resourceSpans": [ + { + "resource": { + "attributes": [ + {"key": "project_name", "value": {"stringValue": "harness"}}, + {"key": "project_type", "value": {"stringValue": "observe"}}, + ] + }, + "scopeSpans": [{"spans": [{"name": "with.resource"}]}], + } + ] + } + + with Receiver() as receiver: + status = _post_chunked( + receiver.collector_endpoint, + json.dumps(payload).encode("utf-8"), + { + "Content-Type": "application/json", + "X-Api-Key": "test-key", + "X-Secret-Key": "test-secret", + }, + ) + assert status == 200 + [request] = receiver.requests() + assert request["path"] == "/tracer/v1/traces" + assert request["headers"]["x-api-key"] == "test-key" + assert request["headers"]["x-secret-key"] == "test-secret" + assert "authorization" not in request["headers"] + assert request["resource_attributes"] == [ + {"project_name": "harness", "project_type": "observe"} + ] + receiver.clear() + assert receiver.requests() == [] + + +def test_run_captures_output_and_kills_timed_out_process() -> None: + completed = run( + [ + sys.executable, + "-c", + "import sys; print('stdout'); sys.stderr.write('stderr\\n')", + ], + env={}, + stdin=b"", + timeout=1, + ) + assert completed.returncode == 0 + assert completed.stdout == b"stdout\n" + assert completed.stderr == b"stderr\n" + assert completed.timed_out is False + + timed_out = run( + [sys.executable, "-c", "import time; time.sleep(30)"], + env={}, + stdin=b"", + timeout=0.1, + ) + assert timed_out.timed_out is True + assert timed_out.returncode != 0 + + +def test_compare_accepts_normalized_match_and_prints_mismatch_diff(tmp_path, capsys) -> None: + golden = [ + { + "traceId": "golden-trace", + "spanId": "golden-span", + "startTimeUnixNano": "1", + "endTimeUnixNano": "2", + "name": "harness.compare", + "attributes": [ + {"key": "alpha", "value": {"stringValue": "a"}}, + {"key": "beta", "value": {"intValue": "2"}}, + ], + "status": {"code": 1}, + } + ] + golden_path = tmp_path / "spans.golden.json" + golden_path.write_text(json.dumps(golden), encoding="utf-8") + + actual = [ + { + "traceId": "other-trace", + "spanId": "other-span", + "startTimeUnixNano": "3", + "endTimeUnixNano": "4", + "name": "harness.compare", + "attributes": list(reversed(golden[0]["attributes"])), + "status": {"code": 1}, + } + ] + compare(actual, golden_path) + + actual[0]["name"] = "harness.mismatch" + with pytest.raises(AssertionError, match="spans do not match golden"): + compare(actual, golden_path) + assert "---" in capsys.readouterr().out