diff --git a/python/examples/openlit/README.md b/python/examples/openlit/README.md new file mode 100644 index 000000000..a00eeb6c3 --- /dev/null +++ b/python/examples/openlit/README.md @@ -0,0 +1,611 @@ +# OpenLIT → Future AGI (recipe) + +There is no traceAI package for OpenLIT and no OpenLIT adapter in Future +AGI. OpenLIT already exports OTLP/HTTP. This example points `openlit.init()` +at fi-collector, passes the Future AGI keys as OTLP headers, puts the +project on the resource with `OTEL_RESOURCE_ATTRIBUTES`, and turns off +metrics, events, OpenLIT's price download and its message capture. With +message capture off, the prompt and the answer are not exported, but +model-generated tool-call arguments, the `user` argument and error bodies +still are (see "Privacy"). The recipe adds no mapping or filtering of its +own: the key inventory below lists what OpenLIT sent and which of those keys +fi-collector reads. + +Pinned: `openlit` 1.45.0. Its wheel METADATA declares `License: Apache-2.0` +and the OSI Apache classifier, ships the Apache 2.0 text, and declares +`Requires-Python: >=3.9.0,<4.0.0`. It requires `openai>=1.92.0,<3.0.0`, so +`requirements.txt` pins `openai` 2.54.0, not the 3.x line. Both are +dependencies of this example only, not of any traceAI package. + +The contract test (see "Tests") passes on Python 3.10, 3.11, 3.12 and 3.13 +with `openai` 2.54.0, `opentelemetry-sdk` and +`opentelemetry-exporter-otlp-proto-http` 1.45.0 and +`opentelemetry-instrumentation-httpx` 0.66b0, the versions uv resolved on +2026-10-05. Python 3.9 is inside openlit's range but was not tested here. + +## Run + +```bash +cd python/examples/openlit +pip install -r requirements.txt + +export FI_API_KEY="YOUR_API_KEY" +export FI_SECRET_KEY="YOUR_SECRET_KEY" +export OTEL_EXPORTER_OTLP_ENDPOINT="https://YOUR_FI_COLLECTOR_ORIGIN" # no path +export OTEL_RESOURCE_ATTRIBUTES="project_name=my-chatbot,project_type=observe" +export OPENAI_API_KEY="YOUR_OPENAI_KEY" # optional: OPENAI_BASE_URL, OPENAI_MODEL + +python src/app.py "What is the refund window?" +``` + +## The recipe + +`init_tracing()` in `src/app.py` is the whole integration. It first checks +that `OTEL_EXPORTER_OTLP_ENDPOINT`, `FI_API_KEY`, `FI_SECRET_KEY` and the +project are set, and otherwise exits with a message saying what to set +(tested). Then it calls: + +```python +openlit.init( + service_name=APP_NAME, # "openlit-recipe", exported as service.name + otlp_endpoint=endpoint, # OTEL_EXPORTER_OTLP_ENDPOINT + otlp_headers={ + "x-api-key": quote(api_key, safe=""), # FI_API_KEY + "x-secret-key": quote(secret_key, safe=""), # FI_SECRET_KEY + }, + capture_message_content=False, + disable_metrics=True, + disable_events=True, + pricing_json=str(NO_PRICING), # src/no_pricing.json, an empty table +) +``` + +A test checks that this call has the same arguments as the one in +`src/app.py`, and that the "Run" block sets a `project_name`. + +The 1.45.0 signature, quoted from the wheel's `openlit/__init__.py:185-209`: + +```python +def init( + environment="default", + application_name="default", + service_name="default", + otlp_endpoint=None, + otlp_headers=None, + disable_batch=False, + capture_message_content=True, + disabled_instrumentors=None, + disable_metrics=False, + disable_events=False, + pricing_json=None, + controller_mode=None, + collect_gpu_stats=False, + collect_system_metrics=False, + capture_db_parameters=False, + max_content_length=None, + custom_span_attributes=None, + custom_metrics_attributes=None, + openlit_api_key=None, + openlit_url=None, + *, + guards=None, + guard_fail_open=True, +): +``` + +It has no resource argument and no telemetry argument. `init()` logs and +swallows any error raised inside it (`__init__.py:487-488`), so a mistake +there shows up as a log line, not an exception. + +### Endpoint + +Set `otlp_endpoint` to the collector origin, with no path. OpenLIT copies +it into `OTEL_EXPORTER_OTLP_ENDPOINT` (`openlit/otel/tracing.py:72-73`) and +creates OpenTelemetry's OTLP/HTTP span exporter with no arguments +(`tracing.py:121-128`). That exporter (`opentelemetry-exporter-otlp-proto-http` +1.45.0, `_common/__init__.py:129-138`) removes one trailing `/` and always +appends `/v1/traces`. Tested results: + +| `otlp_endpoint` | Request path | Reply from a fi-collector-like mux | +|---|---|---| +| `http://host:port` | `/v1/traces` | 200 | +| `http://host:port/` | `/v1/traces` | 200 | +| `http://host:port/v1/traces` | `/v1/traces/v1/traces` | 404 | +| `http://host:port/tracer` | `/tracer/v1/traces` | 200 | + +fi-collector serves `/v1/traces` and `/tracer/v1/traces` +(`fi-collector/pkg/server/server.go:233-234` at future-agi `main` 4af5338). +These paths were tested against a loopback recorder that answers the way +that mux does; no deployed endpoint was called. + +Read from source, not tested: with `otlp_endpoint` left out, OpenLIT uses +`OTEL_EXPORTER_OTLP_ENDPOINT` from the environment, and with neither it +prints every span to stdout (`tracing.py:121-131`). `src/app.py` refuses to +start without the variable instead (tested). A set +`OTEL_EXPORTER_OTLP_TRACES_ENDPOINT` wins over all of this (see "Environment +that overrides the recipe"). `OTEL_EXPORTER_OTLP_PROTOCOL=grpc` switches +OpenLIT to the gRPC exporter (`tracing.py:23-26`); this recipe does not test +gRPC. + +### Auth headers + +`otlp_headers` is a dict (`__init__.py:222`). OpenLIT joins it into +`name=value,name=value` and writes that to `OTEL_EXPORTER_OTLP_HEADERS` +(`tracing.py:75-83`). The exporter parses the variable with OpenTelemetry's +`parse_env_headers(..., liberal=True)` (`_common/__init__.py:141-153`), +which splits on commas and percent-decodes each value +(`opentelemetry/util/re.py:52-93`). So the recipe percent-encodes both +values. The contract test proves that they arrive as the `x-api-key` and +`x-secret-key` request headers fi-collector reads +(`fi-collector/pkg/auth/middleware.go:45-46`), using a placeholder secret +that contains `,` and `=`. Without the encoding, a comma cuts the value +short (tested). + +Because OpenLIT writes the joined string to `os.environ`, the keys stay in +the process environment after `init()`. Child processes inherit them (read +from source), and so does any OTLP exporter created later in the same +process (tested; see "Environment that overrides the recipe"). + +### Project + +fi-collector rejects a batch whose resource has no `project_name` (HTTP 400, +`fi-collector/pkg/auth/stamp.go:31-45`) and creates the project on first +use. `openlit.init()` has no resource argument. It builds its resource with +`Resource.create()` (`tracing.py:62-68`), which merges the standard +`OTEL_RESOURCE_ATTRIBUTES` variable and percent-decodes its values +(`opentelemetry-sdk` 1.45.0, `resources/__init__.py:333-348`). The test sets +`project_name=...,project_type=observe` there and finds exactly these +resource attributes on every export: `project_name`, `project_type`, +`service.name` (`openlit-recipe`), `deployment.environment` (`default`), +`telemetry.sdk.name` (`openlit`), `telemetry.sdk.language`, +`telemetry.sdk.version` and `service.instance.id`. `src/app.py` refuses to +start when the variable has no `project_name` (tested). + +### Metrics and events + +By default OpenLIT also exports metrics and, once an endpoint is set, OTel +log records it calls events (`__init__.py:378-400`, +`otel/events.py:124-136`). Their exporters post to `/v1/metrics` and +`/v1/logs`, which fi-collector does not route, and get 404 (tested against +the recorder). The recipe passes `disable_metrics=True` and +`disable_events=True`; with both, the only request is to `/v1/traces` +(tested). With events off, `init()` still installs a global OTel logger +provider with an OTLP log exporter (`__init__.py:378-385`); nothing is +written to it, and no `/v1/logs` request was seen. + +### Price download and telemetry + +`init()` always loads a price table (`__init__.py:426`). Without +`pricing_json` it downloads +`https://raw.githubusercontent.com/openlit/openlit/main/assets/pricing.json`, +with a 20-second timeout, on every start (`__helpers.py:476-482`). The recipe +passes `src/no_pricing.json`, an empty table, so nothing is downloaded and +OpenLIT sends `gen_ai.usage.cost` as 0 (tested). With the argument left out, +the loopback guard refused exactly one attempt, a DNS lookup of +`raw.githubusercontent.com`, and nothing else (tested). + +`init()` has no telemetry switch. The 1.45.0 wheel has no telemetry client, +and the price download is the only URL the `init()` path calls; +`cloud.openlit.io` appears only in the CLI's help text (read from source). +No recipe run in the contract test made any non-loopback connection +attempt. + +### Environment that overrides the recipe + +If these variables are already set when `openlit.init()` runs, they +replace or widen what the recipe configures. `src/app.py` does not check +them. + +- `OTEL_EXPORTER_OTLP_TRACES_HEADERS` replaces the header string OpenLIT + built: the exporter reads it instead of `OTEL_EXPORTER_OTLP_HEADERS` + (`_common/__init__.py:146-147`). The Future AGI keys are then not sent, + and every export gets 401, which is only logged. Tested with + `x-team=support` against a recorder that answers 401 without the keys, as + fi-collector's auth middleware does (`middleware.go:18-21`). +- `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT` wins over `otlp_endpoint` and is used + as the full URL, with nothing appended (`_common/__init__.py:133-134`). + The Future AGI keys go to that URL, wherever it points (tested). +- `OTEL_TRACES_EXPORTER` makes OpenLIT add an OTLP exporter only if the list + includes `otlp` (`tracing.py:86-119`). With `none` nothing is exported; + with `console` each span is printed to stdout instead (both tested). +- `OTEL_EXPORTER_OTLP_HEADERS` is overwritten by OpenLIT with the recipe's + headers (`tracing.py:83`), so headers you put there are dropped (read from + source). + +The keys also leak the other way. OpenLIT writes the endpoint and the +joined keys to `OTEL_EXPORTER_OTLP_ENDPOINT` and `OTEL_EXPORTER_OTLP_HEADERS` +in `os.environ` (`tracing.py:72-83`). Any OTLP/HTTP exporter created later +in the same process without its own headers reads them and sends the Future +AGI keys to whatever endpoint it was given. The test builds a second +exporter for another loopback receiver after `init_tracing()`, and that +receiver got both keys. An exporter built without an endpoint also sends its +spans to fi-collector (read from source). + +When `openlit.init()` reuses an existing tracer provider, it builds no +exporter and writes neither variable (`tracing.py:56-59`; see "traceAI's +`register()`"). + +## What is traced + +With content off, `python src/app.py` exports two spans in one trace: + +| Span | Parent | Kind | Instrumentor | +|---|---|---|---| +| `chat gpt-4o-mini` | none | CLIENT | OpenLIT's OpenAI instrumentor | +| `POST` | `chat gpt-4o-mini` | CLIENT | OpenTelemetry's httpx instrumentor, which `openlit.init()` also enables | + +The model-call span is named `chat `. `init()` also enables +OpenTelemetry's `requests`, `urllib`, `urllib3` and `aiohttp` client +instrumentors when those libraries are installed (`_instrumentors.py:78-82`, +`__init__.py:141-145`), so other HTTP calls in your app become spans too +(read from source). `disabled_instrumentors` turns any of them off (not +tested). + +## Key inventory + +Every attribute key OpenLIT 1.45.0 put on the spans of one non-streaming +chat call against the fake, with the value the test checked. "Stored" means +fi-collector keeps the key in the span's raw attribute maps +(`attrs_string`, `attrs_number`, `attrs_bool`, or the `attributes_extra` +JSON for arrays; `fi-collector/pkg/adapter/adapter.go:62-107`) without +putting it in a column. Whether the Future AGI UI shows raw attributes was +not checked. + +Model-call span (`chat gpt-4o-mini`), content off: + +| Key | Value in the test | fi-collector | +|---|---|---| +| `gen_ai.operation.name` | `chat` | `observation_type` (see "What Future AGI shows") | +| `gen_ai.provider.name` | `openai` | `provider` column | +| `gen_ai.request.model` | `gpt-4o-mini` | `model` column | +| `gen_ai.response.model` | `gpt-4o-mini-2024-07-18` | stored; `model` reads the request model first | +| `gen_ai.usage.input_tokens` | `11` | `prompt_tokens` column | +| `gen_ai.usage.output_tokens` | `7` | `completion_tokens` column | +| `gen_ai.client.token.usage` | `18` | stored; not a total-tokens alias | +| `gen_ai.usage.cost` | `0` (empty price table) | stored; not a cost alias | +| `gen_ai.usage.cache_read.input_tokens` | `0` | stored | +| `gen_ai.usage.cache_creation.input_tokens` | `0` | stored | +| `gen_ai.request.temperature` | `1.0` (OpenLIT's default; the call set none) | stored | +| `gen_ai.request.top_p` | `1.0` (same) | stored | +| `gen_ai.request.frequency_penalty` | `0.0` (same) | stored | +| `gen_ai.request.presence_penalty` | `0.0` (same) | stored | +| `gen_ai.request.seed` | `0` (same) | stored | +| `gen_ai.request.user` | empty string | stored | +| `gen_ai.request.stream` | `false` | stored | +| `gen_ai.response.id` | `chatcmpl-fake` | stored | +| `gen_ai.response.finish_reasons` | `["stop"]` | stored (array) | +| `gen_ai.output.type` | `text` | stored | +| `gen_ai.sdk.version` | `2.54.0`, the `openai` package version | stored | +| `gen_ai.server.time_to_first_token` | seconds, a float | stored | +| `gen_ai.server.time_per_output_token` | `0` | stored | +| `openai.api.type` | `chat_completions` | stored | +| `openlit.agent.version_hash` | 16 hex digits | stored | +| `server.address` | `127.0.0.1` | stored | +| `server.port` | the fake's port | stored | +| `service.name` | `openlit-recipe` | stored (also on the resource) | +| `deployment.environment` | `default` | stored (also on the resource) | +| `telemetry.sdk.name` | `openlit` | stored (also on the resource) | + +Added to the model-call span only with content on, OpenLIT's default: + +| Key | Value in the test | fi-collector | +|---|---|---| +| `gen_ai.input.messages` | JSON string: the system and user messages | stored; not read as the span's input | +| `gen_ai.output.messages` | JSON string: the assistant message | stored; not read as the span's output | +| `gen_ai.system_instructions` | JSON string: the system prompt | stored | + +HTTP span (`POST`): + +| Key | Value in the test | fi-collector | +|---|---|---| +| `http.method` | `POST` | stored | +| `http.url` | the full OpenAI request URL | stored | +| `http.status_code` | `200` | stored | + +Not sent (tested): `gen_ai.usage.total_tokens`, `gen_ai.system`, and every +key fi-collector reads a span kind from: `fi.span.kind`, `gen_ai.span.kind`, +`llm.request.type`, `openinference.span.kind` +(`exporter/clickhouse25exporter/converter.go:79-84`). No span events were +sent. A tool call adds `gen_ai.tool.name`, `gen_ai.tool.call.id` and +`gen_ai.tool.args`, and a failed call adds `error.type`, two `exception` +events and an error status; "Privacy" covers them because they are sent +with content off (tested). Streaming and embeddings were not exercised. + +## What Future AGI shows + +fi-collector promotes a few keys to columns. This table is everything this +recipe calls "displayed". It was read from fi-collector at future-agi `main` +4af5338 (`pkg/adapter/adapter.go`, +`exporter/clickhouse25exporter/converter.go`); the opt-in test (see +"Tests") re-reads from that source the alias lists and the keys `input` and +`output` are read from, and checks this table against them and against the +keys the run emitted. The contract test runs no fi-collector; "Live check" below +records a run against a local Future AGI stack. + +| Column | Filled from | Source | +|---|---|---| +| `model` | `gen_ai.request.model`, which wins over `gen_ai.response.model` | `adapter.go:279-284` | +| `provider` | `gen_ai.provider.name` | `adapter.go:289-295` | +| `prompt_tokens` | `gen_ai.usage.input_tokens` | `adapter.go:296-300` | +| `completion_tokens` | `gen_ai.usage.output_tokens` | `adapter.go:301-305` | +| `total_tokens` | derived as input plus output, because no total alias arrived | `adapter.go:230-235`, `:306-310` | +| `cost` | none: `gen_ai.usage.cost` is not one of `gen_ai.cost.*`, `llm.cost.*` | `adapter.go:311-313` | +| `observation_type` | `gen_ai.operation.name`: `chat` maps to `llm` | `converter.go:79-131` | +| `input` | none: only `input.value` is read | `converter.go:343` | +| `output` | none: only `output.value` is read | `converter.go:344` | + +Because OpenLIT's cost key is not read, fi-collector treats the span as +having no user cost and prices it from `model` and the token columns with +its own price table, when one is configured (`converter.go:285-294`). That +pricing was not exercised here. This recipe computes no price, and the 0 +OpenLIT sends with the empty table is not used. + +The `POST` span carries none of these keys: its `observation_type` resolves +to `unknown` and its model and token columns stay empty (read from source). + +Gaps for the shared processor work (SF-1), from this inventory: + +- `gen_ai.client.token.usage` is not a total-tokens alias. The column is + derived instead, and for this call the derived value equals it. +- `gen_ai.usage.cost` is not a cost alias. +- `gen_ai.input.messages`, `gen_ai.output.messages` and + `gen_ai.system_instructions` are not read as the span's input or output. +- A `@openlit.trace` span (see "Privacy") has no span-kind key, so it is + `unknown`. + +## Privacy + +Content capture is on unless you turn it off: `capture_message_content` +defaults to `True` (`__init__.py:192`). With it on, the model-call span +carries `gen_ai.input.messages` (system prompt and question), +`gen_ai.output.messages` (the answer) and `gen_ai.system_instructions` +(tested). The recipe passes `False`, and the test finds none of those keys, +no span events, and none of the question, answer or system-prompt text +anywhere in the export. The switch covers those messages only. The list +below is what it does not cover; the recipe filters nothing. + +`OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT=false` turns content off +even when `init()` is called with `True` (tested; `__init__.py:410-414`). +The variable cannot turn content on when the argument is `False`: OpenLIT +applies it only while the argument is still `True` (`__init__.py:292-293`, +read from source). + +Not covered by the content switch: + +- Tool calls. When the request passes `tools` and the model answers with + tool calls, the model-call span carries `gen_ai.tool.name`, + `gen_ai.tool.call.id` and `gen_ai.tool.args`, the arguments the model + generated, verbatim (`instrumentation/openai/utils.py:1524-1551`, before + the content check at `:1616`). Those arguments often repeat what the user + typed: an email address, an order number. The test plants an email address + in the fake model's tool-call arguments and finds it in `gen_ai.tool.args` + (`tests/tool_call.py`). The Responses API sets the same keys + (`utils.py:991-1022`; not tested). +- `user`. The `user` argument of `chat.completions.create()` is sent as + `gen_ai.request.user` (`utils.py:1484-1487`). The test passes an email + address and finds it there. Without the argument the key is an empty + string. +- Errors. When the OpenAI call raises, OpenLIT records the exception on the + model-call span (`instrumentation/openai/openai.py:186-209`, + `utils.py:77-84`) and re-raises inside `start_as_current_span`, whose + context manager records it again and sets the span status + (`opentelemetry-api` 1.45.0 `trace/__init__.py:609-622`, called from + `opentelemetry-sdk` `trace/__init__.py:1130-1136`). So a failed call + carries two `exception` events, each with `exception.message` and + `exception.stacktrace`, and the same text as the status description, + `status.message`. The `openai` SDK puts the whole error response body in + that text (`openai/_base_client.py:430-435`). Tested with a fake 500 and + `max_retries=0` (`tests/failed_call.py`): the fake's error body appeared + in all three places, and no prompt or answer text did. An + OpenAI-compatible server or proxy behind `OPENAI_BASE_URL` whose error + bodies echo the request would put prompt text there (not tested). +- `@openlit.trace` records the decorated function's arguments + (`function.args`, `function.kwargs`) and its return value + (`gen_ai.output.messages`) whatever the switch says + (`__init__.py:953-965`). With content off, the test finds the question in + `function.args` and the answer in `gen_ai.output.messages`. `src/app.py` + does not use the decorator; `tests/traced_function.py` is the fixture. +- The `POST` span's `http.url` is the full request URL (tested). +- OpenLIT's Firecrawl instrumentor ignores the switch (see "Firecrawl"). +- With traceai-openai also enabled, its span carries the prompt and the + answer whatever OpenLIT's switch says (tested; see "traceai-openai"). + +If your tool arguments or `user` values can carry personal data, choose one: + +- Turn OpenLIT's OpenAI instrumentor off with + `disabled_instrumentors=["openai"]`. OpenLIT then makes no model-call span, + so none of the tool, `user` or error keys above is recorded by it (tested + with traceai-openai enabled; see "traceai-openai"). The httpx `POST` span + and its `http.url` stay. +- Or accept that these keys reach Future AGI. + +No Future AGI key, secret or OpenAI key appears in any exported span or +resource, or in the app's output (tested). + +## Using OpenLIT with traceAI + +### traceai-openai + +OpenLIT wraps `Completions.create` +(`instrumentation/openai/__init__.py:185-189`) and traceai-openai wraps +`OpenAI.request` (`python/frameworks/openai/traceai_openai/__init__.py:55-59`), +so with both enabled one call is traced twice. The test runs the recipe and +then enables traceai-openai 0.1.10's `OpenAIInstrumentor` on the tracer +provider OpenLIT created. One call to the fake gave three spans in one +trace: + +| Span | Parent | From | +|---|---|---| +| `chat gpt-4o-mini` | none | OpenLIT | +| `ChatCompletion` | `chat gpt-4o-mini` | traceai-openai (`gen_ai.span.kind` = `LLM`) | +| `POST` | `ChatCompletion` | OpenTelemetry httpx, enabled by OpenLIT | + +Both model-call spans carry the same token counts (tested), so fi-collector +stores two LLM spans with tokens for one call (read from source) and a sum +over spans counts the call twice. OpenLIT's content switch does not reach +traceai-openai's span: with the recipe's content off, that span still +carried the system prompt in `input.value`, the question in +`gen_ai.input.messages.1.message.content` and the answer in `output.value` +(tested). + +Enable one of the two. Neither package disables the other, and this recipe +does not do it for you. To keep only traceai-openai's span, pass +`disabled_instrumentors=["openai"]` to `openlit.init()`; the test then sees +only `ChatCompletion` and `POST`. To keep only OpenLIT's, do not enable +traceai-openai. + +### traceAI's `register()` + +`openlit.init()` reuses the global tracer provider if it is an SDK +`TracerProvider`, and otherwise creates its own (`tracing.py:56-59`). +traceAI's `register()` makes its provider global only when asked +(`set_global_tracer_provider: bool = False`, +`python/fi_instrumentation/otel.py:102`, `:245-246`). The test +(`tests/register_first.py`) calls `register()` first, then the recipe's +`init_tracing()`, and makes the model call inside a span from `register()`'s +provider. Both cases were tested: + +- `register(set_global_tracer_provider=True)` first: `openlit.init()` reuses + that provider and adds no exporter, so `otlp_endpoint`, `otlp_headers` and + `OTEL_RESOURCE_ATTRIBUTES` are ignored for spans. All three spans went to + `register()`'s endpoint (`FI_BASE_URL` plus `/tracer/v1/traces`) with + `register()`'s keys and project. +- `register()` with its default first: the process has two providers and + two exports. OpenLIT's spans (`chat gpt-4o-mini`, `POST`) went to + `otlp_endpoint` under the `OTEL_RESOURCE_ATTRIBUTES` project, and the + `register()` span went to `FI_BASE_URL` under `register()`'s project. They + are still one trace: the model-call span's parent is the `register()` span, + which is in the other export, and in the other project unless the two + project names match. + +### Firecrawl + +OpenLIT 1.45.0's `FireCrawlInstrumentor` wraps `FirecrawlApp` in +`firecrawl.firecrawl` (`scrape_url`, `crawl_url`, `map_url`, `search`, +`extract`, `batch_scrape_urls`, `check_crawl_status`) and the matching +`AsyncFirecrawlApp` methods (`instrumentation/firecrawl/__init__.py:40-273`): +the Firecrawl v1 client. The architecture note (D-8338) confirms the same at +OpenLIT commit `897838a`. Per that note, the traceAI Firecrawl package +(TH-8325) wraps the v2 `Firecrawl` client instead. If you use both, pick +one. This recipe does not disable either and does not change the Firecrawl +package. `disabled_instrumentors=["firecrawl"]` turns OpenLIT's off +(`_instrumentors.py:50`, `__init__.py:131-133`; not tested). + +OpenLIT's content switch does not reach its Firecrawl instrumentor. `init()` +passes `capture_message_content` (`__init__.py:148-156`), but the +instrumentor reads `trace_content`, which defaults to `True` +(`instrumentation/firecrawl/__init__.py:33`), and uses it to decide whether +to record scraped content (`instrumentation/firecrawl/utils.py:264`, `:431`). +Read from source; no Firecrawl call was made. + +## Notes + +- OpenLIT uses a `BatchSpanProcessor`, and the SDK tracer provider flushes + it at exit, so a short script exports on exit (tested) and a server + exports in batches. +- When fi-collector rejects an export (401 without valid keys, + `middleware.go:20`), the exporter logs + `Failed to export spans batch code: 401, reason: Unauthorized` to stderr + and the app carries on and exits 0; nothing raises in your code (tested + against a recorder that answers 401). +- The contract test calls no fi-collector, OpenLIT host, GitHub or OpenAI. + Auth, project stamping and storage are fi-collector behaviour the harness + receiver does not reproduce; see "Live check" for a run that did. + +## Live check + +Run on 2026-10-16 against a local Future AGI stack (fi-collector on `:14338`, +ClickHouse spans table, trace UI), with the recipe's `src/app.py` +`init_tracing()` unchanged, `openlit` 1.45.0 (the latest release on PyPI that +day), `openai` 2.54.0 (the latest 2.x; `openlit` requires `<3`), Python 3.12. +One agent run made two chat calls with a tool in between (the tool function +wrapped in `@openlit.trace`), plus one call to an unknown model. + +- The export was accepted with the origin as `OTEL_EXPORTER_OTLP_ENDPOINT` + and both percent-encoded key headers. The project came from + `OTEL_RESOURCE_ATTRIBUTES`. +- Stored tree: `agent` > two `chat ` spans (each with a child `POST`) and + `lookup_order`. The chat spans had `observation_type=llm`, `model`, `provider`, + and prompt, completion and total tokens. The `agent`, `lookup_order` and + `POST` spans were `unknown` kind, as listed under "What Future AGI shows". +- The failed call was stored with status `ERROR`; its status message holds + the provider's error body (see "Privacy"). The stored `span_events` kept only + the two `exception` event names and timestamps, with no `exception.message` + or `exception.stacktrace` attributes. The status message, tool arguments and + `user` were stored as the "Privacy" section says. +- With `capture_message_content=True`, the prompt and answer were stored only + in `attrs_string` (`gen_ai.input.messages`, `gen_ai.output.messages`); the + span's `input` and `output` columns stayed empty, so the trace UI shows no + input or output for the model-call span. +- `gen_ai.usage.cost` (0 with the recipe's empty price table) was not in the stored + attributes of this local stack, and the `cost` column was 0. The key + inventory table above records what OpenLIT sent, not what this stack kept. + +## Tests + +`tests/test_openlit_recipe.py` runs `src/app.py` as written, in a +subprocess, with the real `openlit` 1.45.0 instrumenting the real `openai` +client. `tests/_fake_openai.py` is a loopback fake of the Chat Completions +API that answers with text, a tool call or an error; the spans go to the +shared harness `Receiver` (`python/tests/harness`), which serves +`/v1/traces` and `/tracer/v1/traces` on 127.0.0.1. +`tests/_guarded_run.py` blocks and logs every non-loopback connection; a +positive control (`tests/_guard_probe.py`) proves it refuses and logs IPv4, +IPv6 and DNS attempts. All keys are placeholders. + +`tests/recipe_variant.py` runs `src/app.py` with one `openlit.init()` +argument changed or removed, or with traceai-openai also enabled. The other +scripts run the recipe's own `init_tracing()` first: +`tests/traced_function.py` wraps the model call in `@openlit.trace`, +`tests/tool_call.py` makes a call with `tools` and `user`, +`tests/failed_call.py` makes the call with `max_retries=0` against an error, +`tests/register_first.py` calls traceAI's `register()` before it, and +`tests/second_exporter.py` builds another OTLP exporter after it. All are +fixtures, not part of the recipe. + +The tests check the request path for each endpoint form, the auth headers, +the resource, the span tree, the exact key inventory and its values, +content off and on, the content environment override, the metrics and +events paths, the price download, what content off still exports (the +decorator, tool calls, `user`, a failed call), a 401 reply, the environment +that overrides the recipe, the app's start-up checks, both instrumentors +together, `register()` before `openlit.init()`, that no key reaches the +export, and that the README's `openlit.init()` call and "Run" block match +`src/app.py`. + +From the repository root, Python 3.11: + +```bash +env -u PYTHONPATH PYTHONPATH="python/examples/openlit:python:python/tests" \ + uv run --no-project --python 3.11 \ + --with pytest --with jsonschema --with protobuf --with opentelemetry-proto \ + --with 'openlit==1.45.0' --with 'openai==2.54.0' \ + pytest python/examples/openlit/tests -q -p no:cacheprovider \ + --noconftest -o addopts= -rfEs +``` + +For Python 3.13, replace `--python 3.11`. `python` is on the path and +`jsonschema` is installed because pytest imports `python/__init__.py`, which +imports `fi_instrumentation`, and because the `register()` test imports +`fi_instrumentation` from `python/`. The recipe needs neither. + +The two traceai-openai tests skip in that environment. To run them, add +`--with 'traceai-openai==0.1.10'`. That package requires `wrapt<2` (through +`fi-instrumentation-otel` 1.1.0), so uv resolves `wrapt` 1.17.3 there and +2.5.0 without it. The whole suite passes in both environments. + +One test is opt-in. It reads the alias lists from fi-collector's +`adapter.go` and `converter.go`, and the keys passed to `overflowAsString` +for `input` and `output` in `converter.go`, and checks the "What Future AGI +shows" table against them and against the emitted keys. Point +`FI_COLLECTOR_SRC` at the `fi-collector` directory of a future-agi checkout +(tested with `main` 4af5338): + +```bash +export FI_COLLECTOR_SRC=/fi-collector # then run the command above +``` + +Without it the test is skipped. No CI job runs this example, so run it +whenever fi-collector's alias lists or this recipe change; otherwise the +table can drift unnoticed. A full run starts the app in a new process about +30 times and takes under a minute. diff --git a/python/examples/openlit/requirements.txt b/python/examples/openlit/requirements.txt new file mode 100644 index 000000000..7d067fff6 --- /dev/null +++ b/python/examples/openlit/requirements.txt @@ -0,0 +1,4 @@ +# Not a traceAI package: these are the example's own dependencies. +openlit==1.45.0 +# openlit 1.45.0 requires openai>=1.92.0,<3.0.0; this is the version the contract test ran with. +openai==2.54.0 diff --git a/python/examples/openlit/src/app.py b/python/examples/openlit/src/app.py new file mode 100644 index 000000000..2fbe1289b --- /dev/null +++ b/python/examples/openlit/src/app.py @@ -0,0 +1,89 @@ +"""Send OpenLIT traces to Future AGI. + +Recipe example, not a package. OpenLIT already exports OTLP/HTTP; this file +only calls ``openlit.init`` with the collector endpoint, the Future AGI keys +and the switches README.md explains, then makes one OpenAI call that +OpenLIT's auto-instrumentation traces. The project comes from the standard +``OTEL_RESOURCE_ATTRIBUTES`` variable (see README.md). + + python src/app.py "What is the refund window?" +""" + +from __future__ import annotations + +import os +import sys +from pathlib import Path +from urllib.parse import quote + +import openlit +from openai import OpenAI +from opentelemetry.sdk.resources import OTELResourceDetector + +APP_NAME = "openlit-recipe" # exported as the service.name resource attribute +MODEL = os.environ.get("OPENAI_MODEL", "gpt-4o-mini") +# An empty price table. Without pricing_json, openlit.init() downloads one +# from raw.githubusercontent.com on every start. +NO_PRICING = Path(__file__).resolve().with_name("no_pricing.json") + +_POLICY = "Refunds are accepted within 30 days of purchase." + + +def init_tracing() -> None: + """Start OpenLIT's exporter. Call once per process, before any model call.""" + endpoint = os.environ.get("OTEL_EXPORTER_OTLP_ENDPOINT") + if not endpoint: + # Without an endpoint OpenLIT prints every span to stdout instead. + raise SystemExit("Set OTEL_EXPORTER_OTLP_ENDPOINT to the fi-collector origin.") + api_key = os.environ.get("FI_API_KEY") + secret_key = os.environ.get("FI_SECRET_KEY") + if not (api_key and secret_key): + # fi-collector answers 401 without both, and OpenLIT only logs that. + raise SystemExit("Set FI_API_KEY and FI_SECRET_KEY to your Future AGI keys.") + # openlit.init() has no resource argument; its Resource.create() merges + # OTEL_RESOURCE_ATTRIBUTES. fi-collector rejects (HTTP 400) a batch whose + # resource has no project_name, so stop here instead. + if not OTELResourceDetector().detect().attributes.get("project_name"): + raise SystemExit( + "Set OTEL_RESOURCE_ATTRIBUTES=project_name=,project_type=observe" + ) + openlit.init( + service_name=APP_NAME, + # The origin, no path: the OTLP/HTTP exporter appends /v1/traces. + otlp_endpoint=endpoint, + # OpenLIT joins this dict into OTEL_EXPORTER_OTLP_HEADERS, which the + # exporter splits on commas and percent-decodes: encode the values. + otlp_headers={ + "x-api-key": quote(api_key, safe=""), + "x-secret-key": quote(secret_key, safe=""), + }, + capture_message_content=False, # OpenLIT's default is True + disable_metrics=True, # fi-collector serves no /v1/metrics + disable_events=True, # events go to /v1/logs, which fi-collector does not serve + pricing_json=str(NO_PRICING), + ) + + +def ask(question: str) -> str: + # OpenAI() reads OPENAI_API_KEY and OPENAI_BASE_URL. + response = OpenAI().chat.completions.create( + model=MODEL, + messages=[ + {"role": "system", "content": "Answer using only this policy: " + _POLICY}, + {"role": "user", "content": question}, + ], + ) + return response.choices[0].message.content or "" + + +def main(argv: list[str]) -> int: + init_tracing() + question = argv[1] if len(argv) > 1 else "What is the refund window?" + print(ask(question)) + # The SDK TracerProvider flushes its batch processor at exit, so a short + # script needs no more. A long-lived server keeps exporting in batches. + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/python/examples/openlit/src/no_pricing.json b/python/examples/openlit/src/no_pricing.json new file mode 100644 index 000000000..0967ef424 --- /dev/null +++ b/python/examples/openlit/src/no_pricing.json @@ -0,0 +1 @@ +{} diff --git a/python/examples/openlit/tests/_fake_openai.py b/python/examples/openlit/tests/_fake_openai.py new file mode 100644 index 000000000..2b9f96058 --- /dev/null +++ b/python/examples/openlit/tests/_fake_openai.py @@ -0,0 +1,106 @@ +"""A loopback stand-in for the OpenAI Chat Completions API (non-streaming). + +Every POST to /v1/chat/completions gets one assistant message whose text is +``answer_text``, with fixed token usage, so the instrumentor records model +and usage attributes. With ``tool_call`` the message is that one tool call +instead of text; with ``error`` every request gets that status and JSON body. +Each request body and Authorization header is kept for assertions. Listens +on 127.0.0.1 only. +""" + +from __future__ import annotations + +import json +import threading +from http import HTTPStatus +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from typing import Any, Optional + +RESPONSE_MODEL = "gpt-4o-mini-2024-07-18" +USAGE = {"prompt_tokens": 11, "completion_tokens": 7, "total_tokens": 18} + + +class FakeOpenAI: + """Serve /v1/chat/completions on 127.0.0.1 and record each request. + + ``tool_call`` is ``{"id": ..., "name": ..., "arguments": }``; + ``error`` is ``[status, body]``. + """ + + def __init__( + self, + answer_text: str, + tool_call: Optional[dict[str, str]] = None, + error: Optional[tuple[int, dict[str, Any]]] = None, + ) -> None: + self.requests: list[dict[str, Any]] = [] + self.authorizations: list[str | None] = [] + lock = threading.Lock() + owner = self + + if tool_call is not None: + message: dict[str, Any] = { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": tool_call["id"], + "type": "function", + "function": { + "name": tool_call["name"], + "arguments": tool_call["arguments"], + }, + } + ], + } + finish_reason = "tool_calls" + else: + message = {"role": "assistant", "content": answer_text} + finish_reason = "stop" + + class Handler(BaseHTTPRequestHandler): + def do_POST(self) -> None: # noqa: N802 - BaseHTTPRequestHandler API + if self.path.rstrip("/") != "/v1/chat/completions": + self.send_error(HTTPStatus.NOT_FOUND) + return + length = int(self.headers.get("Content-Length", "0")) + request = json.loads(self.rfile.read(length).decode("utf-8")) + with lock: + owner.requests.append(request) + owner.authorizations.append(self.headers.get("Authorization")) + if error is not None: + status, reply = HTTPStatus(error[0]), error[1] + else: + status = HTTPStatus.OK + reply = { + "id": "chatcmpl-fake", + "object": "chat.completion", + "created": 1_700_000_000, + "model": RESPONSE_MODEL, + "choices": [ + {"index": 0, "message": message, "finish_reason": finish_reason} + ], + "usage": USAGE, + } + body = json.dumps(reply).encode("utf-8") + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, format: str, *args: Any) -> None: # noqa: A002 + return + + self._server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + self.base_url = "http://127.0.0.1:{0}/v1".format(self._server.server_port) + self._thread = threading.Thread(target=self._server.serve_forever, daemon=True) + self._thread.start() + + def __enter__(self) -> "FakeOpenAI": + return self + + def __exit__(self, *_exc: Any) -> None: + self._server.shutdown() + self._server.server_close() + self._thread.join() diff --git a/python/examples/openlit/tests/_guard_probe.py b/python/examples/openlit/tests/_guard_probe.py new file mode 100644 index 000000000..5ebeff59d --- /dev/null +++ b/python/examples/openlit/tests/_guard_probe.py @@ -0,0 +1,26 @@ +"""A test-only probe: try three non-loopback connections under the loopback guard. + +The contract test runs this through ``_guarded_run.py`` to prove the guard +really refuses and logs a non-loopback attempt, so the ``guard_attempts == []`` +assertions elsewhere cannot pass because the guard silently stopped working. +The targets are reserved and never reachable: TEST-NET-1 (RFC 5737), the IPv6 +documentation prefix (RFC 3849) and the ``.invalid`` TLD (RFC 2606). +Not part of the recipe. +""" + +from __future__ import annotations + +import socket + +refused = [] +for label, attempt in ( + ("connect", lambda: socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect(("192.0.2.1", 80))), + ("connect_ex", lambda: socket.socket(socket.AF_INET6, socket.SOCK_STREAM).connect_ex(("2001:db8::1", 80, 0, 0))), + ("getaddrinfo", lambda: socket.getaddrinfo("example.invalid", 443)), +): + try: + attempt() + except OSError as error: + if "blocked by loopback guard" in str(error): + refused.append(label) +print("refused:", ",".join(refused)) diff --git a/python/examples/openlit/tests/_guarded_run.py b/python/examples/openlit/tests/_guarded_run.py new file mode 100644 index 000000000..2295b36fc --- /dev/null +++ b/python/examples/openlit/tests/_guarded_run.py @@ -0,0 +1,81 @@ +"""Run a script with every non-loopback network connection blocked and logged. + +Usage: python _guarded_run.py [args...] + +Before the script starts, ``socket.socket.connect``/``connect_ex`` and +``socket.getaddrinfo`` refuse any host other than 127.0.0.1, ::1 or +localhost. Each refused attempt is appended as a JSON line to the file named +by ``LOOPBACK_GUARD_LOG``, so a test can assert that the SDK, its +instrumentors and the example tried to reach nothing else (no pricing +download, no OpenLIT host, no real OpenAI). Unix sockets are not affected. +""" + +from __future__ import annotations + +import json +import os +import runpy +import socket +import sys + +_ALLOWED_HOSTS = {"127.0.0.1", "::1", "localhost"} +_LOG_PATH = os.environ.get("LOOPBACK_GUARD_LOG", "") + + +def _host(value: object) -> str: + if isinstance(value, bytes): + return value.decode("ascii", "replace") + return str(value) + + +def _refuse(kind: str, target: object) -> None: + if _LOG_PATH: + with open(_LOG_PATH, "a", encoding="utf-8") as log: + log.write(json.dumps({"kind": kind, "target": repr(target)}) + "\n") + + +_original_connect = socket.socket.connect +_original_connect_ex = socket.socket.connect_ex +_original_getaddrinfo = socket.getaddrinfo + + +def _is_blocked(sock: socket.socket, address: object) -> bool: + if sock.family not in (socket.AF_INET, socket.AF_INET6): + return False + host = address[0] if isinstance(address, tuple) else address + return _host(host) not in _ALLOWED_HOSTS + + +def _connect(self: socket.socket, address: object) -> None: + if _is_blocked(self, address): + _refuse("connect", address) + raise OSError("blocked by loopback guard: {0!r}".format(address)) + return _original_connect(self, address) + + +def _connect_ex(self: socket.socket, address: object) -> int: + if _is_blocked(self, address): + _refuse("connect_ex", address) + raise OSError("blocked by loopback guard: {0!r}".format(address)) + return _original_connect_ex(self, address) + + +def _getaddrinfo(host: object, *args: object, **kwargs: object): # type: ignore[no-untyped-def] + if host is not None and _host(host) not in _ALLOWED_HOSTS: + _refuse("getaddrinfo", host) + raise socket.gaierror("blocked by loopback guard: {0!r}".format(host)) + return _original_getaddrinfo(host, *args, **kwargs) + + +def main() -> None: + socket.socket.connect = _connect # type: ignore[method-assign] + socket.socket.connect_ex = _connect_ex # type: ignore[method-assign] + socket.getaddrinfo = _getaddrinfo # type: ignore[assignment] + script = sys.argv[1] + sys.argv = sys.argv[1:] + sys.path.insert(0, os.path.dirname(os.path.abspath(script))) + runpy.run_path(script, run_name="__main__") + + +if __name__ == "__main__": + main() diff --git a/python/examples/openlit/tests/failed_call.py b/python/examples/openlit/tests/failed_call.py new file mode 100644 index 000000000..c702c98ec --- /dev/null +++ b/python/examples/openlit/tests/failed_call.py @@ -0,0 +1,27 @@ +"""A test-only fixture: the recipe's model call against a server error. + +It runs ``init_tracing()`` and ``ask()`` from ``src/app.py`` as written +(content off). The one change: the OpenAI client is built with +``max_retries=0``, so the fake's error is raised at once instead of being +retried twice with back-off. The error is caught and its type printed, so +the process exits 0. Not part of the recipe. +""" + +from __future__ import annotations + +import functools +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "src")) + +from openai import APIStatusError # noqa: E402 + +import app # noqa: E402 + +app.OpenAI = functools.partial(app.OpenAI, max_retries=0) +app.init_tracing() +try: + app.ask(sys.argv[1]) +except APIStatusError as error: + print("raised", type(error).__name__, error.status_code) diff --git a/python/examples/openlit/tests/recipe_variant.py b/python/examples/openlit/tests/recipe_variant.py new file mode 100644 index 000000000..1b51d7af3 --- /dev/null +++ b/python/examples/openlit/tests/recipe_variant.py @@ -0,0 +1,40 @@ +"""A test-only fixture: ``src/app.py`` as written, with one thing changed. + +``RECIPE_INIT_OVERRIDES`` is a JSON object of ``openlit.init()`` keyword +arguments to replace; a ``null`` value removes the argument, so OpenLIT's +own default applies. With ``RECIPE_ADD_TRACEAI_OPENAI=1``, traceAI's +``OpenAIInstrumentor`` is also enabled right after ``openlit.init()``, on the +tracer provider OpenLIT created, so both instrument the same client. +Everything else, including the model call, is the recipe's own code. +Not part of the recipe. +""" + +from __future__ import annotations + +import json +import os +import runpy +from pathlib import Path + +import openlit + +APP = Path(__file__).resolve().parent.parent / "src" / "app.py" +_OVERRIDES = json.loads(os.environ.get("RECIPE_INIT_OVERRIDES", "{}")) +_real_init = openlit.init + + +def _init(**kwargs): # type: ignore[no-untyped-def] + for key, value in _OVERRIDES.items(): + if value is None: + kwargs.pop(key, None) + else: + kwargs[key] = value + _real_init(**kwargs) + if os.environ.get("RECIPE_ADD_TRACEAI_OPENAI") == "1": + from traceai_openai import OpenAIInstrumentor + + OpenAIInstrumentor().instrument() + + +openlit.init = _init +runpy.run_path(str(APP), run_name="__main__") diff --git a/python/examples/openlit/tests/register_first.py b/python/examples/openlit/tests/register_first.py new file mode 100644 index 000000000..f7eba6389 --- /dev/null +++ b/python/examples/openlit/tests/register_first.py @@ -0,0 +1,35 @@ +"""A test-only fixture: traceAI's ``register()`` before the recipe's ``init_tracing()``. + +``register()`` runs first: with ``set_global_tracer_provider=True`` when +``REGISTER_GLOBAL=1``, else with its default (``False``). Then +``init_tracing()`` from ``src/app.py`` runs as written, and the recipe's +model call runs inside one span from ``register()``'s provider, standing in +for a traceAI instrumentor's span. ``register()`` exports to ``FI_BASE_URL`` +under the project ``REGISTER_PROJECT``. traceAI is imported from this +repository's ``python/`` directory. Not part of the recipe. +""" + +from __future__ import annotations + +import os +import sys +from pathlib import Path + +HERE = Path(__file__).resolve() +sys.path.insert(0, str(HERE.parents[3])) # python/, for fi_instrumentation +sys.path.insert(0, str(HERE.parents[1] / "src")) + +from fi_instrumentation import register # noqa: E402 +from fi_instrumentation.fi_types import ProjectType # noqa: E402 + +from app import ask, init_tracing # noqa: E402 + +provider = register( + project_name=os.environ["REGISTER_PROJECT"], + project_type=ProjectType.OBSERVE, + set_global_tracer_provider=os.environ.get("REGISTER_GLOBAL") == "1", + verbose=False, +) +init_tracing() +with provider.get_tracer("register_first").start_as_current_span("traceai_span"): + print(ask(sys.argv[1])) diff --git a/python/examples/openlit/tests/second_exporter.py b/python/examples/openlit/tests/second_exporter.py new file mode 100644 index 000000000..160203760 --- /dev/null +++ b/python/examples/openlit/tests/second_exporter.py @@ -0,0 +1,31 @@ +"""A test-only fixture: another OTLP exporter in the same process, after ``init_tracing()``. + +It runs ``init_tracing()`` from ``src/app.py`` as written, then builds a +plain OpenTelemetry OTLP/HTTP span exporter for ``SECOND_COLLECTOR`` with no +headers, as another library in the app might, and exports one span through +it. Not part of the recipe. +""" + +from __future__ import annotations + +import os +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "src")) + +from opentelemetry.exporter.otlp.proto.http.trace_exporter import ( # noqa: E402 + OTLPSpanExporter, +) +from opentelemetry.sdk.trace import TracerProvider # noqa: E402 +from opentelemetry.sdk.trace.export import SimpleSpanProcessor # noqa: E402 + +from app import init_tracing # noqa: E402 + +init_tracing() +other = TracerProvider() +other.add_span_processor( + SimpleSpanProcessor(OTLPSpanExporter(endpoint=os.environ["SECOND_COLLECTOR"])) +) +other.get_tracer("second_exporter").start_span("other_library_span").end() +other.shutdown() diff --git a/python/examples/openlit/tests/test_openlit_recipe.py b/python/examples/openlit/tests/test_openlit_recipe.py new file mode 100644 index 000000000..19b6f14dc --- /dev/null +++ b/python/examples/openlit/tests/test_openlit_recipe.py @@ -0,0 +1,1118 @@ +"""Contract test for the OpenLIT recipe. + +Runs ``src/app.py`` as written with the real ``openlit`` 1.45.0 and a real +``openai`` client, in a subprocess whose network is limited to 127.0.0.1 +(``_guarded_run.py``). The model is a loopback fake of the OpenAI Chat +Completions API; spans go to the shared harness ``Receiver``, which serves +``/v1/traces`` and ``/tracer/v1/traces`` like fi-collector's HTTP mux but +does not authenticate, stamp projects or store anything. + +All keys are placeholders. Nothing here contacts OpenLIT, GitHub, OpenAI or +Future AGI. +""" + +from __future__ import annotations + +import ast +import importlib.metadata +import importlib.util +import json +import os +import re +import sys +import threading +from http import HTTPStatus +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from typing import Any, Optional +from urllib.parse import quote, urlsplit + +import pytest + +from harness import Receiver, run + +TESTS_DIR = Path(__file__).resolve().parent +RECIPE_DIR = TESTS_DIR.parent +APP = RECIPE_DIR / "src" / "app.py" +README = RECIPE_DIR / "README.md" +VARIANT = TESTS_DIR / "recipe_variant.py" +TRACED_FUNCTION = TESTS_DIR / "traced_function.py" +TOOL_CALL_SCRIPT = TESTS_DIR / "tool_call.py" +FAILED_CALL = TESTS_DIR / "failed_call.py" +REGISTER_FIRST = TESTS_DIR / "register_first.py" +SECOND_EXPORTER = TESTS_DIR / "second_exporter.py" +GUARD = TESTS_DIR / "_guarded_run.py" +GUARD_PROBE = TESTS_DIR / "_guard_probe.py" + +sys.path.insert(0, str(TESTS_DIR)) +from _fake_openai import RESPONSE_MODEL, USAGE, FakeOpenAI # noqa: E402 + +PROJECT = "openlit-recipe-contract" +SERVICE_NAME = "openlit-recipe" +FI_API_KEY = "fi-api-placeholder-0000" +# A comma and an equals sign: the recipe's percent-encoding must carry them. +FI_SECRET_KEY = "fi-secret-placeholder,part=0000" +OPENAI_KEY = "sk-openai-placeholder-0000" + +# Content markers: the question (input), the fake model's answer (output) +# and the policy text in the system prompt. None may reach an export with +# content off. +QUESTION = "QMARK7c1e what is the refund window?" +ANSWER = "AMARK5d2b refunds are accepted for 30 days." +POLICY = "Refunds are accepted within 30 days of purchase." +CONTENT_MARKERS = ("QMARK7c1e", "AMARK5d2b", POLICY) + +# Markers that DO reach the export with content off; README.md, Privacy, +# lists them as exceptions. A tool call whose generated arguments carry an +# email, the caller's ``user`` parameter, and a server error body. +TOOL_ARGS_MARKER = "TMARK3f9a" +TOOL_CALL = { + "id": "call_fake0001", + "name": "lookup_order", + "arguments": json.dumps({"email": TOOL_ARGS_MARKER + "@example.com"}), +} +END_USER = "UMARK8e4c@example.com" +ERROR_MARKER = "EMARK41d7" +ERROR_BODY = { + "error": { + "message": ERROR_MARKER + " The server had an error processing your request.", + "type": "server_error", + "param": None, + "code": None, + } +} + +LLM_SPAN = "chat gpt-4o-mini" +HTTP_SPAN = "POST" + +# The key inventory: every attribute key openlit 1.45.0 put on each span of +# one non-streaming chat call, content off. README.md lists the same keys. +LLM_SPAN_KEYS = { + "deployment.environment", + "gen_ai.client.token.usage", + "gen_ai.operation.name", + "gen_ai.output.type", + "gen_ai.provider.name", + "gen_ai.request.frequency_penalty", + "gen_ai.request.model", + "gen_ai.request.presence_penalty", + "gen_ai.request.seed", + "gen_ai.request.stream", + "gen_ai.request.temperature", + "gen_ai.request.top_p", + "gen_ai.request.user", + "gen_ai.response.finish_reasons", + "gen_ai.response.id", + "gen_ai.response.model", + "gen_ai.sdk.version", + "gen_ai.server.time_per_output_token", + "gen_ai.server.time_to_first_token", + "gen_ai.usage.cache_creation.input_tokens", + "gen_ai.usage.cache_read.input_tokens", + "gen_ai.usage.cost", + "gen_ai.usage.input_tokens", + "gen_ai.usage.output_tokens", + "openai.api.type", + "openlit.agent.version_hash", + "server.address", + "server.port", + "service.name", + "telemetry.sdk.name", +} +# Added to the model-call span with content on (OpenLIT's default). +CONTENT_KEYS = { + "gen_ai.input.messages", + "gen_ai.output.messages", + "gen_ai.system_instructions", +} +# OpenLIT also enables OpenTelemetry's httpx instrumentor; the OpenAI +# client's HTTP request becomes this child span. +HTTP_SPAN_KEYS = {"http.method", "http.status_code", "http.url"} +RESOURCE_KEYS = { + "deployment.environment", + "project_name", + "project_type", + "service.instance.id", + "service.name", + "telemetry.sdk.language", + "telemetry.sdk.name", + "telemetry.sdk.version", +} + +# A hang guard, not a speed check: one run imports OpenLIT and its +# instrumentors. +RUN_TIMEOUT_SECONDS = 180 + + +def fi_headers() -> dict[str, str]: + """The otlp_headers dict exactly as src/app.py builds it.""" + return { + "x-api-key": quote(FI_API_KEY, safe=""), + "x-secret-key": quote(FI_SECRET_KEY, safe=""), + } + + +class _PathRecorder: + """Loopback catch-all that records every request's method, path and reply. + + By default it answers /v1/traces and /tracer/v1/traces with 200 and + anything else with 404, as fi-collector's HTTP mux does + (pkg/server/server.go:233-234). ``status`` forces one reply for all. + With ``require_keys``, a request without the placeholder Future AGI keys + gets 401 first, as fi-collector's auth middleware does + (pkg/auth/middleware.go:18-21). + """ + + def __init__(self, status: Optional[HTTPStatus] = None, require_keys: bool = False) -> None: + self.calls: list[tuple[str, str, int]] = [] + lock = threading.Lock() + owner = self + + class Handler(BaseHTTPRequestHandler): + def do_POST(self) -> None: # noqa: N802 + path = urlsplit(self.path).path + self.rfile.read(int(self.headers.get("Content-Length", "0"))) + keys = (self.headers.get("x-api-key"), self.headers.get("x-secret-key")) + if status is not None: + reply = status + elif require_keys and keys != (FI_API_KEY, FI_SECRET_KEY): + reply = HTTPStatus.UNAUTHORIZED + elif path in ("/v1/traces", "/tracer/v1/traces"): + reply = HTTPStatus.OK + else: + reply = HTTPStatus.NOT_FOUND + with lock: + owner.calls.append(("POST", path, int(reply))) + self.send_response(reply) + self.send_header("Content-Length", "0") + self.end_headers() + + def log_message(self, format: str, *args: Any) -> None: # noqa: A002 + return + + self._server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + self.origin = "http://127.0.0.1:{0}".format(self._server.server_port) + self._thread = threading.Thread(target=self._server.serve_forever, daemon=True) + self._thread.start() + + def __enter__(self) -> "_PathRecorder": + return self + + def __exit__(self, *_exc: Any) -> None: + self._server.shutdown() + self._server.server_close() + self._thread.join() + + def paths(self) -> set[str]: + return {path for _method, path, _status in self.calls} + + +def _child_env( + home: Path, guard_log: Path, endpoint: str, openai_base_url: str, **overrides: Optional[str] +) -> dict[str, str]: + """The recipe's environment, built from scratch (nothing inherited but PATH). + + Bytecode writing stays on so later runs reuse the SDK's compiled modules; + the scripts themselves run through runpy and leave no .pyc in the repo. + """ + env = { + "PATH": os.environ.get("PATH", ""), + "HOME": str(home), + "LOOPBACK_GUARD_LOG": str(guard_log), + "OTEL_EXPORTER_OTLP_ENDPOINT": endpoint, + "OTEL_RESOURCE_ATTRIBUTES": "project_name={0},project_type=observe".format(PROJECT), + "FI_API_KEY": FI_API_KEY, + "FI_SECRET_KEY": FI_SECRET_KEY, + "OPENAI_API_KEY": OPENAI_KEY, + "OPENAI_BASE_URL": openai_base_url, + } + for key, value in overrides.items(): + if value is None: + env.pop(key, None) + else: + env[key] = value + return env + + +def _guard_attempts(guard_log: Path) -> list[dict[str, Any]]: + if not guard_log.exists(): + return [] + return [json.loads(line) for line in guard_log.read_text().splitlines()] + + +def _launch( + tmp_path: Path, script: Path, endpoint: str, fake: FakeOpenAI, **overrides: Optional[str] +) -> tuple[Any, list[dict[str, Any]]]: + """Run one script under the loopback guard; return its result and the guard log.""" + tmp_path.mkdir(parents=True, exist_ok=True) + guard_log = tmp_path / "guard.jsonl" + env = _child_env(tmp_path, guard_log, endpoint, fake.base_url, **overrides) + result = run( + [sys.executable, str(GUARD), str(script), QUESTION], + env=env, + stdin=None, + timeout=RUN_TIMEOUT_SECONDS, + ) + return result, _guard_attempts(guard_log) + + +def _run_script( + tmp_path: Path, script: Path, fake: Optional[dict[str, Any]] = None, **overrides: Optional[str] +) -> dict[str, Any]: + """Run one script against a Receiver; return everything the tests read. + + ``fake`` holds extra FakeOpenAI arguments (``tool_call``, ``error``). + """ + with Receiver() as receiver, FakeOpenAI(ANSWER, **(fake or {})) as fake_openai: + result, guard_attempts = _launch( + tmp_path, script, receiver.origin, fake_openai, **overrides + ) + record = { + "result": result, + "stdout": result.stdout.decode("utf-8", "replace"), + "stderr": result.stderr.decode("utf-8", "replace"), + "requests": receiver.requests(), + "spans": receiver.spans(), + "model_requests": list(fake_openai.requests), + "model_authorizations": list(fake_openai.authorizations), + "guard_attempts": guard_attempts, + } + return record + + +def _run_variant(tmp_path: Path, **init_overrides: Any) -> dict[str, Any]: + """Run src/app.py with some openlit.init() arguments replaced (None removes one).""" + return _run_script(tmp_path, VARIANT, RECIPE_INIT_OVERRIDES=json.dumps(init_overrides)) + + +def _run_against_recorder( + tmp_path: Path, + suffix: str = "", + status: Optional[HTTPStatus] = None, + require_keys: bool = False, + **overrides: Optional[str], +) -> tuple[Any, list[tuple[str, str, int]], list[dict[str, Any]]]: + """Run a script with a _PathRecorder as the collector; return result, calls, guard log.""" + script = VARIANT if "RECIPE_INIT_OVERRIDES" in overrides else APP + with _PathRecorder(status, require_keys) as recorder, FakeOpenAI(ANSWER) as fake: + result, guard_attempts = _launch( + tmp_path, script, recorder.origin + suffix, fake, **overrides + ) + calls = list(recorder.calls) + return result, calls, guard_attempts + + +def _value(value: dict[str, Any]) -> Any: + for kind in ("stringValue", "boolValue", "doubleValue"): + if kind in value: + return value[kind] + if "intValue" in value: + return int(value["intValue"]) + if "arrayValue" in value: + return [_value(item) for item in value["arrayValue"].get("values", [])] + return value + + +def _attrs(span: dict[str, Any]) -> dict[str, Any]: + return {item["key"]: _value(item.get("value", {})) for item in span.get("attributes", [])} + + +def _by_name(spans: list[dict[str, Any]]) -> dict[str, dict[str, Any]]: + named = {span["name"]: span for span in spans} + assert len(named) == len(spans), "span names repeat: {0}".format( + sorted(span["name"] for span in spans) + ) + return named + + +def _export_dump(record: dict[str, Any]) -> str: + """Everything the export carried except HTTP headers: spans and resources.""" + return json.dumps( + { + "spans": record["spans"], + "resources": [r["resource_attributes"] for r in record["requests"]], + }, + sort_keys=True, + ) + + +def _keys_carrying(record: dict[str, Any], marker: str) -> set[str]: + """Span attribute, event attribute and status keys whose value contains ``marker``.""" + keys: set[str] = set() + for span in record["spans"]: + for attributes in [_attrs(span)] + [_attrs(event) for event in span.get("events", [])]: + keys |= {key for key, value in attributes.items() if marker in str(value)} + if marker in span.get("status", {}).get("message", ""): + keys.add("status.message") + return keys + + +def _assert_ran(record: dict[str, Any]) -> None: + result = record["result"] + assert not result.timed_out, record["stderr"] + assert result.returncode == 0, record["stderr"] + assert record["guard_attempts"] == [], record["guard_attempts"] + assert record["requests"], "no export reached the receiver" + assert record["spans"], "the export carried no spans" + + +# -------------------------------------------------------------------------- +# The recipe as documented: content, metrics and events off, local prices. +# -------------------------------------------------------------------------- + + +@pytest.fixture(scope="module") +def recipe_run(tmp_path_factory: pytest.TempPathFactory) -> dict[str, Any]: + record = _run_script(tmp_path_factory.mktemp("recipe"), APP) + _assert_ran(record) + return record + + +def test_recipe_runs_with_loopback_network_only(recipe_run: dict[str, Any]) -> None: + assert recipe_run["guard_attempts"] == [] + assert ANSWER in recipe_run["stdout"] + # The model call reached the fake, with the OpenAI key the client was given. + assert len(recipe_run["model_requests"]) == 1 + assert recipe_run["model_authorizations"] == ["Bearer " + OPENAI_KEY] + + +def test_guard_refuses_and_logs_non_loopback_connections(tmp_path: Path) -> None: + """Positive control for every ``guard_attempts == []`` assertion in this file.""" + guard_log = tmp_path / "guard.jsonl" + result = run( + [sys.executable, str(GUARD), str(GUARD_PROBE)], + env={"PATH": os.environ.get("PATH", ""), "HOME": str(tmp_path), "LOOPBACK_GUARD_LOG": str(guard_log)}, + stdin=None, + timeout=60, + ) + assert not result.timed_out + assert result.returncode == 0, result.stderr.decode("utf-8", "replace") + assert "refused: connect,connect_ex,getaddrinfo" in result.stdout.decode("utf-8", "replace") + logged = _guard_attempts(guard_log) + assert [entry["kind"] for entry in logged] == ["connect", "connect_ex", "getaddrinfo"] + assert "192.0.2.1" in logged[0]["target"] + assert "2001:db8::1" in logged[1]["target"] + assert "example.invalid" in logged[2]["target"] + + +def test_origin_endpoint_posts_to_v1_traces(recipe_run: dict[str, Any]) -> None: + assert recipe_run["requests"], "no export reached the receiver" + assert {request["path"] for request in recipe_run["requests"]} == {"/v1/traces"} + + +def test_otlp_headers_carry_the_future_agi_keys(recipe_run: dict[str, Any]) -> None: + assert recipe_run["requests"], "no export reached the receiver" + for request in recipe_run["requests"]: + headers = request["headers"] + assert headers.get("x-api-key") == FI_API_KEY + assert headers.get("x-secret-key") == FI_SECRET_KEY + assert "authorization" not in headers + assert all(OPENAI_KEY not in value for value in headers.values()) + + +def test_resource_carries_the_project(recipe_run: dict[str, Any]) -> None: + assert recipe_run["requests"], "no export reached the receiver" + for request in recipe_run["requests"]: + assert request["resource_attributes"], "export without a resource" + for resource in request["resource_attributes"]: + assert set(resource) == RESOURCE_KEYS + assert resource["project_name"] == PROJECT + assert resource["project_type"] == "observe" + assert resource["service.name"] == SERVICE_NAME + assert resource["telemetry.sdk.name"] == "openlit" + assert resource["deployment.environment"] == "default" + + +def test_one_trace_model_call_with_an_http_child(recipe_run: dict[str, Any]) -> None: + spans = _by_name(recipe_run["spans"]) + assert set(spans) == {LLM_SPAN, HTTP_SPAN} + assert len({span["traceId"] for span in spans.values()}) == 1 + llm, http = spans[LLM_SPAN], spans[HTTP_SPAN] + assert not llm.get("parentSpanId") + assert http["parentSpanId"] == llm["spanId"] + assert llm["kind"] == http["kind"] == "SPAN_KIND_CLIENT" + assert llm["status"].get("code") == "STATUS_CODE_OK" + + +def test_key_inventory(recipe_run: dict[str, Any]) -> None: + spans = _by_name(recipe_run["spans"]) + llm = _attrs(spans[LLM_SPAN]) + assert set(llm) == LLM_SPAN_KEYS + assert set(_attrs(spans[HTTP_SPAN])) == HTTP_SPAN_KEYS + + # The values the collector's alias lists can read. + assert llm["gen_ai.operation.name"] == "chat" + assert llm["gen_ai.provider.name"] == "openai" + assert llm["gen_ai.request.model"] == "gpt-4o-mini" + assert llm["gen_ai.response.model"] == RESPONSE_MODEL + assert llm["gen_ai.usage.input_tokens"] == USAGE["prompt_tokens"] + assert llm["gen_ai.usage.output_tokens"] == USAGE["completion_tokens"] + # The total is sent under OpenLIT's own key, not gen_ai.usage.total_tokens. + assert llm["gen_ai.client.token.usage"] == USAGE["total_tokens"] + # With an empty price table OpenLIT still sends a cost, of 0. + assert llm["gen_ai.usage.cost"] == 0 + assert llm["gen_ai.usage.cache_read.input_tokens"] == 0 + assert llm["gen_ai.usage.cache_creation.input_tokens"] == 0 + # Request parameters the call did not set are sent with OpenLIT's defaults. + assert llm["gen_ai.request.temperature"] == 1.0 + assert llm["gen_ai.request.top_p"] == 1.0 + assert llm["gen_ai.request.frequency_penalty"] == 0.0 + assert llm["gen_ai.request.presence_penalty"] == 0.0 + assert llm["gen_ai.request.seed"] == 0 + assert llm["gen_ai.request.user"] == "" + assert llm["gen_ai.request.stream"] is False + assert llm["gen_ai.response.id"] == "chatcmpl-fake" + assert llm["gen_ai.response.finish_reasons"] == ["stop"] + assert llm["gen_ai.output.type"] == "text" + assert llm["gen_ai.sdk.version"] == importlib.metadata.version("openai") + assert isinstance(llm["gen_ai.server.time_to_first_token"], float) + assert llm["gen_ai.server.time_per_output_token"] == 0 + assert llm["openai.api.type"] == "chat_completions" + assert re.fullmatch(r"[0-9a-f]{16}", llm["openlit.agent.version_hash"]) + assert llm["server.address"] == "127.0.0.1" + assert isinstance(llm["server.port"], int) + assert llm["service.name"] == SERVICE_NAME + assert llm["deployment.environment"] == "default" + assert llm["telemetry.sdk.name"] == "openlit" + # None of the keys fi-collector reads a span kind from (converter.go:79-84). + for key in ("fi.span.kind", "gen_ai.span.kind", "llm.request.type", "openinference.span.kind"): + assert key not in llm + assert "gen_ai.system" not in llm + + http = _attrs(spans[HTTP_SPAN]) + assert http["http.method"] == "POST" + assert http["http.url"].startswith("http://127.0.0.1:") + assert http["http.url"].endswith("/v1/chat/completions") + assert http["http.status_code"] == 200 + + +def test_content_off_exports_no_content(recipe_run: dict[str, Any]) -> None: + dump = _export_dump(recipe_run) + for marker in CONTENT_MARKERS: + assert marker not in dump, marker + for span in recipe_run["spans"]: + assert not CONTENT_KEYS & set(_attrs(span)), span["name"] + assert not span.get("events"), span["name"] + + +def test_no_secret_in_the_export_or_output(recipe_run: dict[str, Any]) -> None: + dump = _export_dump(recipe_run) + output = recipe_run["stdout"] + recipe_run["stderr"] + secrets = (FI_API_KEY, FI_SECRET_KEY, quote(FI_SECRET_KEY, safe=""), OPENAI_KEY) + for secret in secrets: + assert secret not in dump + assert secret not in output + + +# -------------------------------------------------------------------------- +# Controls and the documented edges. +# -------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + ("suffix", "expected"), + [ + ("", ("/v1/traces", 200)), + ("/", ("/v1/traces", 200)), + # The exporter always appends v1/traces, so a full path doubles it. + ("/v1/traces", ("/v1/traces/v1/traces", 404)), + # fi-collector also serves /tracer/v1/traces. + ("/tracer", ("/tracer/v1/traces", 200)), + ], +) +def test_endpoint_is_the_origin_and_v1_traces_is_appended( + tmp_path: Path, suffix: str, expected: tuple[str, int] +) -> None: + result, calls, guard_attempts = _run_against_recorder(tmp_path, suffix) + assert not result.timed_out + assert result.returncode == 0, result.stderr.decode("utf-8", "replace") + assert guard_attempts == [] + assert calls, "no export reached the recorder" + assert {(path, status) for _method, path, status in calls} == {expected} + + +def test_unencoded_comma_in_a_header_value_truncates_it(tmp_path: Path) -> None: + """Why the recipe percent-encodes: the exporter splits the headers on commas.""" + record = _run_variant( + tmp_path, otlp_headers={"x-api-key": FI_API_KEY, "x-secret-key": FI_SECRET_KEY} + ) + _assert_ran(record) + for request in record["requests"]: + assert request["headers"].get("x-api-key") == FI_API_KEY + assert request["headers"].get("x-secret-key") == FI_SECRET_KEY.split(",")[0] + + +def test_content_is_exported_when_the_argument_is_left_out(tmp_path: Path) -> None: + """OpenLIT's default is content ON. Control for the content-off test.""" + record = _run_variant(tmp_path, capture_message_content=None) + _assert_ran(record) + llm = _attrs(_by_name(record["spans"])[LLM_SPAN]) + assert set(llm) == LLM_SPAN_KEYS | CONTENT_KEYS + assert "QMARK7c1e" in llm["gen_ai.input.messages"] + assert POLICY in llm["gen_ai.input.messages"] # the system prompt + assert POLICY in llm["gen_ai.system_instructions"] + assert "AMARK5d2b" in llm["gen_ai.output.messages"] + dump = _export_dump(record) + for secret in (FI_API_KEY, FI_SECRET_KEY, OPENAI_KEY): + assert secret not in dump + + +def test_env_false_turns_content_off_over_an_explicit_true(tmp_path: Path) -> None: + record = _run_script( + tmp_path, + VARIANT, + RECIPE_INIT_OVERRIDES=json.dumps({"capture_message_content": True}), + OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT="false", + ) + _assert_ran(record) + dump = _export_dump(record) + for marker in CONTENT_MARKERS: + assert marker not in dump, marker + + +@pytest.mark.parametrize( + ("left_out", "extra_path"), + [ + (None, None), + ("disable_metrics", "/v1/metrics"), + ("disable_events", "/v1/logs"), + ], +) +def test_metrics_and_events_go_to_paths_fi_collector_does_not_serve( + tmp_path: Path, left_out: Optional[str], extra_path: Optional[str] +) -> None: + overrides = {left_out: None} if left_out else {} + result, calls, guard_attempts = _run_against_recorder( + tmp_path, RECIPE_INIT_OVERRIDES=json.dumps(overrides) + ) + assert not result.timed_out + assert result.returncode == 0, result.stderr.decode("utf-8", "replace") + assert guard_attempts == [] + paths = {path for _method, path, _status in calls} + if extra_path is None: + assert paths == {"/v1/traces"} + else: + assert paths == {"/v1/traces", extra_path} + assert {status for _method, path, status in calls if path == extra_path} == {404} + + +def test_default_pricing_download_is_the_only_outbound_attempt(tmp_path: Path) -> None: + """Without pricing_json, init() fetches prices from GitHub. The guard refuses it.""" + record = _run_variant(tmp_path, pricing_json=None) + assert not record["result"].timed_out + assert record["result"].returncode == 0, record["stderr"] + assert [entry["kind"] for entry in record["guard_attempts"]] == ["getaddrinfo"] + assert "raw.githubusercontent.com" in record["guard_attempts"][0]["target"] + # The run still exported; the recipe's run above made no attempt at all. + assert record["requests"] and record["spans"] + + +def test_trace_decorator_ignores_the_content_switch(tmp_path: Path) -> None: + """@openlit.trace records arguments and return value with content off.""" + record = _run_script(tmp_path, TRACED_FUNCTION) + _assert_ran(record) + spans = _by_name(record["spans"]) + assert set(spans) == {"support_request", LLM_SPAN, HTTP_SPAN} + decorated = _attrs(spans["support_request"]) + assert "QMARK7c1e" in decorated["function.args"] + assert decorated["gen_ai.output.messages"] == ANSWER + # The model call under it stays content-free. + assert not CONTENT_KEYS & set(_attrs(spans[LLM_SPAN])) + assert spans[LLM_SPAN]["parentSpanId"] == spans["support_request"]["spanId"] + + +# The next tests prove the exceptions README.md lists under Privacy. They +# show what still leaks with content off; they are not privacy guarantees. + + +@pytest.fixture(scope="module") +def tool_call_run(tmp_path_factory: pytest.TempPathFactory) -> dict[str, Any]: + record = _run_script( + tmp_path_factory.mktemp("tool_call"), + TOOL_CALL_SCRIPT, + fake={"tool_call": TOOL_CALL}, + RECIPE_USER=END_USER, + ) + _assert_ran(record) + return record + + +def test_content_off_still_exports_the_models_tool_calls(tool_call_run: dict[str, Any]) -> None: + """Tool name, call ID and the model's generated arguments (utils.py:1524-1551).""" + request = tool_call_run["model_requests"][0] + assert [tool["function"]["name"] for tool in request["tools"]] == [TOOL_CALL["name"]] + assert tool_call_run["stdout"].strip() == TOOL_CALL["name"] + llm = _attrs(_by_name(tool_call_run["spans"])[LLM_SPAN]) + assert llm["gen_ai.tool.name"] == TOOL_CALL["name"] + assert llm["gen_ai.tool.call.id"] == TOOL_CALL["id"] + # The arguments arrive verbatim, planted email included. + assert llm["gen_ai.tool.args"] == TOOL_CALL["arguments"] + assert TOOL_ARGS_MARKER in llm["gen_ai.tool.args"] + # The messages themselves stay out. + assert not CONTENT_KEYS & set(llm) + dump = _export_dump(tool_call_run) + for marker in CONTENT_MARKERS: + assert marker not in dump, marker + + +def test_content_off_still_exports_the_request_user(tool_call_run: dict[str, Any]) -> None: + """The ``user`` argument goes to gen_ai.request.user (utils.py:1484-1487).""" + assert tool_call_run["model_requests"][0]["user"] == END_USER + llm = _attrs(_by_name(tool_call_run["spans"])[LLM_SPAN]) + assert llm["gen_ai.request.user"] == END_USER + + +@pytest.fixture(scope="module") +def failed_call_run(tmp_path_factory: pytest.TempPathFactory) -> dict[str, Any]: + record = _run_script( + tmp_path_factory.mktemp("failed_call"), + FAILED_CALL, + fake={"error": (HTTPStatus.INTERNAL_SERVER_ERROR, ERROR_BODY)}, + ) + _assert_ran(record) + return record + + +def test_failed_call_records_the_error_body_but_no_content(failed_call_run: dict[str, Any]) -> None: + """OpenLIT records the exception, then the SDK's span context records it again.""" + assert failed_call_run["stdout"].strip() == "raised InternalServerError 500" + assert len(failed_call_run["model_requests"]) == 1 # max_retries=0 + spans = _by_name(failed_call_run["spans"]) + assert set(spans) == {LLM_SPAN, HTTP_SPAN} + llm = spans[LLM_SPAN] + # openai.py:186-209 -> utils.py:77-84, then opentelemetry-api use_span. + events = llm.get("events", []) + assert [event["name"] for event in events] == ["exception", "exception"] + for event in events: + attributes = _attrs(event) + assert attributes["exception.type"] == "openai.InternalServerError" + # The openai SDK puts the whole response body in the message. + assert attributes["exception.message"] == "Error code: 500 - {0}".format(ERROR_BODY) + assert attributes["exception.message"] in attributes["exception.stacktrace"] + assert llm["status"]["code"] == "STATUS_CODE_ERROR" + assert llm["status"]["message"] == "InternalServerError: Error code: 500 - {0}".format( + ERROR_BODY + ) + assert _attrs(llm)["error.type"] == "InternalServerError" + assert _keys_carrying(failed_call_run, ERROR_MARKER) == { + "exception.message", + "exception.stacktrace", + "status.message", + } + # The fake's error body echoes no input, so no prompt or answer text arrives. + dump = _export_dump(failed_call_run) + for marker in CONTENT_MARKERS: + assert marker not in dump, marker + + +def test_rejected_export_is_logged_and_the_app_still_exits_zero(tmp_path: Path) -> None: + """fi-collector answers 401 without valid keys; OpenLIT only logs it.""" + result, calls, guard_attempts = _run_against_recorder( + tmp_path, status=HTTPStatus.UNAUTHORIZED + ) + assert not result.timed_out + assert guard_attempts == [] + assert {(path, status) for _method, path, status in calls} == {("/v1/traces", 401)} + assert result.returncode == 0 + stderr = result.stderr.decode("utf-8", "replace") + assert "Failed to export spans batch code: 401" in stderr, stderr + assert ANSWER in result.stdout.decode("utf-8", "replace") + + +# Environment that overrides the recipe (README.md, The recipe). + + +@pytest.mark.parametrize( + ("traces_headers", "expected_status"), + [(None, HTTPStatus.OK), ("x-team=support", HTTPStatus.UNAUTHORIZED)], +) +def test_traces_headers_variable_replaces_the_recipes_headers( + tmp_path: Path, traces_headers: Optional[str], expected_status: HTTPStatus +) -> None: + """The exporter reads OTEL_EXPORTER_OTLP_TRACES_HEADERS instead of OpenLIT's string.""" + result, calls, guard_attempts = _run_against_recorder( + tmp_path, require_keys=True, OTEL_EXPORTER_OTLP_TRACES_HEADERS=traces_headers + ) + assert not result.timed_out + assert result.returncode == 0, result.stderr.decode("utf-8", "replace") + assert guard_attempts == [] + assert {(path, status) for _method, path, status in calls} == { + ("/v1/traces", int(expected_status)) + } + stderr = result.stderr.decode("utf-8", "replace") + rejected = "Failed to export spans batch code: 401" in stderr + assert rejected == (expected_status == HTTPStatus.UNAUTHORIZED), stderr + + +def test_traces_endpoint_variable_wins_over_otlp_endpoint(tmp_path: Path) -> None: + """A set OTEL_EXPORTER_OTLP_TRACES_ENDPOINT is used as is, and the keys go there.""" + with _PathRecorder() as recipe_target, Receiver() as elsewhere, FakeOpenAI(ANSWER) as fake: + result, guard_attempts = _launch( + tmp_path, + APP, + recipe_target.origin, + fake, + OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=elsewhere.collector_endpoint, + ) + calls, requests = list(recipe_target.calls), elsewhere.requests() + assert not result.timed_out + assert result.returncode == 0, result.stderr.decode("utf-8", "replace") + assert guard_attempts == [] + assert calls == [] + # The full URL, nothing appended: /tracer/v1/traces/v1/traces would 404. + assert requests, "no export reached the other collector" + assert {request["path"] for request in requests} == {"/tracer/v1/traces"} + for request in requests: + assert request["headers"].get("x-api-key") == FI_API_KEY + assert request["headers"].get("x-secret-key") == FI_SECRET_KEY + + +@pytest.mark.parametrize("exporter", ["none", "console"]) +def test_traces_exporter_variable_can_leave_out_the_otlp_exporter( + tmp_path: Path, exporter: str +) -> None: + result, calls, guard_attempts = _run_against_recorder( + tmp_path, OTEL_TRACES_EXPORTER=exporter + ) + assert not result.timed_out + assert result.returncode == 0, result.stderr.decode("utf-8", "replace") + assert guard_attempts == [] + assert calls == [] + stdout = result.stdout.decode("utf-8", "replace") + assert ANSWER in stdout + # console prints each span to stdout instead. + assert ('"name": "{0}"'.format(LLM_SPAN) in stdout) == (exporter == "console") + + +def test_a_later_otlp_exporter_in_the_process_sends_the_future_agi_keys(tmp_path: Path) -> None: + """OpenLIT leaves the keys in OTEL_EXPORTER_OTLP_HEADERS for every later exporter.""" + with Receiver() as recipe_collector, Receiver() as elsewhere, FakeOpenAI(ANSWER) as fake: + result, guard_attempts = _launch( + tmp_path, + SECOND_EXPORTER, + recipe_collector.origin, + fake, + SECOND_COLLECTOR=elsewhere.endpoint, + ) + requests, spans = elsewhere.requests(), elsewhere.spans() + assert not result.timed_out + assert result.returncode == 0, result.stderr.decode("utf-8", "replace") + assert guard_attempts == [] + assert [span["name"] for span in spans] == ["other_library_span"] + for request in requests: + assert request["headers"].get("x-api-key") == FI_API_KEY + assert request["headers"].get("x-secret-key") == FI_SECRET_KEY + + +@pytest.mark.parametrize( + ("unset", "message"), + [ + ("OTEL_EXPORTER_OTLP_ENDPOINT", "Set OTEL_EXPORTER_OTLP_ENDPOINT"), + ("OTEL_RESOURCE_ATTRIBUTES", "Set OTEL_RESOURCE_ATTRIBUTES"), + ("FI_API_KEY", "Set FI_API_KEY and FI_SECRET_KEY"), + ("FI_SECRET_KEY", "Set FI_API_KEY and FI_SECRET_KEY"), + ], +) +def test_app_refuses_to_start_without_endpoint_keys_or_project( + tmp_path: Path, unset: str, message: str +) -> None: + record = _run_script(tmp_path, APP, **{unset: None}) + assert not record["result"].timed_out + assert record["result"].returncode != 0 + assert message in record["stderr"] + assert "Traceback" not in record["stderr"] + for secret in (FI_API_KEY, FI_SECRET_KEY): + assert secret not in record["stderr"] + assert record["requests"] == [] + assert record["model_requests"] == [] + assert record["guard_attempts"] == [] + + +def _skip_without_traceai_openai() -> None: + if importlib.util.find_spec("traceai_openai") is None: + pytest.skip("traceai-openai is not installed; see README.md, Tests") + + +def test_openlit_and_traceai_openai_both_trace_the_same_call(tmp_path: Path) -> None: + """Both enabled: two model-call spans for one call. The README says enable one.""" + _skip_without_traceai_openai() + record = _run_script(tmp_path, VARIANT, RECIPE_ADD_TRACEAI_OPENAI="1") + _assert_ran(record) + spans = _by_name(record["spans"]) + assert set(spans) == {LLM_SPAN, "ChatCompletion", HTTP_SPAN} + assert len(record["model_requests"]) == 1 + assert len({span["traceId"] for span in spans.values()}) == 1 + # OpenLIT's span wraps traceAI's, which wraps the HTTP request. + assert spans["ChatCompletion"]["parentSpanId"] == spans[LLM_SPAN]["spanId"] + assert spans[HTTP_SPAN]["parentSpanId"] == spans["ChatCompletion"]["spanId"] + openlit_llm, traceai_llm = _attrs(spans[LLM_SPAN]), _attrs(spans["ChatCompletion"]) + assert openlit_llm["gen_ai.operation.name"] == "chat" + assert traceai_llm["gen_ai.span.kind"] == "LLM" + # Both carry the same usage, so a sum over spans counts the call twice. + for llm in (openlit_llm, traceai_llm): + assert llm["gen_ai.usage.input_tokens"] == USAGE["prompt_tokens"] + assert llm["gen_ai.usage.output_tokens"] == USAGE["completion_tokens"] + # OpenLIT's content switch does not reach traceAI's span. + assert POLICY in traceai_llm["input.value"] + assert "QMARK7c1e" in traceai_llm["gen_ai.input.messages.1.message.content"] + assert "AMARK5d2b" in traceai_llm["output.value"] + assert not CONTENT_KEYS & set(openlit_llm) + + +def test_disabling_openlits_openai_instrumentor_leaves_one_model_span(tmp_path: Path) -> None: + """The README's way to enable only traceAI's: disabled_instrumentors=["openai"].""" + _skip_without_traceai_openai() + record = _run_script( + tmp_path, + VARIANT, + RECIPE_INIT_OVERRIDES=json.dumps({"disabled_instrumentors": ["openai"]}), + RECIPE_ADD_TRACEAI_OPENAI="1", + ) + _assert_ran(record) + spans = _by_name(record["spans"]) + assert set(spans) == {"ChatCompletion", HTTP_SPAN} + assert spans[HTTP_SPAN]["parentSpanId"] == spans["ChatCompletion"]["spanId"] + + +REGISTER_PROJECT = "register-project" + + +@pytest.mark.parametrize("global_provider", [False, True], ids=["default", "global"]) +def test_register_before_openlit_init(tmp_path: Path, global_provider: bool) -> None: + """traceAI's register() first: one provider only if it was made global.""" + with Receiver() as recipe_collector, Receiver() as register_collector, FakeOpenAI( + ANSWER + ) as fake: + result, guard_attempts = _launch( + tmp_path, + REGISTER_FIRST, + recipe_collector.origin, + fake, + FI_BASE_URL=register_collector.origin, + REGISTER_PROJECT=REGISTER_PROJECT, + REGISTER_GLOBAL="1" if global_provider else None, + ) + recipe_requests, recipe_spans = recipe_collector.requests(), recipe_collector.spans() + register_requests = register_collector.requests() + register_spans = register_collector.spans() + assert not result.timed_out + assert result.returncode == 0, result.stderr.decode("utf-8", "replace") + assert guard_attempts == [] + assert ANSWER in result.stdout.decode("utf-8", "replace") + + def projects(requests: list[dict[str, Any]]) -> set[str]: + return {r["project_name"] for request in requests for r in request["resource_attributes"]} + + # One trace either way: the model call is a child of register()'s span. + spans = _by_name(recipe_spans + register_spans) + assert set(spans) == {"traceai_span", LLM_SPAN, HTTP_SPAN} + assert len({span["traceId"] for span in spans.values()}) == 1 + assert spans[LLM_SPAN]["parentSpanId"] == spans["traceai_span"]["spanId"] + if global_provider: + # openlit.init() reused register()'s provider: otlp_endpoint, + # otlp_headers and OTEL_RESOURCE_ATTRIBUTES did nothing for spans. + assert recipe_requests == [] + assert {span["name"] for span in register_spans} == set(spans) + else: + # Two providers, two exports, two projects. + assert {span["name"] for span in recipe_spans} == {LLM_SPAN, HTTP_SPAN} + assert {request["path"] for request in recipe_requests} == {"/v1/traces"} + assert projects(recipe_requests) == {PROJECT} + assert {span["name"] for span in register_spans} == {"traceai_span"} + assert {request["path"] for request in register_requests} == {"/tracer/v1/traces"} + assert projects(register_requests) == {REGISTER_PROJECT} + for request in register_requests: + assert request["headers"].get("x-api-key") == FI_API_KEY + assert request["headers"].get("x-secret-key") == FI_SECRET_KEY + + +# -------------------------------------------------------------------------- +# README checks, and an opt-in check against the collector's alias lists. +# -------------------------------------------------------------------------- + + +def _readme_section(title: str, level: int = 2) -> str: + """The README text from a heading of this level to the next of the same or higher.""" + text = README.read_text(encoding="utf-8") + start = text.index("\n{0} {1}\n".format("#" * level, title)) + ends = [text.find("\n{0} ".format("#" * n), start + 1) for n in range(2, level + 1)] + ends = [end for end in ends if end != -1] + return text[start : min(ends) if ends else len(text)] + + +def test_readme_key_inventory_matches_the_emitted_keys() -> None: + section = _readme_section("Key inventory") + listed = set(re.findall(r"^\| `([a-z_.]+)` \|", section, flags=re.MULTILINE)) + assert listed == LLM_SPAN_KEYS | CONTENT_KEYS | HTTP_SPAN_KEYS + + +def test_readme_privacy_names_what_content_off_still_exports( + tool_call_run: dict[str, Any], failed_call_run: dict[str, Any] +) -> None: + """Each key a tool call adds, and each key a planted marker reached, is listed.""" + privacy = _readme_section("Privacy") + not_covered = privacy[privacy.index("Not covered by the content switch") :] + tool_llm = _attrs(_by_name(tool_call_run["spans"])[LLM_SPAN]) + exported = ( + (set(tool_llm) - LLM_SPAN_KEYS) + | _keys_carrying(tool_call_run, TOOL_ARGS_MARKER) + | _keys_carrying(tool_call_run, END_USER) + | _keys_carrying(failed_call_run, ERROR_MARKER) + ) + assert {"gen_ai.tool.args", "gen_ai.request.user", "exception.message"} <= exported + missing = sorted(key for key in exported if "`{0}`".format(key) not in not_covered) + assert missing == [], "not listed in README Privacy: {0}".format(missing) + + +def test_readme_names_the_environment_that_overrides_the_recipe() -> None: + """The variables the tests above show replacing or widening the recipe's settings.""" + section = _readme_section("Environment that overrides the recipe", level=3) + for name in ( + "OTEL_EXPORTER_OTLP_TRACES_HEADERS", + "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", + "OTEL_TRACES_EXPORTER", + "OTEL_EXPORTER_OTLP_HEADERS", + ): + assert "`{0}`".format(name) in section, name + + +def _code_blocks(section: str, language: str) -> list[str]: + return re.findall(r"```{0}\n(.*?)```".format(language), section, flags=re.DOTALL) + + +def _init_keywords(source: str) -> dict[Optional[str], ast.expr]: + """The keyword arguments of the one ``openlit.init(...)`` call in ``source``.""" + calls = [ + node + for node in ast.walk(ast.parse(source)) + if isinstance(node, ast.Call) and ast.unparse(node.func) == "openlit.init" + ] + assert len(calls) == 1, source + assert calls[0].args == [] + return {keyword.arg: keyword.value for keyword in calls[0].keywords} + + +def test_readme_init_call_is_the_one_in_app_py() -> None: + blocks = [ + block + for block in _code_blocks(_readme_section("The recipe"), "python") + if "openlit.init(" in block + ] + assert len(blocks) == 1 + readme = _init_keywords(blocks[0]) + app = _init_keywords(APP.read_text(encoding="utf-8")) + assert {key: ast.dump(value) for key, value in readme.items()} == { + key: ast.dump(value) for key, value in app.items() + } + assert ast.literal_eval(app["capture_message_content"]) is False + + +def test_readme_run_block_sets_what_app_py_checks(monkeypatch: pytest.MonkeyPatch) -> None: + from opentelemetry.sdk.resources import OTELResourceDetector + + (block,) = _code_blocks(_readme_section("Run"), "bash") + exports = dict(re.findall(r'^export (\w+)="([^"]*)"', block, flags=re.MULTILINE)) + for name in ( + "FI_API_KEY", + "FI_SECRET_KEY", + "OTEL_EXPORTER_OTLP_ENDPOINT", + "OTEL_RESOURCE_ATTRIBUTES", + "OPENAI_API_KEY", + ): + assert name in exports, name + # The parser src/app.py checks the project with. + monkeypatch.setenv("OTEL_RESOURCE_ATTRIBUTES", exports["OTEL_RESOURCE_ATTRIBUTES"]) + assert OTELResourceDetector().detect().attributes.get("project_name") + + +def _collector_aliases() -> dict[str, Any]: + src = os.environ.get("FI_COLLECTOR_SRC") + if not src: + pytest.skip("set FI_COLLECTOR_SRC to a future-agi checkout's fi-collector directory") + adapter = (Path(src) / "pkg" / "adapter" / "adapter.go").read_text(encoding="utf-8") + converter = (Path(src) / "exporter" / "clickhouse25exporter" / "converter.go").read_text( + encoding="utf-8" + ) + + def string_slice(source: str, name: str) -> list[str]: + match = re.search(r"\b{0}\s*=\s*\[\]string\{{(.*?)\}}".format(name), source, re.DOTALL) + assert match, name + body = re.sub(r"//[^\n]*", "", match.group(1)) + return re.findall(r'"([^"]+)"', body) + + synonyms_body = re.search( + r"\bspanKindSynonyms\s*=\s*map\[string\]string\{(.*?)\}", converter, re.DOTALL + ) + assert synonyms_body + names = ( + "modelNameKeys", + "providerKeys", + "inputTokenKeys", + "outputTokenKeys", + "totalTokenKeys", + "costTotalKeys", + "costInputKeys", + "costOutputKeys", + ) + aliases: dict[str, Any] = {name: string_slice(adapter, name) for name in names} + aliases["spanKindAttrKeys"] = string_slice(converter, "spanKindAttrKeys") + aliases["operationNameAttrKeys"] = string_slice(converter, "operationNameAttrKeys") + aliases["spanKindSynonyms"] = dict( + re.findall(r'"([^"]+)"\s*:\s*"([^"]+)"', synonyms_body.group(1)) + ) + # The input and output columns: `input := overflowAsString(overflow, "...")`. + for column in ("input", "output"): + aliases[column] = re.findall( + r'\b{0}\s*:?=\s*overflowAsString\(\s*\w+\s*,\s*"([^"]+)"\s*\)'.format(column), + converter, + ) + assert aliases[column], column + return aliases + + +def _first(attrs: dict[str, Any], keys: list[str]) -> Optional[str]: + return next((key for key in keys if attrs.get(key) not in (None, "")), None) + + +def test_readme_display_table_matches_the_collector_aliases(recipe_run: dict[str, Any]) -> None: + """Which emitted key fills each fi-collector column, read from the Go source.""" + aliases = _collector_aliases() + llm = _attrs(_by_name(recipe_run["spans"])[LLM_SPAN]) + kind_key = _first(llm, aliases["spanKindAttrKeys"]) or _first( + llm, aliases["operationNameAttrKeys"] + ) + expected = { + "model": _first(llm, aliases["modelNameKeys"]), + "provider": _first(llm, aliases["providerKeys"]), + "prompt_tokens": _first(llm, aliases["inputTokenKeys"]), + "completion_tokens": _first(llm, aliases["outputTokenKeys"]), + "total_tokens": _first(llm, aliases["totalTokenKeys"]) or "derived", + "cost": _first( + llm, + aliases["costTotalKeys"] + aliases["costInputKeys"] + aliases["costOutputKeys"], + ), + "observation_type": kind_key, + "input": _first(llm, aliases["input"]), + "output": _first(llm, aliases["output"]), + } + # gen_ai.operation.name=chat resolves to the llm observation type. + assert aliases["spanKindSynonyms"].get(llm[kind_key]) == "llm" + + section = _readme_section("What Future AGI shows") + rows = { + column: key or word + for column, key, word in re.findall( + r"^\| `([a-z_]+)` \| (?:`([a-z_.]+)`|(derived|none))", + section, + flags=re.MULTILINE, + ) + } + assert rows == {column: key or "none" for column, key in expected.items()} + # The input and output rows name exactly the keys converter.go reads, and + # the content-on keys (Key inventory: "not read as the span's input") are + # not among them. + for column in ("input", "output"): + row = re.search(r"^\| `{0}` \|([^|]*)\|".format(column), section, flags=re.MULTILINE) + assert row, column + assert set(re.findall(r"`([a-z_.]+)`", row.group(1))) == set(aliases[column]), column + assert not CONTENT_KEYS & set(aliases["input"] + aliases["output"]) diff --git a/python/examples/openlit/tests/tool_call.py b/python/examples/openlit/tests/tool_call.py new file mode 100644 index 000000000..161239a8d --- /dev/null +++ b/python/examples/openlit/tests/tool_call.py @@ -0,0 +1,45 @@ +"""A test-only fixture: one tool-using model call after the recipe's ``init_tracing()``. + +It runs ``init_tracing()`` from ``src/app.py`` as written (content off), then +makes one Chat Completions call with the recipe's messages, a ``tools`` +list and ``user`` set to ``RECIPE_USER``. The fake answers with a tool call, +so the contract test can show what OpenLIT exports for tool calls and for +``user`` with content off. Not part of the recipe. +""" + +from __future__ import annotations + +import os +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "src")) + +from openai import OpenAI # noqa: E402 + +from app import _POLICY, MODEL, init_tracing # noqa: E402 + +LOOKUP_ORDER = { + "type": "function", + "function": { + "name": "lookup_order", + "description": "Find a customer's latest order.", + "parameters": { + "type": "object", + "properties": {"email": {"type": "string"}}, + "required": ["email"], + }, + }, +} + +init_tracing() +response = OpenAI().chat.completions.create( + model=MODEL, + messages=[ + {"role": "system", "content": "Answer using only this policy: " + _POLICY}, + {"role": "user", "content": sys.argv[1]}, + ], + tools=[LOOKUP_ORDER], + user=os.environ["RECIPE_USER"], +) +print(response.choices[0].message.tool_calls[0].function.name) diff --git a/python/examples/openlit/tests/traced_function.py b/python/examples/openlit/tests/traced_function.py new file mode 100644 index 000000000..91c0f8016 --- /dev/null +++ b/python/examples/openlit/tests/traced_function.py @@ -0,0 +1,26 @@ +"""A test-only fixture: the recipe's model call inside OpenLIT's ``@openlit.trace``. + +It runs ``init_tracing()`` from ``src/app.py`` as written (content off), so +the contract test can show which content the ``@openlit.trace`` decorator +records anyway. Not part of the recipe. +""" + +from __future__ import annotations + +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "src")) + +import openlit # noqa: E402 +from app import ask, init_tracing # noqa: E402 + +init_tracing() + + +@openlit.trace +def support_request(question: str) -> str: + return ask(question) + + +print(support_request(sys.argv[1])) diff --git a/python/tests/harness/__init__.py b/python/tests/harness/__init__.py new file mode 100644 index 000000000..cf79047f1 --- /dev/null +++ b/python/tests/harness/__init__.py @@ -0,0 +1,238 @@ +"""Small, loopback-only utilities for OTLP contract tests.""" + +from __future__ import annotations + +import copy +import difflib +import json +import subprocess +import threading +from http import HTTPStatus +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from typing import Any, Mapping, Optional, Sequence, Union +from urllib.parse import urlsplit +from urllib.request import Request, urlopen + + +class Receiver: + """An in-memory OTLP/HTTP receiver for tests. + + The receiver only listens on loopback and never forwards received spans. + """ + + def __init__(self) -> None: + self._spans: list[dict[str, Any]] = [] + self._requests: list[dict[str, Any]] = [] + self._lock = threading.Lock() + owner = self + + class Handler(BaseHTTPRequestHandler): + def _read_body(self) -> bytes: + # The Node OTLP exporter streams with Transfer-Encoding: chunked + # and sends no Content-Length, so both framings are accepted. + if "chunked" in self.headers.get("Transfer-Encoding", "").lower(): + chunks = [] + while True: + size_line = self.rfile.readline().split(b";", 1)[0].strip() + size = int(size_line, 16) + if size == 0: + while self.rfile.readline() not in (b"\r\n", b"\n", b""): + pass + return b"".join(chunks) + chunks.append(self.rfile.read(size)) + self.rfile.readline() + length = int(self.headers.get("Content-Length", "0")) + return self.rfile.read(length) + + def do_POST(self) -> None: # noqa: N802 - required by BaseHTTPRequestHandler + path = urlsplit(self.path).path + if path not in ("/v1/traces", "/tracer/v1/traces"): + self.send_error(HTTPStatus.NOT_FOUND) + return + + try: + body = self._read_body() + content_type = self.headers.get("Content-Type", "").lower() + + if content_type.startswith("application/json"): + request = json.loads(body.decode("utf-8")) + elif content_type.startswith(("application/x-protobuf", "application/protobuf")): + from google.protobuf.json_format import MessageToDict + from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ( + ExportTraceServiceRequest, + ) + + protobuf_request = ExportTraceServiceRequest() + protobuf_request.ParseFromString(body) + request = MessageToDict(protobuf_request) + else: + self.send_error(HTTPStatus.UNSUPPORTED_MEDIA_TYPE) + return + + spans: list[dict[str, Any]] = [] + resource_attributes: list[dict[str, Any]] = [] + for resource_spans in request.get( + "resourceSpans", request.get("resource_spans", []) + ): + resource_attributes.append( + _flatten_attributes( + resource_spans.get("resource", {}).get("attributes", []) + ) + ) + for scope_spans in resource_spans.get( + "scopeSpans", resource_spans.get("scope_spans", []) + ): + spans.extend(scope_spans.get("spans", [])) + except (ImportError, TypeError, UnicodeDecodeError, ValueError) as error: + self.send_error(HTTPStatus.BAD_REQUEST, str(error)) + return + + with owner._lock: + owner._spans.extend(copy.deepcopy(spans)) + owner._requests.append( + { + "path": path, + "headers": { + key.lower(): value for key, value in self.headers.items() + }, + "resource_attributes": resource_attributes, + } + ) + self.send_response(HTTPStatus.OK) + self.send_header("Content-Length", "0") + self.end_headers() + + def log_message(self, _format: str, *args: Any) -> None: + return + + self._server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + self.origin = "http://127.0.0.1:{0}".format(self._server.server_port) + self.endpoint = "{0}/v1/traces".format(self.origin) + self.collector_endpoint = "{0}/tracer/v1/traces".format(self.origin) + self._thread = threading.Thread(target=self._server.serve_forever, daemon=True) + self._thread.start() + + def __enter__(self) -> "Receiver": + return self + + def __exit__(self, _type: Any, _value: Any, _traceback: Any) -> None: + self._server.shutdown() + self._server.server_close() + self._thread.join() + + def spans(self) -> list[dict[str, Any]]: + """Return a copy of the decoded spans received so far.""" + with self._lock: + return copy.deepcopy(self._spans) + + def requests(self) -> list[dict[str, Any]]: + """Return one record per accepted export: path, lower-cased headers, resource attributes.""" + with self._lock: + return copy.deepcopy(self._requests) + + def clear(self) -> None: + """Remove every decoded span and request record received so far.""" + with self._lock: + self._spans.clear() + self._requests.clear() + + +def _flatten_attributes(attributes: Sequence[Mapping[str, Any]]) -> dict[str, Any]: + """Turn OTLP key/value attributes into a plain dict of scalar values.""" + flat: dict[str, Any] = {} + for attribute in attributes: + value = attribute.get("value", {}) + for kind in ("stringValue", "boolValue", "intValue", "doubleValue"): + if kind in value: + flat[attribute.get("key", "")] = value[kind] + break + else: + flat[attribute.get("key", "")] = value + return flat + + +def run( + argv: Sequence[str], + env: Mapping[str, str], + stdin: Optional[bytes], + timeout: float, +) -> subprocess.CompletedProcess[bytes]: + """Run one process and return its captured output, including after a timeout.""" + process = subprocess.Popen( + argv, + env=dict(env), + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + timed_out = False + try: + stdout, stderr = process.communicate(input=stdin, timeout=timeout) + except subprocess.TimeoutExpired: + timed_out = True + process.kill() + stdout, stderr = process.communicate() + + result = subprocess.CompletedProcess(argv, process.returncode, stdout, stderr) + result.timed_out = timed_out + return result + + +def post_otlp(spans: dict[str, Any], endpoint: str) -> int: + """Post one JSON-encoded OTLP request to a loopback receiver.""" + parsed = urlsplit(endpoint) + if parsed.scheme != "http" or parsed.hostname != "127.0.0.1": + raise ValueError("post_otlp only sends to a loopback HTTP endpoint") + + request = Request( + endpoint, + data=json.dumps(spans).encode("utf-8"), + headers={"Content-Type": "application/json"}, + method="POST", + ) + with urlopen(request, timeout=5) as response: + return response.status + + +def compare(actual: Sequence[dict[str, Any]], golden_path: Union[str, Path]) -> None: + """Fail with a diff unless span names, attributes, and statuses match a golden.""" + with Path(golden_path).open(encoding="utf-8") as golden_file: + expected = json.load(golden_file) + + def canonical(spans: Sequence[dict[str, Any]]) -> list[dict[str, Any]]: + normalized = [] + for span in spans: + attributes = span.get("attributes", []) + if isinstance(attributes, dict): + attributes = [ + {"key": key, "value": value} for key, value in attributes.items() + ] + normalized.append( + { + "name": span.get("name", ""), + "attributes": sorted( + attributes, + key=lambda attribute: json.dumps( + attribute, sort_keys=True, separators=(",", ":") + ), + ), + "status": span.get("status", {}), + } + ) + return normalized + + actual_normalized = canonical(actual) + expected_normalized = canonical(expected) + if actual_normalized != expected_normalized: + diff = "\n".join( + difflib.unified_diff( + json.dumps(expected_normalized, indent=2, sort_keys=True).splitlines(), + json.dumps(actual_normalized, indent=2, sort_keys=True).splitlines(), + fromfile=str(golden_path), + tofile="actual", + lineterm="", + ) + ) + print(diff) + raise AssertionError("spans do not match golden") diff --git a/python/tests/harness/test_harness.py b/python/tests/harness/test_harness.py new file mode 100644 index 000000000..bd61d5adc --- /dev/null +++ b/python/tests/harness/test_harness.py @@ -0,0 +1,189 @@ +"""Contract tests for the shared loopback-only harness.""" + +import json +import sys + +import pytest + +from tests.harness import Receiver, compare, post_otlp, run + + +def test_receiver_stores_posted_span_and_can_clear() -> None: + payload = { + "resourceSpans": [ + { + "scopeSpans": [ + { + "spans": [ + { + "traceId": "trace-id", + "spanId": "span-id", + "name": "harness.receiver", + "attributes": [ + {"key": "component", "value": {"stringValue": "test"}} + ], + "status": {"code": 1}, + } + ] + } + ] + } + ] + } + + with Receiver() as receiver: + assert post_otlp(payload, receiver.endpoint) == 200 + assert receiver.spans() == payload["resourceSpans"][0]["scopeSpans"][0]["spans"] + receiver.clear() + assert receiver.spans() == [] + + +def test_receiver_accepts_the_collector_path() -> None: + payload = { + "resourceSpans": [ + {"scopeSpans": [{"spans": [{"name": "collector.path"}]}]} + ] + } + + with Receiver() as receiver: + assert post_otlp(payload, receiver.collector_endpoint) == 200 + assert receiver.spans() == [{"name": "collector.path"}] + + +def _post_chunked(url: str, body: bytes, headers: dict) -> int: + """Send a body with Transfer-Encoding: chunked, as the Node OTLP exporter does.""" + import http.client + from urllib.parse import urlsplit + + parts = urlsplit(url) + connection = http.client.HTTPConnection(parts.hostname, parts.port, timeout=5) + try: + chunks = [body[: len(body) // 2], body[len(body) // 2 :]] + connection.request( + "POST", + parts.path, + body=iter(chunks), + headers={**headers, "Transfer-Encoding": "chunked"}, + encode_chunked=True, + ) + return connection.getresponse().status + finally: + connection.close() + + +def test_receiver_decodes_a_chunked_body() -> None: + payload = { + "resourceSpans": [ + {"scopeSpans": [{"spans": [{"name": "chunked.body"}]}]} + ] + } + + with Receiver() as receiver: + status = _post_chunked( + receiver.collector_endpoint, + json.dumps(payload).encode("utf-8"), + {"Content-Type": "application/json"}, + ) + assert status == 200 + assert receiver.spans() == [{"name": "chunked.body"}] + + +def test_receiver_records_path_headers_and_resource_attributes() -> None: + payload = { + "resourceSpans": [ + { + "resource": { + "attributes": [ + {"key": "project_name", "value": {"stringValue": "harness"}}, + {"key": "project_type", "value": {"stringValue": "observe"}}, + ] + }, + "scopeSpans": [{"spans": [{"name": "with.resource"}]}], + } + ] + } + + with Receiver() as receiver: + status = _post_chunked( + receiver.collector_endpoint, + json.dumps(payload).encode("utf-8"), + { + "Content-Type": "application/json", + "X-Api-Key": "test-key", + "X-Secret-Key": "test-secret", + }, + ) + assert status == 200 + [request] = receiver.requests() + assert request["path"] == "/tracer/v1/traces" + assert request["headers"]["x-api-key"] == "test-key" + assert request["headers"]["x-secret-key"] == "test-secret" + assert "authorization" not in request["headers"] + assert request["resource_attributes"] == [ + {"project_name": "harness", "project_type": "observe"} + ] + receiver.clear() + assert receiver.requests() == [] + + +def test_run_captures_output_and_kills_timed_out_process() -> None: + completed = run( + [ + sys.executable, + "-c", + "import sys; print('stdout'); sys.stderr.write('stderr\\n')", + ], + env={}, + stdin=b"", + timeout=1, + ) + assert completed.returncode == 0 + assert completed.stdout == b"stdout\n" + assert completed.stderr == b"stderr\n" + assert completed.timed_out is False + + timed_out = run( + [sys.executable, "-c", "import time; time.sleep(30)"], + env={}, + stdin=b"", + timeout=0.1, + ) + assert timed_out.timed_out is True + assert timed_out.returncode != 0 + + +def test_compare_accepts_normalized_match_and_prints_mismatch_diff(tmp_path, capsys) -> None: + golden = [ + { + "traceId": "golden-trace", + "spanId": "golden-span", + "startTimeUnixNano": "1", + "endTimeUnixNano": "2", + "name": "harness.compare", + "attributes": [ + {"key": "alpha", "value": {"stringValue": "a"}}, + {"key": "beta", "value": {"intValue": "2"}}, + ], + "status": {"code": 1}, + } + ] + golden_path = tmp_path / "spans.golden.json" + golden_path.write_text(json.dumps(golden), encoding="utf-8") + + actual = [ + { + "traceId": "other-trace", + "spanId": "other-span", + "startTimeUnixNano": "3", + "endTimeUnixNano": "4", + "name": "harness.compare", + "attributes": list(reversed(golden[0]["attributes"])), + "status": {"code": 1}, + } + ] + compare(actual, golden_path) + + actual[0]["name"] = "harness.mismatch" + with pytest.raises(AssertionError, match="spans do not match golden"): + compare(actual, golden_path) + assert "---" in capsys.readouterr().out