diff --git a/integrations/openwebui/README.md b/integrations/openwebui/README.md new file mode 100644 index 000000000..3ada867bf --- /dev/null +++ b/integrations/openwebui/README.md @@ -0,0 +1,177 @@ +# Open WebUI → Future AGI (filter function) + +`futureagi_filter.py` is an Open WebUI +[Function](https://docs.openwebui.com/features/extensibility/plugin/functions/filter) +of type filter. It turns each completed chat turn into a Future AGI trace. It +does not modify Open WebUI and does not import traceAI. + +Tested end to end with Open WebUI 0.11.4 (`pip install open-webui==0.11.4`), +an OpenAI-compatible model connection and a local Future AGI stack. The UI +chat was streamed through a real browser, and direct API calls were checked +on both streaming and non-streaming paths. + +## Install + +1. In Open WebUI, open **Admin Panel → Functions** and create a function. + Paste the whole of `futureagi_filter.py` and save. Keep the opening `"""` + on its own first line: Open WebUI reads the title and requirements from + that block only then. +2. Open WebUI installs the frontmatter requirements when + `ENABLE_PIP_INSTALL_FRONTMATTER_REQUIREMENTS=true` (the default). It does + not install them when `OFFLINE_MODE=true`; saving then fails visibly if + the OTel exporter is unavailable. The filter requires the OTLP/HTTP + exporter and `certifi`. `opentelemetry-api` and `opentelemetry-sdk` are + intentionally not listed: Open WebUI supplies the SDK. Keep the exporter + and SDK on the same OpenTelemetry release line; if your image pins an SDK, + use its matching exporter release and restart after changing packages. +3. Turn the function on, and either mark it **Global** (every model) or add it + to the models you want traced (**Workspace → Models → model → Filters**). + A filter that is off, or neither global nor attached to a model, never runs. +4. Open the function's **Valves** (gear icon) and set: + +| Valve | Default | Meaning | +|---|---|---| +| `api_key`, `secret_key` | empty | Future AGI keys, sent as `X-Api-Key` and `X-Secret-Key`. They are password fields. With either one empty, nothing is exported. | +| `endpoint` | `https://api.futureagi.com` | Future AGI origin. The filter posts OTLP/HTTP to `{endpoint}/tracer/v1/traces`; a value already ending in that path is used as is. | +| `project` | `open-webui` | Future AGI project, sent as `project_name` and `project_type=observe`. | +| `redact` | `true` | Leaves prompts and replies out of every span. **Set `redact=false` to see prompts and replies in Observe.** | +| `include_email_hash` | `false` | Adds a pseudonymous HMAC of the normalised email. `redact=true` prevents content capture; with `redact=false`, the email can appear inside captured prompt/history content even though it is not emitted as a standalone user attribute. | +| `email_hash_salt` | empty | Optional admin-controlled HMAC salt. Use a secret value if email joins must be resistant to offline candidate testing. | +| `max_chars` | `32000` | Maximum characters copied into each input/output value. `0` means unlimited. | +| `priority` | `0` | Open WebUI filter priority; lower values run first. Put a PII-scrubbing filter before this one when `redact=false`. | + +Enable `ENABLE_VALVE_ENCRYPTION=true` so Open WebUI encrypts these valves in +its database. Do not rely on the default plaintext storage for production +keys. + +## What a turn looks like + +Open WebUI calls the filter's `inlet` before the model call, its `stream` +hook for decoded streaming events, and its `outlet` after the reply. The +filter stores turn state on Open WebUI's request `__metadata__`; it keeps no +per-turn message dictionary. Each completed turn becomes one trace of two +spans: + +| Span | Type | Attributes | +|---|---|---| +| `openwebui.chat` | chain (`fi.span.kind=CHAIN`) | `session.id` and `gen_ai.conversation.id` (the Open WebUI chat id), `user.id`, optional pseudonymous `user.email_hash`, and with `redact=false` `input.value` and `output.value` | +| `openwebui.llm`, its child | llm (`fi.span.kind=LLM`) | model, usage, the same ids, and with `redact=false` the conversation JSON as `input.value` plus the reply as `output.value` | + +The root is explicitly detached from Open WebUI's own current server span, so +it is a real Future AGI trace root even when native Open WebUI tracing is on. +The LLM span is a child of that root. The LLM span's latency is **turn +latency**, not model-only latency: it includes Open WebUI preprocessing, +retrieval, tools, database work and response publishing between `inlet` and +`outlet`. + +- The exporter is cached per valve configuration and uses a bounded batch + queue with explicit delay, batch-size, and timeout values. Open WebUI's + process-wide `OTEL_BSP_*` environment variables cannot change or invalidate + this filter's processor. The chat does not wait for Future AGI, and a + collector outage does not break the chat. A process killed before its queue + flushes can lose spans. +- For ordinary replies, usage reads `prompt_tokens`/`completion_tokens`. + For a reply containing function-call output items, it reads cumulative + `input_tokens`/`output_tokens` from Open WebUI's native tool loop. Total is + computed as input plus output because Open WebUI can double-count its stored + `total_tokens` when providers repeat usage chunks. +- A provider error observed by the filter's `stream` hook is copied to the + request `__metadata__` and survives until `outlet`. On the stock SSE path, + Open WebUI parses a `data:` line and invokes `process_filter_functions` with + the decoded event before it emits that event, so the filter can mark a + partial-content outlet `ERROR` without discarding the partial content. The + filter also accepts a raw JSON/bytes line for deployments that explicitly + adapt the host stream branch, but that is not stock Open WebUI behavior. +- **Stock Open WebUI 0.11.4 limitation:** in `open_webui/utils/middleware.py` + lines 5071-5093, a plain JSON provider error line such as + `{"error": ...}` is parsed, persisted/emitted, and then `continue`s before + `process_filter_functions(..., filter_type='stream', ...)` at lines + 5101-5109. The Function therefore receives no `stream` call and no failure + marker is added to request `__metadata__`. Later, `outlet_filter_handler` + builds the outlet body from selected message fields at lines 4123-4143; + the error is not included there either. A filter-only implementation cannot + distinguish a successful no-usage reply from a prior plain-JSON error after + this branch, so this path cannot be marked reliably as `ERROR` without an + Open WebUI change. Empty replies with no usage are still marked `ERROR`; + failures that bypass both the stream hook and outlet can produce no + application span. +- Upstream change needed: invoke stream filters for the plain JSON error + object before the branch's `continue`, or expose that error in the metadata + or outlet payload passed to Function filters. +- An outlet call without a matching inlet is skipped. This avoids duplicate + zero-duration traces from the legacy `/api/chat/completed` endpoint. + +## Live model verification (2026-10-19) + +Re-run against the real OpenAI API (`https://api.openai.com/v1`, `gpt-4o-mini`) as the Open WebUI model connection, instead of a local stand-in. Open WebUI 0.11.4 (`pip install open-webui==0.11.4`, Python 3.12), filter `futureagi_filter.py` version 0.3.0 installed through the Open WebUI API as in the install steps, exporting to a project on a local Future AGI. 0.12.3 was not re-run. + +| Turn | Trace id in Observe | Result | +| --- | --- | --- | +| Plain, default valves (`redact=true`) | `0ed539cf-1c03-5a63-e9c0-24a2b04964bc` | `openwebui.chat` (chain) with child `openwebui.llm`, model `gpt-4o-mini`, 3189 in / 2 out tokens, session = chat id, no content | +| Native tool call (`add`) | `43ac29ee-713e-368e-a601-e6cd784c29cc` | same two spans, 6471 in / 31 out tokens (cumulative over the tool loop), no content | +| Plain, `redact=false` | `6d2b79b4-ab6e-214a-8abe-4fb776d1ef11` | prompt and reply stored (`input` 53 B / `output` 2 B on the chain span) | +| Unknown model id | none | see below | + +Privacy: with `redact=true` the unique marker (in the prompt of both turns) was in none of the 4 stored rows, in any column, and the OpenAI key was not either. With `redact=false` the marker was in the 2 rows of that turn. + +Unknown model id: a model id that Open WebUI does not list is rejected by Open WebUI itself with HTTP 400 `Model not found` before any filter runs, so there is no trace. With the id listed in the connection (`gpt-4o-mini-not-a-model`), OpenAI returned HTTP 404 `model_not_found`; Open WebUI logged the failure and the filter exported **no span**. This is the stock-0.11.4 limitation already described above (a plain provider error is not passed to the filter), now confirmed against the real API: an upstream error does not produce an ERROR span. + +## Privacy + +- Content is off by default (`redact=true`): no message text leaves Open + WebUI. Ids, model and token counts remain visible in Observe. +- Set `redact=false` to see prompts and replies in Observe. For a saved chat, + the stored history is copied; it does not necessarily include the system + prompt, RAG context or tool results. For a temporary chat, Open WebUI's + processed request may include system/RAG context, so `temporary:` and + legacy `local:` chat ids force redaction even when `redact=false`. +- `include_email_hash` is pseudonymous, not anonymous: the filter hashes + `email.strip().lower()` with HMAC-SHA256. An optional salt valve is + available. The raw email is not put in attributes by this filter, although + an admin who explicitly turns content capture on should treat prompt text + as capable of containing any user field. +- The Future AGI keys travel only as request headers, never as span + attributes. `OTEL_RESOURCE_ATTRIBUTES`, `OTEL_SERVICE_NAME`, sampler, + `OTEL_BSP_*`, certificate, client-key, timeout, and compression environment + variables are not inherited by this filter's provider: it supplies an + explicit resource, `ALWAYS_ON` sampler, safe batch settings, a 30-second + request timeout, no request compression, and the system CA bundle. + `OTEL_SDK_DISABLED=true` still disables the OpenTelemetry SDK globally, + including this filter; unset it when the filter should run. + +## Native Open WebUI tracing + +The native Open WebUI exporter (`ENABLE_OTEL` and +`ENABLE_OTEL_TRACES`) is a separate path for HTTP, database and Redis spans. +The filter's root is detached from that server span, so its application traces +are not parented under native infrastructure traces. They remain separate +traces in Future AGI; enable native tracing only if you also want the +infrastructure view. + +## Limits + +- Model ids are Open WebUI's, which may differ from the provider's model name. +- A failed non-streaming request or user Stop can bypass `outlet`, so no + application span is possible through this Function hook. +- Each Open WebUI worker process exports on its own. Function edits create a + new instance; the old provider is finalised and shut down. Spans still + queued when a worker is killed are lost. + +## Tests + +From the repository root: + +```bash +uv run --no-project --with pytest --with pydantic \ + --with opentelemetry-sdk --with opentelemetry-exporter-otlp-proto-http \ + pytest integrations/openwebui/tests/test_filter.py --noconftest -q +``` + +The tests call the hooks with the arguments Open WebUI 0.11.4 passes +(`body`, `__user__`, `__metadata__`; valves set on the instance before each +hook), check the real Open WebUI frontmatter parser when that package is +available (otherwise the pinned parser rules are reproduced), and export to +an in-memory exporter or a loopback OTLP/HTTP server. They cover detached +roots, tool-loop usage, in-stream empty-reply errors, env isolation, bounded +content, temporary-chat redaction, valve caching, and the no-inlet duplicate +case. No external network is used by the unit tests. diff --git a/integrations/openwebui/futureagi_filter.py b/integrations/openwebui/futureagi_filter.py new file mode 100644 index 000000000..b9cbb557a --- /dev/null +++ b/integrations/openwebui/futureagi_filter.py @@ -0,0 +1,561 @@ +""" +title: Future AGI +author: Future AGI +version: 0.3.0 +license: Apache-2.0 +description: Send Open WebUI chat turns to Future AGI. +requirements: opentelemetry-exporter-otlp-proto-http,certifi +""" + +# Standalone on purpose. Open WebUI installs the frontmatter requirements +# into its own environment. The OTel imports are deliberately top-level: when +# Open WebUI cannot install the requirements, saving the function fails rather +# than silently making every hook a no-op. + +import hashlib +import hmac +import json +import logging +import os +import threading +import time +import weakref +from collections.abc import Iterator +from contextlib import contextmanager +from typing import Any + +import certifi +from opentelemetry import context, trace +from opentelemetry.exporter.otlp.proto.http.trace_exporter import ( + Compression, + OTLPSpanExporter, +) +from opentelemetry.sdk.resources import Resource +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import BatchSpanProcessor +from opentelemetry.sdk.trace.sampling import ALWAYS_ON +from opentelemetry.trace import Status, StatusCode +from pydantic import BaseModel, Field, SecretStr, field_serializer + +logger = logging.getLogger(__name__) + +_COLLECTOR_PATH = "/tracer/v1/traces" +_STARTED_KEY = "_futureagi_turn_started_ns" +_STREAM_FAILURE_KEY = "_futureagi_stream_failure" +_TEMPORARY_PREFIXES = ("temporary:", "local:") +_DEFAULT_EMAIL_HMAC_KEY = b"future-agi-openwebui-email-hash-v1" + +# Open WebUI enables native OTel telemetry in the same process. Keep this +# filter's queue independent from that process-wide configuration: the +# explicit values below prevent malformed or pathological OTEL_BSP_* values +# from changing provider creation or making the filter fail closed. +_BSP_MAX_QUEUE_SIZE = 2048 +_BSP_SCHEDULE_DELAY_MILLIS = 5000 +_BSP_MAX_EXPORT_BATCH_SIZE = 512 +_BSP_EXPORT_TIMEOUT_MILLIS = 30000 +_EXPORT_TIMEOUT_SECONDS = _BSP_EXPORT_TIMEOUT_MILLIS / 1000 +_OTLP_EXPORTER_ENV_VARS_TO_MASK = ( + "OTEL_EXPORTER_OTLP_TRACES_CLIENT_CERTIFICATE", + "OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE", + "OTEL_EXPORTER_OTLP_TRACES_CLIENT_KEY", + "OTEL_EXPORTER_OTLP_CLIENT_KEY", + "OTEL_EXPORTER_OTLP_TRACES_TIMEOUT", + "OTEL_EXPORTER_OTLP_TIMEOUT", + "OTEL_EXPORTER_OTLP_TRACES_COMPRESSION", + "OTEL_EXPORTER_OTLP_COMPRESSION", +) +_OTLP_EXPORTER_ENV_LOCK = threading.Lock() + + +@contextmanager +def _without_native_otlp_exporter_env() -> Iterator[None]: + with _OTLP_EXPORTER_ENV_LOCK: + original = { + key: os.environ.pop(key) + for key in _OTLP_EXPORTER_ENV_VARS_TO_MASK + if key in os.environ + } + try: + yield + finally: + os.environ.update(original) + + +def collector_endpoint(endpoint: str) -> str: + trimmed = endpoint.rstrip("/") + if trimmed.endswith(_COLLECTOR_PATH): + return trimmed + return f"{trimmed}{_COLLECTOR_PATH}" + + +def email_hash(email: str, salt: str = "") -> str: + """Return a stable pseudonymous HMAC of a normalised email address.""" + normalised = email.strip().lower().encode("utf-8") + key = salt.encode("utf-8") if salt else _DEFAULT_EMAIL_HMAC_KEY + return hmac.new(key, normalised, hashlib.sha256).hexdigest() + + +def _secret(description: str) -> Any: + return Field( + default=SecretStr(""), + description=description, + json_schema_extra={"input": {"type": "password"}}, + ) + + +def _text(content: Any) -> str: + if isinstance(content, str): + return content + if isinstance(content, list): + return "".join( + part.get("text", "") + for part in content + if isinstance(part, dict) and part.get("type") == "text" + ) + return "" + + +def _usage(message: dict[str, Any]) -> dict[str, int]: + usage = message.get("usage") or (message.get("info") or {}).get("usage") or {} + if not isinstance(usage, dict): + return {} + output = message.get("output") or [] + tool_loop = isinstance(output, list) and any( + isinstance(item, dict) + and ( + item.get("type") in {"function_call", "function_call_output"} + or "function_call" in item + ) + for item in output + ) + if tool_loop: + prompt = usage.get("input_tokens") + completion = usage.get("output_tokens") + else: + prompt = usage.get("prompt_tokens", usage.get("input_tokens")) + completion = usage.get("completion_tokens", usage.get("output_tokens")) + result: dict[str, int] = {} + if isinstance(prompt, int): + result["prompt"] = prompt + if isinstance(completion, int): + result["completion"] = completion + if "prompt" in result and "completion" in result: + result["total"] = result["prompt"] + result["completion"] + return result + + +def _truncate(value: str, max_chars: int) -> str: + return value if max_chars <= 0 else value[:max_chars] + + +def turn_attributes( + body: dict[str, Any], user: dict[str, Any] | None, valves: "Filter.Valves" +) -> tuple[dict[str, Any], dict[str, Any]]: + """Build root and child attributes from Open WebUI's outlet payload.""" + user = user or {} + messages = body.get("messages") or [] + reply = ( + messages[-1] + if messages + and isinstance(messages[-1], dict) + and messages[-1].get("role") == "assistant" + else {} + ) + prompt_messages = messages[:-1] if reply else messages + chat_id = body.get("chat_id") or "" + redact = valves.redact or chat_id.startswith(_TEMPORARY_PREFIXES) + + common: dict[str, Any] = {} + if user.get("id"): + common["user.id"] = user["id"] + if chat_id: + common["session.id"] = chat_id + common["gen_ai.conversation.id"] = chat_id + + root = dict(common, **{"fi.span.kind": "CHAIN"}) + if valves.include_email_hash and user.get("email"): + root["user.email_hash"] = email_hash( + user["email"], valves.email_hash_salt.get_secret_value() + ) + + llm = dict(common, **{"fi.span.kind": "LLM", "gen_ai.operation.name": "chat"}) + model = body.get("model") + if model: + llm["gen_ai.request.model"] = model + llm["llm.model_name"] = model + usage = _usage(reply) + if not usage: + llm["gen_ai.usage.unavailable"] = True + if "prompt" in usage: + llm["gen_ai.usage.input_tokens"] = usage["prompt"] + llm["llm.token_count.prompt"] = usage["prompt"] + if "completion" in usage: + llm["gen_ai.usage.output_tokens"] = usage["completion"] + llm["llm.token_count.completion"] = usage["completion"] + if "total" in usage: + llm["llm.token_count.total"] = usage["total"] + + if not redact: + last_user = next( + ( + m + for m in reversed(prompt_messages) + if isinstance(m, dict) and m.get("role") == "user" + ), + None, + ) + if last_user is not None: + root["input.value"] = _truncate( + _text(last_user.get("content")), valves.max_chars + ) + prompt_json = json.dumps( + [ + {"role": m.get("role"), "content": _text(m.get("content"))} + for m in prompt_messages + if isinstance(m, dict) + ], + ensure_ascii=False, + ) + llm["input.value"] = _truncate(prompt_json, valves.max_chars) + llm["input.mime_type"] = "application/json" + if reply: + output = _truncate(_text(reply.get("content")), valves.max_chars) + root["output.value"] = llm["output.value"] = output + return root, llm + + +class _ProviderCleanup: + """Own providers until the filter finalizer has shut them down.""" + + def __init__(self) -> None: + self._lock = threading.Lock() + self._providers: list[Any] = [] + self._shutdown_ids: set[int] = set() + + def add(self, provider: Any) -> None: + with self._lock: + self._providers.append(provider) + + def shutdown(self, provider: Any) -> None: + provider_id = id(provider) + with self._lock: + if provider_id in self._shutdown_ids or not any( + candidate is provider for candidate in self._providers + ): + return + self._shutdown_ids.add(provider_id) + try: + provider.shutdown() + except Exception: + with self._lock: + self._shutdown_ids.remove(provider_id) + raise + with self._lock: + self._shutdown_ids.remove(provider_id) + self._providers = [ + candidate for candidate in self._providers if candidate is not provider + ] + + def shutdown_all(self) -> None: + with self._lock: + providers = tuple(self._providers) + for provider in providers: + self.shutdown(provider) + + +def _shutdown_provider_cleanup(cleanup: _ProviderCleanup) -> None: + cleanup.shutdown_all() + + +_FAILURE_STATUSES = frozenset({"error", "failed", "failure"}) + + +def _is_failure_status(value: Any) -> bool: + return isinstance(value, str) and value.strip().lower() in _FAILURE_STATUSES + + +def _decode_stream_event(value: Any) -> Any: + """Decode a host stream line without changing the value returned to Open WebUI.""" + if isinstance(value, (bytes, bytearray)): + try: + value = value.decode("utf-8", "replace") + except UnicodeError: + return value + if not isinstance(value, str): + return value + payload = value.strip() + if payload.startswith("data:"): + payload = payload[5:].strip() + if not payload or payload == "[DONE]": + return value + try: + decoded = json.loads(payload) + except (TypeError, ValueError): + return value + return decoded if isinstance(decoded, dict) else value + + +def _has_provider_failure_signal(value: Any) -> bool: + """Recognise explicit host/provider failure markers without guessing from content.""" + value = _decode_stream_event(value) + if not isinstance(value, dict): + return False + if value.get("type") == "response.failed": + return True + if value.get("failed") is True or value.get("response_failed") is True: + return True + if _is_failure_status(value.get("status")): + return True + if value.get("error"): + return True + + response = value.get("response") + if isinstance(response, dict): + if response.get("type") == "response.failed": + return True + if _is_failure_status(response.get("status")) or response.get("error"): + return True + + for key in ("data", "event", "metadata"): + nested = value.get(key) + if isinstance(nested, str) and nested.strip() == "response.failed": + return True + if _has_provider_failure_signal(nested): + return True + + messages = value.get("messages") + if isinstance(messages, list) and messages: + reply = messages[-1] + if isinstance(reply, dict) and reply.get("role") == "assistant": + return _has_provider_failure_signal(reply) + return False + + +def _is_empty_failed_reply(body: dict[str, Any]) -> bool: + messages = body.get("messages") or [] + if not messages or not isinstance(messages[-1], dict): + return True + reply = messages[-1] + return ( + not _text(reply.get("content")) + and not _usage(reply) + and not reply.get("output") + ) + + +def _remember_stream_failure(event: Any, metadata: dict[str, Any] | None) -> None: + """Persist a failure marker on the request object shared with outlet.""" + if isinstance(metadata, dict) and _has_provider_failure_signal(event): + metadata[_STREAM_FAILURE_KEY] = True + + +class Filter: + class Valves(BaseModel): + api_key: SecretStr = _secret("Future AGI API key, sent as X-Api-Key.") + secret_key: SecretStr = _secret("Future AGI secret key, sent as X-Secret-Key.") + endpoint: str = Field( + default="https://api.futureagi.com", + description="Future AGI origin; /tracer/v1/traces is added.", + ) + project: str = Field( + default="open-webui", description="Future AGI project name." + ) + redact: bool = Field( + default=True, + description="Leave prompts and replies out. Set false to see them in Observe.", + ) + include_email_hash: bool = Field( + default=False, description="Add a pseudonymous HMAC of the user's email." + ) + email_hash_salt: SecretStr = _secret( + "Optional HMAC salt for the pseudonymous email hash." + ) + max_chars: int = Field( + default=32000, + ge=0, + description="Maximum input or output characters per span; 0 means unlimited.", + ) + priority: int = Field( + default=0, description="Open WebUI filter priority; lower runs first." + ) + + @field_serializer("api_key", "secret_key", "email_hash_salt") + def _stored_value(self, value: SecretStr) -> str: + return value.get_secret_value() + + def __init__(self) -> None: + self.valves = self.Valves() + self._lock = threading.Lock() + self._provider: Any = None + self._provider_key: str | None = None + self._provider_cleanup = _ProviderCleanup() + self._provider_finalizer = weakref.finalize( + self, _shutdown_provider_cleanup, self._provider_cleanup + ) + + def inlet( + self, + body: dict[str, Any], + __user__: dict[str, Any] | None = None, + __metadata__: dict[str, Any] | None = None, + ) -> dict[str, Any]: + try: + self._start_turn( + __metadata__ if __metadata__ is not None else body.get("metadata") + ) + except Exception as error: # noqa: BLE001 + _log_hook_error("inlet", error) + return body + + def stream( + self, + event: Any, + __metadata__: dict[str, Any] | None = None, + ) -> Any: + """Remember provider errors from decoded events or raw host stream lines. + + Open WebUI normally passes a decoded dict here. A small host adapter may + also forward a non-SSE JSON line (or bytes) from its plain-JSON error + branch; those values are inspected without changing the value returned + to Open WebUI. + """ + try: + _remember_stream_failure(event, __metadata__) + except Exception as error: # noqa: BLE001 + _log_hook_error("stream", error) + return event + + def outlet( + self, + body: dict[str, Any], + __user__: dict[str, Any] | None = None, + __metadata__: dict[str, Any] | None = None, + ) -> dict[str, Any]: + try: + self._record_turn(body, __user__, __metadata__ or {}) + except Exception as error: # noqa: BLE001 + _log_hook_error("outlet", error) + return body + + def flush(self, timeout_millis: int = 10000) -> None: + if self._provider is not None: + self._provider.force_flush(timeout_millis) + + def _start_turn(self, metadata: dict[str, Any] | None) -> None: + if isinstance(metadata, dict): + # Metadata is owned by Open WebUI and can be reused by a retry. Clear + # any marker left by an earlier turn before starting this one. + metadata.pop(_STREAM_FAILURE_KEY, None) + metadata[_STARTED_KEY] = time.time_ns() + + def _record_turn( + self, + body: dict[str, Any], + user: dict[str, Any] | None, + metadata: dict[str, Any], + ) -> None: + started = metadata.pop(_STARTED_KEY, None) + stream_failed = metadata.pop(_STREAM_FAILURE_KEY, False) + if started is None: + return + valves = self.valves + if ( + not valves.api_key.get_secret_value() + or not valves.secret_key.get_secret_value() + ): + return + ended = time.time_ns() + root_attributes, llm_attributes = turn_attributes(body, user, valves) + provider = self._provider_for(valves) + tracer = provider.get_tracer("openwebui-futureagi") + root = tracer.start_span( + "openwebui.chat", + context=context.Context(), + start_time=started, + attributes=root_attributes, + ) + llm = tracer.start_span( + "openwebui.llm", + context=trace.set_span_in_context(root), + start_time=started, + attributes=llm_attributes, + ) + if ( + stream_failed + or _has_provider_failure_signal(body) + or _has_provider_failure_signal(metadata) + or _is_empty_failed_reply(body) + ): + status = Status(StatusCode.ERROR, "provider returned a failed response") + llm.set_status(status) + root.set_status(status) + llm.end(end_time=ended) + root.end(end_time=ended) + + def _provider_for(self, valves: "Filter.Valves") -> Any: + key_material = "\0".join( + ( + collector_endpoint(valves.endpoint), + valves.project, + valves.api_key.get_secret_value(), + valves.secret_key.get_secret_value(), + ) + ) + key = hashlib.sha256(key_material.encode("utf-8")).hexdigest() + with self._lock: + if self._provider is not None and self._provider_key == key: + return self._provider + exporter = self._make_exporter(valves) + provider = TracerProvider( + sampler=ALWAYS_ON, + resource=Resource( + {"project_name": valves.project, "project_type": "observe"} + ), + shutdown_on_exit=False, + ) + provider.add_span_processor( + BatchSpanProcessor( + exporter, + max_queue_size=_BSP_MAX_QUEUE_SIZE, + schedule_delay_millis=_BSP_SCHEDULE_DELAY_MILLIS, + max_export_batch_size=_BSP_MAX_EXPORT_BATCH_SIZE, + export_timeout_millis=_BSP_EXPORT_TIMEOUT_MILLIS, + ) + ) + self._provider_cleanup.add(provider) + old = self._provider + self._provider, self._provider_key = provider, key + if old is not None: + threading.Thread( + target=self._provider_cleanup.shutdown, + args=(old,), + daemon=True, + ).start() + return provider + + def _make_exporter(self, valves: "Filter.Valves") -> Any: + # Open WebUI's native OTLP exporter shares this process environment. The + # OTLP exporter reads client TLS, timeout, and compression settings from + # that environment even when this filter supplies other constructor args. + with _without_native_otlp_exporter_env(): + return OTLPSpanExporter( + endpoint=collector_endpoint(valves.endpoint), + certificate_file=certifi.where(), + client_certificate_file=None, + client_key_file=None, + timeout=_EXPORT_TIMEOUT_SECONDS, + compression=Compression.NoCompression, + headers={ + "X-Api-Key": valves.api_key.get_secret_value(), + "X-Secret-Key": valves.secret_key.get_secret_value(), + }, + ) + + +def _log_hook_error(hook: str, error: Exception) -> None: + logger.warning("Future AGI %s failed: %s", hook, type(error).__name__) + logger.debug( + "Future AGI %s failure details", + hook, + exc_info=(type(error), error, error.__traceback__), + ) diff --git a/integrations/openwebui/tests/test_filter.py b/integrations/openwebui/tests/test_filter.py new file mode 100644 index 000000000..49e9007ca --- /dev/null +++ b/integrations/openwebui/tests/test_filter.py @@ -0,0 +1,790 @@ +"""Contract tests for the Future AGI Open WebUI filter. + +The hook arguments mirror what Open WebUI 0.11.4 passes a Function filter +(``open_webui/utils/filter.py``, ``utils/middleware.py`` ``outlet_filter_handler``): +the instance is cached between calls, its ``valves`` are set from the +database before every hook, ``inlet``/``outlet`` get ``body``, ``__user__`` +and ``__metadata__``, and the outlet body carries the stored messages with +the reply's ``usage`` on the last one. The stream contract accepts decoded +SSE events from stock Open WebUI; it also accepts raw lines for an explicit +host adapter, which stock Open WebUI 0.11.4 does not provide. Spans go to an +in-memory exporter or to a loopback OTLP/HTTP server in this file; nothing +leaves the machine. +""" + +import gc +import importlib.util +import json +import os +import re +import threading +import time +import weakref +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path + +import pytest +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SpanExporter, SpanExportResult +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +from opentelemetry.trace import StatusCode + +FILTER_PATH = Path(__file__).resolve().parents[1] / "futureagi_filter.py" +README = Path(__file__).resolve().parents[1] / "README.md" + +_SPEC = importlib.util.spec_from_file_location("futureagi_filter", FILTER_PATH) +assert _SPEC and _SPEC.loader +_FILTER = importlib.util.module_from_spec(_SPEC) +_SPEC.loader.exec_module(_FILTER) + +CHAT_ID = "2f1c9e4e-7d4b-4a57-9a55-1d1f1a2b3c4d" +USER_MESSAGE_ID = "8a6d2a3e-1111-4c2b-9d11-aaaaaaaaaaaa" +ASSISTANT_MESSAGE_ID = "8a6d2a3e-2222-4c2b-9d11-bbbbbbbbbbbb" +USER = {"id": "u-123", "email": "ada@example.com", "name": "Ada", "role": "user"} +QUESTION = "What is the capital of France? QMARK-owui" +ANSWER = "Paris. AMARK-owui" + + +def metadata(): + return { + "user_id": USER["id"], + "chat_id": CHAT_ID, + "user_message_id": USER_MESSAGE_ID, + "assistant_message_id": None, + "session_id": "socket-session-1", + "user_message": {"id": USER_MESSAGE_ID, "role": "user", "content": QUESTION}, + } + + +def inlet_body(): + return { + "stream": True, + "model": "acme-chat-model", + "messages": [{"role": "user", "content": QUESTION}], + "metadata": metadata(), + } + + +def outlet_body(usage=True): + reply = { + "id": ASSISTANT_MESSAGE_ID, + "role": "assistant", + "content": ANSWER, + "info": None, + "timestamp": 1791777326, + } + if usage: + reply["usage"] = { + "prompt_tokens": 12, + "completion_tokens": 3, + "total_tokens": 15, + "input_tokens": 12, + "output_tokens": 3, + } + return { + "model": "acme-chat-model", + "messages": [ + { + "id": USER_MESSAGE_ID, + "role": "user", + "content": QUESTION, + "info": None, + "timestamp": 1791777326, + }, + reply, + ], + "filter_ids": [], + "chat_id": CHAT_ID, + "session_id": "socket-session-1", + "id": ASSISTANT_MESSAGE_ID, + } + + +def make_filter(exporter=None, **valves): + """A filter as Open WebUI builds it: Filter(), then valves from the database.""" + filt = _FILTER.Filter() + values = { + "api_key": "fi-key-placeholder", + "secret_key": "fi-secret-placeholder", + "project": "owui-test", + } + values.update(valves) + filt.valves = filt.Valves(**values) + if exporter is not None: + filt._make_exporter = lambda valves: exporter + return filt + + +def run_turn(filt, body=None, user=USER): + # Open WebUI hands inlet and outlet of one request the same metadata dict. + meta = metadata() + filt.inlet(inlet_body(), __user__=user, __metadata__=meta) + time.sleep(0.01) + result = filt.outlet(body or outlet_body(), __user__=user, __metadata__=meta) + filt.flush() + return result + + +def spans_by_name(exporter): + return {span.name: span for span in exporter.get_finished_spans()} + + +# -------------------------------------------------------------------------- +# What Open WebUI needs to load and configure the function +# -------------------------------------------------------------------------- + + +try: + from open_webui.utils.plugin import extract_frontmatter as _REAL_EXTRACT_FRONTMATTER +except Exception: # noqa: BLE001 - Open WebUI is optional for the unit-test environment. + _REAL_EXTRACT_FRONTMATTER = None + + +def _pinned_extract_frontmatter(content): + """Open WebUI 0.11.4 parser, or its pinned rules when Open WebUI is absent.""" + if _REAL_EXTRACT_FRONTMATTER is not None: + return _REAL_EXTRACT_FRONTMATTER(content) + lines = content.splitlines() + if not lines or lines[0].strip() != '"""': + return {} + out = {} + for line in lines[1:]: + if '"""' in line: + break + match = re.match(r"^\s*([a-z_]+):\s*(.*)\s*$", line, re.IGNORECASE) + if match: + out[match.group(1).strip()] = match.group(2).strip() + return out + + +def test_frontmatter_is_parsed_by_open_webui(): + frontmatter = _pinned_extract_frontmatter(FILTER_PATH.read_text(encoding="utf-8")) + assert frontmatter["title"] == "Future AGI" + requirements = [r.strip() for r in frontmatter["requirements"].split(",")] + # Open WebUI 0.11.4 ships the OTel SDK but not the OTLP/HTTP exporter. + assert "opentelemetry-exporter-otlp-proto-http" in requirements + + +def test_valves_are_a_filter_attribute_with_defaults(): + # Open WebUI applies valves only when the Filter instance has both + # ``valves`` and ``Valves`` (utils/filter.py apply_filter_valves), and + # builds ``Valves(**{})`` before an admin saved any. + filt = _FILTER.Filter() + assert hasattr(filt, "Valves") and hasattr(filt, "valves") + defaults = filt.Valves() + assert defaults.endpoint == "https://api.futureagi.com" + # Content off by default: Open WebUI serves many users. + assert defaults.redact is True + assert defaults.include_email_hash is False + + +def test_secret_valves_are_masked_but_stored_by_value(): + valves = _FILTER.Filter.Valves( + api_key="fi-key-placeholder", secret_key="fi-secret-placeholder", project="p" + ) + assert "fi-key-placeholder" not in repr(valves) + # Open WebUI stores valves with model_dump(exclude_unset=True) as JSON. + stored = json.loads(json.dumps(valves.model_dump(exclude_unset=True))) + assert stored["api_key"] == "fi-key-placeholder" + assert stored["secret_key"] == "fi-secret-placeholder" + reloaded = _FILTER.Filter.Valves(**stored) + assert reloaded.api_key.get_secret_value() == "fi-key-placeholder" + + +def test_secret_valves_render_as_password_inputs(): + schema = _FILTER.Filter.Valves.model_json_schema() + for key in ("api_key", "secret_key"): + assert schema["properties"][key]["type"] == "string" + assert schema["properties"][key]["input"] == {"type": "password"} + + +# -------------------------------------------------------------------------- +# One chat turn +# -------------------------------------------------------------------------- + + +def test_turn_is_a_root_span_with_an_llm_child(): + exporter = InMemorySpanExporter() + run_turn(make_filter(exporter)) + spans = spans_by_name(exporter) + assert set(spans) == {"openwebui.chat", "openwebui.llm"} + root, llm = spans["openwebui.chat"], spans["openwebui.llm"] + assert root.parent is None + assert llm.parent.span_id == root.context.span_id + assert llm.context.trace_id == root.context.trace_id + assert root.attributes["fi.span.kind"] == "CHAIN" + assert llm.attributes["fi.span.kind"] == "LLM" + assert root.status.status_code != StatusCode.ERROR + + +def test_root_ignores_an_ambient_open_webui_server_span(): + host = TracerProvider() + exporter = InMemorySpanExporter() + filt = make_filter(exporter, redact=False) + with host.get_tracer("openwebui-native").start_as_current_span( + "POST /api/chat/completions" + ): + run_turn(filt) + root = spans_by_name(exporter)["openwebui.chat"] + assert root.parent is None + + +def test_tool_loop_uses_cumulative_input_output_tokens(): + body = outlet_body() + body["messages"][-1]["output"] = [{"type": "function_call", "name": "lookup"}] + body["messages"][-1]["usage"] = { + "prompt_tokens": 1200, + "completion_tokens": 70, + "input_tokens": 2460, + "output_tokens": 160, + "total_tokens": 2620, + } + exporter = InMemorySpanExporter() + run_turn(make_filter(exporter, redact=False), body=body) + llm = spans_by_name(exporter)["openwebui.llm"].attributes + assert llm["gen_ai.usage.input_tokens"] == 2460 + assert llm["gen_ai.usage.output_tokens"] == 160 + assert llm["llm.token_count.total"] == 2620 + + +def test_empty_reply_without_usage_is_error(): + body = outlet_body(usage=False) + body["messages"][-1]["content"] = "" + exporter = InMemorySpanExporter() + run_turn(make_filter(exporter), body=body) + for span in exporter.get_finished_spans(): + assert span.status.status_code == StatusCode.ERROR + + +def test_decoded_sse_stream_error_marks_outlet_as_error_and_keeps_content(): + # This is the observable stock path: middleware strips ``data:`` and + # invokes process_filter_functions before emitting the decoded event. + body = outlet_body(usage=False) + exporter = InMemorySpanExporter() + filt = make_filter(exporter, redact=False) + meta = metadata() + filt.inlet(inlet_body(), __user__=USER, __metadata__=meta) + error_event = {"error": {"message": "provider disconnected"}} + assert filt.stream(error_event, __metadata__=meta) is error_event + filt.outlet(body, __user__=USER, __metadata__=meta) + filt.flush() + + spans = spans_by_name(exporter) + assert spans["openwebui.chat"].status.status_code == StatusCode.ERROR + assert spans["openwebui.llm"].status.status_code == StatusCode.ERROR + assert spans["openwebui.chat"].attributes["output.value"] == ANSWER + assert spans["openwebui.llm"].attributes["output.value"] == ANSWER + assert _FILTER._STREAM_FAILURE_KEY not in meta + + +def test_successful_no_usage_stream_does_not_mark_outlet_as_error(): + body = outlet_body(usage=False) + exporter = InMemorySpanExporter() + filt = make_filter(exporter) + meta = metadata() + filt.inlet(inlet_body(), __user__=USER, __metadata__=meta) + chunk = {"choices": [{"delta": {"content": "partial"}}]} + assert filt.stream(chunk, __metadata__=meta) is chunk + filt.outlet(body, __user__=USER, __metadata__=meta) + filt.flush() + + spans = spans_by_name(exporter) + assert spans["openwebui.chat"].status.status_code != StatusCode.ERROR + assert spans["openwebui.llm"].status.status_code != StatusCode.ERROR + + +def test_temporary_chat_forces_redaction(): + body = outlet_body() + body["chat_id"] = "temporary:abc" + exporter = InMemorySpanExporter() + run_turn(make_filter(exporter, redact=False), body=body) + for span in exporter.get_finished_spans(): + assert "input.value" not in span.attributes + assert "output.value" not in span.attributes + + +def test_provider_ignores_otel_sampler_and_resource_env(monkeypatch): + monkeypatch.setenv("OTEL_TRACES_SAMPLER", "always_off") + monkeypatch.setenv("OTEL_SDK_DISABLED", "false") + monkeypatch.setenv("OTEL_RESOURCE_ATTRIBUTES", "internal.host=secret-host") + monkeypatch.setenv("OTEL_SERVICE_NAME", "native-openwebui") + exporter = InMemorySpanExporter() + filt = make_filter(exporter) + run_turn(filt) + assert len(exporter.get_finished_spans()) == 2 + resource = exporter.get_finished_spans()[0].resource.attributes + assert dict(resource) == {"project_name": "owui-test", "project_type": "observe"} + + +def test_provider_ignores_hostile_native_bsp_env(monkeypatch): + # Open WebUI's native telemetry shares the process environment. Invalid + # values must not prevent this standalone filter from creating its provider. + for key in ( + "OTEL_BSP_MAX_QUEUE_SIZE", + "OTEL_BSP_SCHEDULE_DELAY", + "OTEL_BSP_MAX_EXPORT_BATCH_SIZE", + "OTEL_BSP_EXPORT_TIMEOUT", + ): + monkeypatch.setenv(key, "not-an-integer") + exporter = InMemorySpanExporter() + filt = make_filter(exporter) + run_turn(filt) + assert len(exporter.get_finished_spans()) == 2 + + +def test_exporter_has_explicit_system_ca(monkeypatch): + captured = {} + + class FakeExporter: + def __init__(self, **kwargs): + captured.update(kwargs) + + monkeypatch.setattr(_FILTER, "OTLPSpanExporter", FakeExporter) + filt = make_filter() + filt._make_exporter(filt.valves) + assert captured["certificate_file"] == _FILTER.certifi.where() + assert captured["client_certificate_file"] is None + assert captured["client_key_file"] is None + assert captured["timeout"] == _FILTER._EXPORT_TIMEOUT_SECONDS + assert captured["compression"] is _FILTER.Compression.NoCompression + + +def test_exporter_masks_native_tls_and_transport_env_during_construction(monkeypatch): + hostile = { + "OTEL_EXPORTER_OTLP_TRACES_CLIENT_CERTIFICATE": "native-cert.pem", + "OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE": "native-cert.pem", + "OTEL_EXPORTER_OTLP_TRACES_CLIENT_KEY": "native-key.pem", + "OTEL_EXPORTER_OTLP_CLIENT_KEY": "native-key.pem", + "OTEL_EXPORTER_OTLP_TRACES_TIMEOUT": "1", + "OTEL_EXPORTER_OTLP_TIMEOUT": "1", + "OTEL_EXPORTER_OTLP_TRACES_COMPRESSION": "gzip", + "OTEL_EXPORTER_OTLP_COMPRESSION": "gzip", + } + for key, value in hostile.items(): + monkeypatch.setenv(key, value) + captured = {} + + class FakeExporter: + def __init__(self, **kwargs): + captured["kwargs"] = kwargs + captured["env"] = {key: os.environ.get(key) for key in hostile} + + monkeypatch.setattr(_FILTER, "OTLPSpanExporter", FakeExporter) + filt = make_filter() + filt._make_exporter(filt.valves) + + assert all(value is None for value in captured["env"].values()) + assert {key: os.environ[key] for key in hostile} == hostile + assert captured["kwargs"]["client_certificate_file"] is None + assert captured["kwargs"]["client_key_file"] is None + assert captured["kwargs"]["timeout"] == _FILTER._EXPORT_TIMEOUT_SECONDS + assert captured["kwargs"]["compression"] is _FILTER.Compression.NoCompression + + +def test_exporter_request_session_has_no_native_client_tls(monkeypatch): + hostile = { + "OTEL_EXPORTER_OTLP_TRACES_CLIENT_CERTIFICATE": "native-cert.pem", + "OTEL_EXPORTER_OTLP_CLIENT_CERTIFICATE": "native-cert.pem", + "OTEL_EXPORTER_OTLP_TRACES_CLIENT_KEY": "native-key.pem", + "OTEL_EXPORTER_OTLP_CLIENT_KEY": "native-key.pem", + "OTEL_EXPORTER_OTLP_TRACES_TIMEOUT": "1", + "OTEL_EXPORTER_OTLP_TIMEOUT": "1", + "OTEL_EXPORTER_OTLP_TRACES_COMPRESSION": "gzip", + "OTEL_EXPORTER_OTLP_COMPRESSION": "gzip", + } + for key, value in hostile.items(): + monkeypatch.setenv(key, value) + + filt = make_filter() + exporter = filt._make_exporter(filt.valves) + if hasattr(exporter, "_session"): + session = exporter._session + timeout = exporter._timeout + client_cert = exporter._client_cert + else: + session = exporter._client._transport._session + timeout = exporter._client._timeout + client_cert = session.cert + + assert client_cert is None + assert session.cert is None + assert timeout == _FILTER._EXPORT_TIMEOUT_SECONDS + assert exporter._compression.value == _FILTER.Compression.NoCompression.value + + +def test_provider_has_a_shutdown_finalizer(): + filt = make_filter() + provider = filt._provider_for(filt.valves) + assert isinstance(filt._provider_finalizer, weakref.finalize) + assert filt._provider_finalizer.alive + provider.shutdown() + + +def test_filter_finalizer_shuts_down_providers_after_gc(monkeypatch): + providers = [] + + class FakeProvider: + def __init__(self, **kwargs): + assert kwargs["shutdown_on_exit"] is False + self.shutdown_calls = 0 + providers.append(self) + + def add_span_processor(self, processor): + return None + + def shutdown(self): + self.shutdown_calls += 1 + + class FakeBatchSpanProcessor: + def __init__(self, exporter, **kwargs): + return None + + monkeypatch.setattr(_FILTER, "TracerProvider", FakeProvider) + monkeypatch.setattr(_FILTER, "BatchSpanProcessor", FakeBatchSpanProcessor) + + filt = make_filter() + first = filt._provider_for(filt.valves) + changed = filt.Valves( + api_key="different-key", + secret_key="fi-secret-placeholder", + project="owui-test", + ) + second = filt._provider_for(changed) + for _ in range(100): + if first.shutdown_calls: + break + time.sleep(0.001) + assert first.shutdown_calls == 1 + assert providers == [first, second] + + finalizer = filt._provider_finalizer + filter_ref = weakref.ref(filt) + del filt + gc.collect() + + assert filter_ref() is None + assert not finalizer.alive + assert second.shutdown_calls == 1 + + +def test_priority_and_max_chars_are_configurable(): + valves = _FILTER.Filter.Valves(priority=9, max_chars=17) + assert valves.priority == 9 + assert valves.max_chars == 17 + exporter = InMemorySpanExporter() + run_turn(make_filter(exporter, redact=False, max_chars=17)) + root = spans_by_name(exporter)["openwebui.chat"] + llm = spans_by_name(exporter)["openwebui.llm"] + assert len(root.attributes["input.value"]) <= 17 + assert len(llm.attributes["input.value"]) <= 17 + + +def test_sdk_disabled_documents_global_disable(monkeypatch): + monkeypatch.setenv("OTEL_SDK_DISABLED", "true") + exporter = InMemorySpanExporter() + run_turn(make_filter(exporter)) + assert exporter.get_finished_spans() == () + + +def test_llm_span_has_model_usage_and_content(): + exporter = InMemorySpanExporter() + run_turn(make_filter(exporter, redact=False)) + spans = spans_by_name(exporter) + llm = spans["openwebui.llm"].attributes + assert llm["gen_ai.request.model"] == "acme-chat-model" + assert llm["llm.model_name"] == "acme-chat-model" + assert llm["gen_ai.usage.input_tokens"] == 12 + assert llm["gen_ai.usage.output_tokens"] == 3 + assert llm["llm.token_count.total"] == 15 + assert json.loads(llm["input.value"]) == [{"role": "user", "content": QUESTION}] + assert llm["output.value"] == ANSWER + root = spans["openwebui.chat"].attributes + assert root["input.value"] == QUESTION + assert root["output.value"] == ANSWER + + +def test_user_and_session_come_from_open_webui(): + exporter = InMemorySpanExporter() + run_turn(make_filter(exporter)) + for span in exporter.get_finished_spans(): + assert span.attributes["user.id"] == USER["id"] + assert span.attributes["session.id"] == CHAT_ID + assert span.attributes["gen_ai.conversation.id"] == CHAT_ID + assert "user.email_hash" not in span.attributes + assert USER["email"] not in json.dumps(dict(span.attributes)) + + +def test_email_hash_is_opt_in_and_never_raw(): + exporter = InMemorySpanExporter() + run_turn(make_filter(exporter, include_email_hash=True)) + root = spans_by_name(exporter)["openwebui.chat"] + assert root.attributes["user.email_hash"] == _FILTER.email_hash(USER["email"]) + salted_exporter = InMemorySpanExporter() + salted = make_filter( + salted_exporter, include_email_hash=True, email_hash_salt="salt" + ) + run_turn(salted) + salted_root = spans_by_name(salted_exporter)["openwebui.chat"] + assert salted_root.attributes["user.email_hash"] == _FILTER.email_hash( + USER["email"], "salt" + ) + for span in exporter.get_finished_spans(): + assert USER["email"] not in json.dumps(dict(span.attributes)) + + +@pytest.mark.parametrize("valves", [{}, {"redact": True}], ids=["default", "redact"]) +def test_content_stays_out_of_every_span(valves): + exporter = InMemorySpanExporter() + run_turn(make_filter(exporter, **valves)) + spans = exporter.get_finished_spans() + assert len(spans) == 2 + wire = json.dumps([dict(span.attributes) for span in spans]) + assert "QMARK-owui" not in wire and "AMARK-owui" not in wire + for span in spans: + assert "input.value" not in span.attributes + assert "output.value" not in span.attributes + # Ids, model and usage still arrive. + llm = spans_by_name(exporter)["openwebui.llm"].attributes + assert llm["gen_ai.request.model"] == "acme-chat-model" + assert llm["gen_ai.usage.input_tokens"] == 12 + + +def test_total_tokens_are_prompt_plus_completion(): + # Open WebUI 0.11.4 sums total/input/output tokens over stream chunks; a + # provider that repeats usage in two chunks gave, live: prompt 8838, + # completion 7, total 17690, input 17676, output 14. The provider said 8845. + body = outlet_body() + body["messages"][-1]["usage"] = { + "prompt_tokens": 8838, + "completion_tokens": 7, + "total_tokens": 17690, + "input_tokens": 17676, + "output_tokens": 14, + } + exporter = InMemorySpanExporter() + run_turn(make_filter(exporter), body=body) + llm = spans_by_name(exporter)["openwebui.llm"].attributes + assert llm["gen_ai.usage.input_tokens"] == 8838 + assert llm["gen_ai.usage.output_tokens"] == 7 + assert llm["llm.token_count.total"] == 8845 + + +def test_missing_usage_is_flagged_without_marking_success_as_error(): + exporter = InMemorySpanExporter() + run_turn(make_filter(exporter), body=outlet_body(usage=False)) + spans = spans_by_name(exporter) + llm = spans["openwebui.llm"].attributes + assert llm["gen_ai.usage.unavailable"] is True + assert "gen_ai.usage.input_tokens" not in llm + for span in spans.values(): + assert span.status.status_code != StatusCode.ERROR + + +def test_turn_starts_at_inlet(): + exporter = InMemorySpanExporter() + filt = make_filter(exporter) + meta = metadata() + filt.inlet(inlet_body(), __user__=USER, __metadata__=meta) + time.sleep(0.2) + filt.outlet(outlet_body(), __user__=USER, __metadata__=meta) + filt.flush() + root = spans_by_name(exporter)["openwebui.chat"] + llm = spans_by_name(exporter)["openwebui.llm"] + assert root.end_time - root.start_time >= 150_000_000 + assert (llm.start_time, llm.end_time) == (root.start_time, root.end_time) + # The start time rode on the request's metadata and is gone again. + assert not any(key.startswith("_futureagi") for key in meta) + + +def test_outlet_without_inlet_skips_duplicate_trace(): + exporter = InMemorySpanExporter() + filt = make_filter(exporter) + filt.outlet(outlet_body(), __user__=USER, __metadata__=metadata()) + filt.flush() + assert exporter.get_finished_spans() == () + + +def test_inlets_without_outlets_leave_no_growing_state(): + # Open WebUI runs no outlet when the model call fails. Metadata is owned by + # Open WebUI, so the filter must not retain messages or per-turn objects. + filt = make_filter(InMemorySpanExporter()) + before = { + key: (len(value) if isinstance(value, (dict, list, set)) else value) + for key, value in vars(filt).items() + } + for _ in range(100): + meta = metadata() + filt.inlet(inlet_body(), __user__=USER, __metadata__=meta) + after = { + key: (len(value) if isinstance(value, (dict, list, set)) else value) + for key, value in vars(filt).items() + } + assert after == before + + +def test_no_keys_means_no_export_and_no_error(): + exporter = InMemorySpanExporter() + filt = _FILTER.Filter() + filt._make_exporter = lambda valves: exporter + body = outlet_body() + assert filt.outlet(body, __user__=USER, __metadata__=metadata()) is body + filt.flush() + assert exporter.get_finished_spans() == () + + +# -------------------------------------------------------------------------- +# The chat never waits for, or breaks on, the export +# -------------------------------------------------------------------------- + + +class SlowExporter(SpanExporter): + def __init__(self): + self.started = threading.Event() + + def export(self, spans): + self.started.set() + time.sleep(2) + return SpanExportResult.FAILURE + + def shutdown(self): + return None + + +def test_outlet_does_not_wait_for_the_export(): + exporter = SlowExporter() + filt = make_filter(exporter) + meta = metadata() + filt.inlet(inlet_body(), __user__=USER, __metadata__=meta) + started = time.monotonic() + body = outlet_body() + assert filt.outlet(body, __user__=USER, __metadata__=meta) is body + assert time.monotonic() - started < 0.5 + filt._provider.shutdown() + + +def test_hooks_return_the_body_when_tracing_fails(monkeypatch): + def boom(*args, **kwargs): + raise RuntimeError("collector down") + + filt = make_filter(InMemorySpanExporter()) + monkeypatch.setattr(filt, "_record_turn", boom) + body = outlet_body() + assert filt.outlet(body, __user__=USER, __metadata__=metadata()) is body + monkeypatch.setattr(filt, "_start_turn", boom) + first = inlet_body() + assert filt.inlet(first, __user__=USER, __metadata__=metadata()) is first + + +def test_new_valves_with_same_values_reuse_the_provider(): + filt = make_filter() + first = filt._provider_for(filt.valves) + same_values = filt.Valves( + api_key="fi-key-placeholder", + secret_key="fi-secret-placeholder", + project="owui-test", + ) + assert filt._provider_for(same_values) is first + changed = filt.Valves( + api_key="other", secret_key="fi-secret-placeholder", project="owui-test" + ) + assert filt._provider_for(changed) is not first + first.shutdown() + filt._provider.shutdown() + + +# -------------------------------------------------------------------------- +# The wire +# -------------------------------------------------------------------------- + + +def test_collector_path(): + assert ( + _FILTER.collector_endpoint("https://api.futureagi.com") + == "https://api.futureagi.com/tracer/v1/traces" + ) + assert ( + _FILTER.collector_endpoint("https://api.futureagi.com/tracer/v1/traces/") + == "https://api.futureagi.com/tracer/v1/traces" + ) + + +class CaptureServer: + """Loopback OTLP/HTTP endpoint that keeps each request's path, headers and decoded body.""" + + def __init__(self): + from google.protobuf.json_format import MessageToDict + from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ( + ExportTraceServiceRequest, + ) + + self.requests = [] + owner = self + + class Handler(BaseHTTPRequestHandler): + def do_POST(self): + body = self.rfile.read(int(self.headers.get("Content-Length", "0"))) + message = ExportTraceServiceRequest() + message.ParseFromString(body) + headers = {k.lower(): v for k, v in self.headers.items()} + owner.requests.append( + { + "path": self.path, + "headers": headers, + "body": MessageToDict(message), + } + ) + self.send_response(200) + self.send_header("Content-Length", "0") + self.end_headers() + + def log_message(self, *args): + return + + self._server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + self.origin = f"http://127.0.0.1:{self._server.server_port}" + threading.Thread(target=self._server.serve_forever, daemon=True).start() + + def close(self): + self._server.shutdown() + self._server.server_close() + + +def test_turn_reaches_the_collector_path_with_keys_and_project(): + server = CaptureServer() + try: + filt = make_filter(endpoint=server.origin + "/") + run_turn(filt) + finally: + server.close() + assert server.requests + names = [] + for request in server.requests: + assert request["path"] == "/tracer/v1/traces" + assert request["headers"]["x-api-key"] == "fi-key-placeholder" + assert request["headers"]["x-secret-key"] == "fi-secret-placeholder" + for resource_spans in request["body"]["resourceSpans"]: + resource = { + a["key"]: a["value"].get("stringValue") + for a in resource_spans["resource"]["attributes"] + } + assert resource["project_name"] == "owui-test" + assert resource["project_type"] == "observe" + for scope_spans in resource_spans["scopeSpans"]: + names.extend(span["name"] for span in scope_spans["spans"]) + assert "fi-key-placeholder" not in json.dumps(request["body"]) + assert sorted(names) == ["openwebui.chat", "openwebui.llm"] + + +def test_readme_lists_the_steps_open_webui_needs(): + readme = README.read_text(encoding="utf-8") + assert not readme.startswith('"""') + for fact in ( + "Global", + "Valves", + "/tracer/v1/traces", + "X-Api-Key", + "X-Secret-Key", + "0.11.4", + "redact", + ): + assert fact in readme, fact diff --git a/python/tests/harness/__init__.py b/python/tests/harness/__init__.py new file mode 100644 index 000000000..cf79047f1 --- /dev/null +++ b/python/tests/harness/__init__.py @@ -0,0 +1,238 @@ +"""Small, loopback-only utilities for OTLP contract tests.""" + +from __future__ import annotations + +import copy +import difflib +import json +import subprocess +import threading +from http import HTTPStatus +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from typing import Any, Mapping, Optional, Sequence, Union +from urllib.parse import urlsplit +from urllib.request import Request, urlopen + + +class Receiver: + """An in-memory OTLP/HTTP receiver for tests. + + The receiver only listens on loopback and never forwards received spans. + """ + + def __init__(self) -> None: + self._spans: list[dict[str, Any]] = [] + self._requests: list[dict[str, Any]] = [] + self._lock = threading.Lock() + owner = self + + class Handler(BaseHTTPRequestHandler): + def _read_body(self) -> bytes: + # The Node OTLP exporter streams with Transfer-Encoding: chunked + # and sends no Content-Length, so both framings are accepted. + if "chunked" in self.headers.get("Transfer-Encoding", "").lower(): + chunks = [] + while True: + size_line = self.rfile.readline().split(b";", 1)[0].strip() + size = int(size_line, 16) + if size == 0: + while self.rfile.readline() not in (b"\r\n", b"\n", b""): + pass + return b"".join(chunks) + chunks.append(self.rfile.read(size)) + self.rfile.readline() + length = int(self.headers.get("Content-Length", "0")) + return self.rfile.read(length) + + def do_POST(self) -> None: # noqa: N802 - required by BaseHTTPRequestHandler + path = urlsplit(self.path).path + if path not in ("/v1/traces", "/tracer/v1/traces"): + self.send_error(HTTPStatus.NOT_FOUND) + return + + try: + body = self._read_body() + content_type = self.headers.get("Content-Type", "").lower() + + if content_type.startswith("application/json"): + request = json.loads(body.decode("utf-8")) + elif content_type.startswith(("application/x-protobuf", "application/protobuf")): + from google.protobuf.json_format import MessageToDict + from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ( + ExportTraceServiceRequest, + ) + + protobuf_request = ExportTraceServiceRequest() + protobuf_request.ParseFromString(body) + request = MessageToDict(protobuf_request) + else: + self.send_error(HTTPStatus.UNSUPPORTED_MEDIA_TYPE) + return + + spans: list[dict[str, Any]] = [] + resource_attributes: list[dict[str, Any]] = [] + for resource_spans in request.get( + "resourceSpans", request.get("resource_spans", []) + ): + resource_attributes.append( + _flatten_attributes( + resource_spans.get("resource", {}).get("attributes", []) + ) + ) + for scope_spans in resource_spans.get( + "scopeSpans", resource_spans.get("scope_spans", []) + ): + spans.extend(scope_spans.get("spans", [])) + except (ImportError, TypeError, UnicodeDecodeError, ValueError) as error: + self.send_error(HTTPStatus.BAD_REQUEST, str(error)) + return + + with owner._lock: + owner._spans.extend(copy.deepcopy(spans)) + owner._requests.append( + { + "path": path, + "headers": { + key.lower(): value for key, value in self.headers.items() + }, + "resource_attributes": resource_attributes, + } + ) + self.send_response(HTTPStatus.OK) + self.send_header("Content-Length", "0") + self.end_headers() + + def log_message(self, _format: str, *args: Any) -> None: + return + + self._server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + self.origin = "http://127.0.0.1:{0}".format(self._server.server_port) + self.endpoint = "{0}/v1/traces".format(self.origin) + self.collector_endpoint = "{0}/tracer/v1/traces".format(self.origin) + self._thread = threading.Thread(target=self._server.serve_forever, daemon=True) + self._thread.start() + + def __enter__(self) -> "Receiver": + return self + + def __exit__(self, _type: Any, _value: Any, _traceback: Any) -> None: + self._server.shutdown() + self._server.server_close() + self._thread.join() + + def spans(self) -> list[dict[str, Any]]: + """Return a copy of the decoded spans received so far.""" + with self._lock: + return copy.deepcopy(self._spans) + + def requests(self) -> list[dict[str, Any]]: + """Return one record per accepted export: path, lower-cased headers, resource attributes.""" + with self._lock: + return copy.deepcopy(self._requests) + + def clear(self) -> None: + """Remove every decoded span and request record received so far.""" + with self._lock: + self._spans.clear() + self._requests.clear() + + +def _flatten_attributes(attributes: Sequence[Mapping[str, Any]]) -> dict[str, Any]: + """Turn OTLP key/value attributes into a plain dict of scalar values.""" + flat: dict[str, Any] = {} + for attribute in attributes: + value = attribute.get("value", {}) + for kind in ("stringValue", "boolValue", "intValue", "doubleValue"): + if kind in value: + flat[attribute.get("key", "")] = value[kind] + break + else: + flat[attribute.get("key", "")] = value + return flat + + +def run( + argv: Sequence[str], + env: Mapping[str, str], + stdin: Optional[bytes], + timeout: float, +) -> subprocess.CompletedProcess[bytes]: + """Run one process and return its captured output, including after a timeout.""" + process = subprocess.Popen( + argv, + env=dict(env), + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + timed_out = False + try: + stdout, stderr = process.communicate(input=stdin, timeout=timeout) + except subprocess.TimeoutExpired: + timed_out = True + process.kill() + stdout, stderr = process.communicate() + + result = subprocess.CompletedProcess(argv, process.returncode, stdout, stderr) + result.timed_out = timed_out + return result + + +def post_otlp(spans: dict[str, Any], endpoint: str) -> int: + """Post one JSON-encoded OTLP request to a loopback receiver.""" + parsed = urlsplit(endpoint) + if parsed.scheme != "http" or parsed.hostname != "127.0.0.1": + raise ValueError("post_otlp only sends to a loopback HTTP endpoint") + + request = Request( + endpoint, + data=json.dumps(spans).encode("utf-8"), + headers={"Content-Type": "application/json"}, + method="POST", + ) + with urlopen(request, timeout=5) as response: + return response.status + + +def compare(actual: Sequence[dict[str, Any]], golden_path: Union[str, Path]) -> None: + """Fail with a diff unless span names, attributes, and statuses match a golden.""" + with Path(golden_path).open(encoding="utf-8") as golden_file: + expected = json.load(golden_file) + + def canonical(spans: Sequence[dict[str, Any]]) -> list[dict[str, Any]]: + normalized = [] + for span in spans: + attributes = span.get("attributes", []) + if isinstance(attributes, dict): + attributes = [ + {"key": key, "value": value} for key, value in attributes.items() + ] + normalized.append( + { + "name": span.get("name", ""), + "attributes": sorted( + attributes, + key=lambda attribute: json.dumps( + attribute, sort_keys=True, separators=(",", ":") + ), + ), + "status": span.get("status", {}), + } + ) + return normalized + + actual_normalized = canonical(actual) + expected_normalized = canonical(expected) + if actual_normalized != expected_normalized: + diff = "\n".join( + difflib.unified_diff( + json.dumps(expected_normalized, indent=2, sort_keys=True).splitlines(), + json.dumps(actual_normalized, indent=2, sort_keys=True).splitlines(), + fromfile=str(golden_path), + tofile="actual", + lineterm="", + ) + ) + print(diff) + raise AssertionError("spans do not match golden") diff --git a/python/tests/harness/test_harness.py b/python/tests/harness/test_harness.py new file mode 100644 index 000000000..bd61d5adc --- /dev/null +++ b/python/tests/harness/test_harness.py @@ -0,0 +1,189 @@ +"""Contract tests for the shared loopback-only harness.""" + +import json +import sys + +import pytest + +from tests.harness import Receiver, compare, post_otlp, run + + +def test_receiver_stores_posted_span_and_can_clear() -> None: + payload = { + "resourceSpans": [ + { + "scopeSpans": [ + { + "spans": [ + { + "traceId": "trace-id", + "spanId": "span-id", + "name": "harness.receiver", + "attributes": [ + {"key": "component", "value": {"stringValue": "test"}} + ], + "status": {"code": 1}, + } + ] + } + ] + } + ] + } + + with Receiver() as receiver: + assert post_otlp(payload, receiver.endpoint) == 200 + assert receiver.spans() == payload["resourceSpans"][0]["scopeSpans"][0]["spans"] + receiver.clear() + assert receiver.spans() == [] + + +def test_receiver_accepts_the_collector_path() -> None: + payload = { + "resourceSpans": [ + {"scopeSpans": [{"spans": [{"name": "collector.path"}]}]} + ] + } + + with Receiver() as receiver: + assert post_otlp(payload, receiver.collector_endpoint) == 200 + assert receiver.spans() == [{"name": "collector.path"}] + + +def _post_chunked(url: str, body: bytes, headers: dict) -> int: + """Send a body with Transfer-Encoding: chunked, as the Node OTLP exporter does.""" + import http.client + from urllib.parse import urlsplit + + parts = urlsplit(url) + connection = http.client.HTTPConnection(parts.hostname, parts.port, timeout=5) + try: + chunks = [body[: len(body) // 2], body[len(body) // 2 :]] + connection.request( + "POST", + parts.path, + body=iter(chunks), + headers={**headers, "Transfer-Encoding": "chunked"}, + encode_chunked=True, + ) + return connection.getresponse().status + finally: + connection.close() + + +def test_receiver_decodes_a_chunked_body() -> None: + payload = { + "resourceSpans": [ + {"scopeSpans": [{"spans": [{"name": "chunked.body"}]}]} + ] + } + + with Receiver() as receiver: + status = _post_chunked( + receiver.collector_endpoint, + json.dumps(payload).encode("utf-8"), + {"Content-Type": "application/json"}, + ) + assert status == 200 + assert receiver.spans() == [{"name": "chunked.body"}] + + +def test_receiver_records_path_headers_and_resource_attributes() -> None: + payload = { + "resourceSpans": [ + { + "resource": { + "attributes": [ + {"key": "project_name", "value": {"stringValue": "harness"}}, + {"key": "project_type", "value": {"stringValue": "observe"}}, + ] + }, + "scopeSpans": [{"spans": [{"name": "with.resource"}]}], + } + ] + } + + with Receiver() as receiver: + status = _post_chunked( + receiver.collector_endpoint, + json.dumps(payload).encode("utf-8"), + { + "Content-Type": "application/json", + "X-Api-Key": "test-key", + "X-Secret-Key": "test-secret", + }, + ) + assert status == 200 + [request] = receiver.requests() + assert request["path"] == "/tracer/v1/traces" + assert request["headers"]["x-api-key"] == "test-key" + assert request["headers"]["x-secret-key"] == "test-secret" + assert "authorization" not in request["headers"] + assert request["resource_attributes"] == [ + {"project_name": "harness", "project_type": "observe"} + ] + receiver.clear() + assert receiver.requests() == [] + + +def test_run_captures_output_and_kills_timed_out_process() -> None: + completed = run( + [ + sys.executable, + "-c", + "import sys; print('stdout'); sys.stderr.write('stderr\\n')", + ], + env={}, + stdin=b"", + timeout=1, + ) + assert completed.returncode == 0 + assert completed.stdout == b"stdout\n" + assert completed.stderr == b"stderr\n" + assert completed.timed_out is False + + timed_out = run( + [sys.executable, "-c", "import time; time.sleep(30)"], + env={}, + stdin=b"", + timeout=0.1, + ) + assert timed_out.timed_out is True + assert timed_out.returncode != 0 + + +def test_compare_accepts_normalized_match_and_prints_mismatch_diff(tmp_path, capsys) -> None: + golden = [ + { + "traceId": "golden-trace", + "spanId": "golden-span", + "startTimeUnixNano": "1", + "endTimeUnixNano": "2", + "name": "harness.compare", + "attributes": [ + {"key": "alpha", "value": {"stringValue": "a"}}, + {"key": "beta", "value": {"intValue": "2"}}, + ], + "status": {"code": 1}, + } + ] + golden_path = tmp_path / "spans.golden.json" + golden_path.write_text(json.dumps(golden), encoding="utf-8") + + actual = [ + { + "traceId": "other-trace", + "spanId": "other-span", + "startTimeUnixNano": "3", + "endTimeUnixNano": "4", + "name": "harness.compare", + "attributes": list(reversed(golden[0]["attributes"])), + "status": {"code": 1}, + } + ] + compare(actual, golden_path) + + actual[0]["name"] = "harness.mismatch" + with pytest.raises(AssertionError, match="spans do not match golden"): + compare(actual, golden_path) + assert "---" in capsys.readouterr().out