diff --git a/examples/coding-agents/.gitignore b/examples/coding-agents/.gitignore new file mode 100644 index 000000000..7a60b85e1 --- /dev/null +++ b/examples/coding-agents/.gitignore @@ -0,0 +1,2 @@ +__pycache__/ +*.pyc diff --git a/examples/coding-agents/cursor/hooks.json b/examples/coding-agents/cursor/hooks.json new file mode 100644 index 000000000..185af59ba --- /dev/null +++ b/examples/coding-agents/cursor/hooks.json @@ -0,0 +1,65 @@ +{ + "version": 1, + "hooks": { + "sessionStart": [ + { + "command": "(python3 -B .cursor/hooks/fi-hook-to-otlp/emit.py cursor sessionStart 2>/dev/null || true)", + "timeout": 10 + } + ], + "beforeSubmitPrompt": [ + { + "command": "(python3 -B .cursor/hooks/fi-hook-to-otlp/emit.py cursor beforeSubmitPrompt 2>/dev/null || true)", + "timeout": 10 + } + ], + "postToolUse": [ + { + "command": "(python3 -B .cursor/hooks/fi-hook-to-otlp/emit.py cursor postToolUse 2>/dev/null || true)", + "timeout": 10 + } + ], + "postToolUseFailure": [ + { + "command": "(python3 -B .cursor/hooks/fi-hook-to-otlp/emit.py cursor postToolUseFailure 2>/dev/null || true)", + "timeout": 10 + } + ], + "subagentStop": [ + { + "command": "(python3 -B .cursor/hooks/fi-hook-to-otlp/emit.py cursor subagentStop 2>/dev/null || true)", + "timeout": 10 + } + ], + "afterAgentThought": [ + { + "command": "(python3 -B .cursor/hooks/fi-hook-to-otlp/emit.py cursor afterAgentThought 2>/dev/null || true)", + "timeout": 10 + } + ], + "afterAgentResponse": [ + { + "command": "(python3 -B .cursor/hooks/fi-hook-to-otlp/emit.py cursor afterAgentResponse 2>/dev/null || true)", + "timeout": 10 + } + ], + "preCompact": [ + { + "command": "(python3 -B .cursor/hooks/fi-hook-to-otlp/emit.py cursor preCompact 2>/dev/null || true)", + "timeout": 10 + } + ], + "stop": [ + { + "command": "(python3 -B .cursor/hooks/fi-hook-to-otlp/emit.py cursor stop 2>/dev/null || true)", + "timeout": 10 + } + ], + "sessionEnd": [ + { + "command": "(python3 -B .cursor/hooks/fi-hook-to-otlp/emit.py cursor sessionEnd 2>/dev/null || true)", + "timeout": 10 + } + ] + } +} diff --git a/examples/coding-agents/cursor/hooks.user.json b/examples/coding-agents/cursor/hooks.user.json new file mode 100644 index 000000000..cf3e0c798 --- /dev/null +++ b/examples/coding-agents/cursor/hooks.user.json @@ -0,0 +1,65 @@ +{ + "version": 1, + "hooks": { + "sessionStart": [ + { + "command": "(python3 -B ./hooks/fi-hook-to-otlp/emit.py cursor sessionStart 2>/dev/null || true)", + "timeout": 10 + } + ], + "beforeSubmitPrompt": [ + { + "command": "(python3 -B ./hooks/fi-hook-to-otlp/emit.py cursor beforeSubmitPrompt 2>/dev/null || true)", + "timeout": 10 + } + ], + "postToolUse": [ + { + "command": "(python3 -B ./hooks/fi-hook-to-otlp/emit.py cursor postToolUse 2>/dev/null || true)", + "timeout": 10 + } + ], + "postToolUseFailure": [ + { + "command": "(python3 -B ./hooks/fi-hook-to-otlp/emit.py cursor postToolUseFailure 2>/dev/null || true)", + "timeout": 10 + } + ], + "subagentStop": [ + { + "command": "(python3 -B ./hooks/fi-hook-to-otlp/emit.py cursor subagentStop 2>/dev/null || true)", + "timeout": 10 + } + ], + "afterAgentThought": [ + { + "command": "(python3 -B ./hooks/fi-hook-to-otlp/emit.py cursor afterAgentThought 2>/dev/null || true)", + "timeout": 10 + } + ], + "afterAgentResponse": [ + { + "command": "(python3 -B ./hooks/fi-hook-to-otlp/emit.py cursor afterAgentResponse 2>/dev/null || true)", + "timeout": 10 + } + ], + "preCompact": [ + { + "command": "(python3 -B ./hooks/fi-hook-to-otlp/emit.py cursor preCompact 2>/dev/null || true)", + "timeout": 10 + } + ], + "stop": [ + { + "command": "(python3 -B ./hooks/fi-hook-to-otlp/emit.py cursor stop 2>/dev/null || true)", + "timeout": 10 + } + ], + "sessionEnd": [ + { + "command": "(python3 -B ./hooks/fi-hook-to-otlp/emit.py cursor sessionEnd 2>/dev/null || true)", + "timeout": 10 + } + ] + } +} diff --git a/examples/coding-agents/cursor/install.sh b/examples/coding-agents/cursor/install.sh new file mode 100755 index 000000000..3f1bdd274 --- /dev/null +++ b/examples/coding-agents/cursor/install.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +# Install the Future AGI hook emitter for Cursor. +# +# ./install.sh project hooks: /.cursor/hooks.json +# ./install.sh --user user hooks: ~/.cursor/hooks.json +# +# Copies emit.py + the Cursor adapter into .cursor/hooks/fi-hook-to-otlp/ and +# writes hooks.json if there is none. An existing hooks.json is never +# modified: the script prints the entries to merge instead. +set -euo pipefail + +HERE="$(cd "$(dirname "$0")" && pwd)" +EMITTER="$HERE/../hook-to-otlp" + +if [[ "${1:-}" == "--user" ]]; then + CURSOR_DIR="$HOME/.cursor" + TEMPLATE="$HERE/hooks.user.json" +elif [[ -n "${1:-}" ]]; then + CURSOR_DIR="$(cd "$1" && pwd)/.cursor" + TEMPLATE="$HERE/hooks.json" +else + echo "usage: $0 | --user" >&2 + exit 2 +fi + +DEST="$CURSOR_DIR/hooks/fi-hook-to-otlp" +mkdir -p "$DEST/adapters" +cp "$EMITTER/emit.py" "$DEST/emit.py" +cp "$EMITTER/adapters/__init__.py" "$EMITTER/adapters/base.py" "$EMITTER/adapters/cursor.py" "$DEST/adapters/" +echo "emitter: $DEST" + +if [[ -e "$CURSOR_DIR/hooks.json" ]]; then + echo "kept existing $CURSOR_DIR/hooks.json; merge these entries into its \"hooks\" object:" + cat "$TEMPLATE" +else + cp "$TEMPLATE" "$CURSOR_DIR/hooks.json" + echo "hooks: $CURSOR_DIR/hooks.json" +fi + +echo +echo "Credentials: export FI_API_KEY / FI_SECRET_KEY / FI_PROJECT_NAME in the environment" +echo "that starts Cursor, or put them in ~/.config/fi-hook-to-otlp/env (chmod 600)." diff --git a/examples/coding-agents/cursor/run_e2e.sh b/examples/coding-agents/cursor/run_e2e.sh new file mode 100755 index 000000000..8e3d7abc2 --- /dev/null +++ b/examples/coding-agents/cursor/run_e2e.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +# Drive the real emitter with Cursor hook payloads into a Future AGI collector. +# +# FI_API_KEY=... FI_SECRET_KEY=... FI_BASE_URL=http://localhost:14338 \ +# FI_PROJECT_NAME=cursor-e2e ./run_e2e.sh [fixture.jsonl] [replay.py args...] +# +# Default input is the documented-shape session in +# ../hook-to-otlp/tests/fixtures/cursor/session.jsonl. Each line runs +# emit.py as its own process, exactly as Cursor runs a hook command. Every +# run replaces the fixture's conversation_id and generation_id with fresh +# UUIDs (Cursor's own id shape, so suffixed generation ids such as +# "-0-ab12" still pair up), printed at the end with the trace id. +# Extra arguments go to replay.py (for example --replace MARKER=my-marker). +# Set FI_HOOK_CAPTURE_CONTENT=1 to exercise the content opt-in. +# REPLAY_SLEEP (default 1s) spaces the hooks out so the fixture's reported +# tool durations fit inside the turn, as they would in a real session. +# HOOK_TRANSPORT=stdin|heredoc installs the hooks into a temporary project +# with install.sh and runs the shipped hooks.json commands through $SHELL -c, +# passing the payload on stdin (IDE agent) or as a trailing heredoc (Cursor's +# headless agent host), instead of calling emit.py directly. +set -euo pipefail + +HERE="$(cd "$(dirname "$0")" && pwd)" +EMITTER="$HERE/../hook-to-otlp" +FIXTURE="${1:-$EMITTER/tests/fixtures/cursor/session.jsonl}" +shift || true +: "${FI_API_KEY:?set FI_API_KEY}" "${FI_SECRET_KEY:?set FI_SECRET_KEY}" +export FI_PROJECT_NAME="${FI_PROJECT_NAME:-cursor-e2e}" + +# The fixture's own ids (first record), and fresh UUIDs to replace them with. +read -r SRC_CONV SRC_GEN CONV GEN < <(python3 -B - "$FIXTURE" <<'PY' +import json, re, sys, uuid +payload = json.loads(open(sys.argv[1], encoding="utf-8").readline())["payload"] +conv = payload.get("conversation_id") or payload.get("session_id") +gen = re.sub(r"-\d+-[a-z0-9]{4}$", "", str(payload.get("generation_id") or "-")) +print(conv, gen, uuid.uuid4(), uuid.uuid4()) +PY +) + +REPLACE=(--replace "$SRC_CONV=$CONV") +if [ "$SRC_GEN" != "-" ]; then REPLACE+=(--replace "$SRC_GEN=$GEN"); fi +VIA=() +if [ -n "${HOOK_TRANSPORT:-}" ]; then + PROJECT="$(mktemp -d)" + trap 'rm -rf "$PROJECT"' EXIT + "$HERE/install.sh" "$PROJECT" >/dev/null + VIA=(--hooks-json "$PROJECT/.cursor/hooks.json" --cwd "$PROJECT" --transport "$HOOK_TRANSPORT") + echo "via installed hooks.json, transport=$HOOK_TRANSPORT" +fi +python3 -B "$EMITTER/replay.py" cursor "$FIXTURE" "${REPLACE[@]}" ${VIA[@]+"${VIA[@]}"} \ + --sleep "${REPLAY_SLEEP:-1}" ${@+"$@"} + +TRACE_ID="$(cd "$EMITTER" && python3 -B -c 'import sys, emit; print(emit.trace_id("cursor", sys.argv[1]))' "$CONV")" +echo "conversation_id=$CONV" +echo "trace_id=$TRACE_ID" diff --git a/examples/coding-agents/hook-to-otlp/README.md b/examples/coding-agents/hook-to-otlp/README.md new file mode 100644 index 000000000..fc1f6a01a --- /dev/null +++ b/examples/coding-agents/hook-to-otlp/README.md @@ -0,0 +1,178 @@ +# hook-to-otlp + +A small, stdlib-only Python (3.9+) hook command that turns a coding agent's +hook events into OTLP/HTTP spans for Future AGI. One agent session becomes one +trace. Docs page: . + +``` +emit.py CLI + core: state, span ids, parenting, privacy, export +adapters/base.py the adapter contract (SpanEvent, scopes, phases, helpers) +adapters/.py one module per app (cursor.py, ...) +replay.py feed a JSONL of recorded hook payloads through emit.py +tests/ unit + CLI tests; fixtures//*.jsonl +``` + +## Run + +```bash +export FI_API_KEY=... FI_SECRET_KEY=... # required, or nothing is sent +export FI_PROJECT_NAME=my-coding-agents # default: the adapter name +# export FI_BASE_URL=https://api.futureagi.com # default; spans go to $FI_BASE_URL/tracer/v1/traces +echo '{"conversation_id":"c1","generation_id":"g1","tool_name":"Shell","duration":12}' \ + | python3 emit.py cursor postToolUse +``` + +`python3 emit.py [] [--env-file PATH]` reads one JSON object on +stdin, prints what the host expects on stdout (`{}` unless the adapter says +otherwise; nothing if the adapter cannot be loaded or the script fails), and exits 0. On macOS and Linux the network send happens in a +detached child process so the host is not kept waiting. Windows has no fork: the +send is inline (default timeout 3 s) and state files are not locked. + +### Registering the hook (canonical guarded form) + +Every app that uses this emitter ships its hook commands in this form, so a missing emitter, a missing +`python3`, or a crash can never return exit code 2 (which many hosts treat as +"block") and never prints a decision: + +```bash +(python3 -B /emit.py 2>/dev/null || true) +``` + +`|| true` makes every failure exit 0 with empty stdout. The parentheses make it a +subshell, so the payload reaches `python3` whether the host pipes it to stdin or +appends it as a trailing heredoc (Cursor's headless agent host runs +` <<'CURSOR_HOOK_EOF'`); without them the heredoc binds to `true` and the +emitter reads nothing. `2>/dev/null` keeps Python's own error text out of host logs +(the emitter never writes to stderr); `-B` keeps `__pycache__` out of the customer's +repo. Pass the event name explicitly. Do not add `echo` fallbacks (a second JSON object +is invalid output). The form needs a POSIX shell; document the equivalent on hosts whose +hook runner is not one (and note that a host that execs the command without a shell gets +no guard). Add a config test like `tests/test_cursor_hooks_config.py`: run every shipped +command through `sh -c` with the payload on stdin and as a trailing heredoc, with the +emitter missing, with `python3` missing, and installed, and assert exit 0 plus the +expected (or empty) stdout and that the installed emitter received the payload. + +| Variable | Default | Meaning | +| --- | --- | --- | +| `FI_API_KEY`, `FI_SECRET_KEY` | none | Sent as `X-Api-Key` / `X-Secret-Key`. Missing: the hook is a no-op. | +| `FI_PROJECT_NAME` | adapter name | `project_name` resource attribute. | +| `FI_BASE_URL` | `https://api.futureagi.com` | Spans go to `$FI_BASE_URL/tracer/v1/traces`. | +| `FI_OTLP_ENDPOINT` | none | Full traces URL; overrides `FI_BASE_URL`. | +| `FI_HOOK_CAPTURE_CONTENT` | off | `1` exports prompts, tool input/output, model text, error text. | +| `FI_HOOK_MAX_CONTENT_CHARS` | `4096` | Per-field cap when content is on. | +| `FI_HOOK_TIMEOUT` | `10` (`3` without `fork`, i.e. Windows) | HTTP timeout in seconds. On macOS/Linux the send runs in a detached process, so the host never waits on it; on Windows it is inline. | +| `FI_HOOK_SYNC` | off | `1` sends inline instead of in a detached child (tests, debugging). | +| `FI_HOOK_STATE_DIR` | `$TMPDIR/fi-hook-to-otlp-` | Per-session state files (0600, removed at session end, swept after 24 h). With content on they also hold the content of spans that have not ended yet (for example the open turn's prompt) and of up to 8 turns closed as incomplete; completed turns keep no content. | +| `FI_HOOK_LOG_FILE` | none | One line per hook: event, span count, HTTP status. No content, no keys. | +| `FI_HOOK_DUMP_FILE` | none | Appends each OTLP/JSON body exactly as sent (privacy checks). | +| `FI_HOOK_DISABLED` | off | `1` turns the hook into a no-op. | +| `FI_HOOK_ENV_FILE` | `~/.config/fi-hook-to-otlp/env` | `KEY=VALUE` file read for `FI_*` keys missing from the environment (GUI apps often start without your shell env). | + +## Span model + +| Scope | Parent | Default `fi.span.kind` | Exported when | +| --- | --- | --- | --- | +| session | none (root) | `AGENT` | with every batch that exports a child span, and at session end (same span id and start time each time, so the latest export wins). A session with no activity is never exported. | +| turn | session | `CHAIN` | turn end | +| tool | turn, else session | `TOOL` | tool end | +| subagent | turn, else session | `AGENT` | subagent end | +| llm | turn, else session | `LLM` | instantly | +| event | turn, else session | `CHAIN` | instantly | + +- Trace id = `sha256("trace", app, session_id)[:32]`; span ids are derived from + `(app, session_id, scope, key)`, so retries of the same hook overwrite rather than duplicate. +- A span's start is the matching START hook if one was seen, else `end - duration_ms` + from the payload, else the hook time, never earlier than the root's start once the + root has been sent. A turn opens on the first event that names it. +- A tool/subagent/llm/event span without `turn_id` goes under the open turn and gets + `coding_agent.turn.inferred=true`; with no open turn it goes under the session with no + turn marker. +- A turn end closes still-open children of that turn (`coding_agent.incomplete=true`); + a session end closes everything and deletes the state file. +- One turn is open per session: a turn START closes any other open turn (and its + children) as incomplete. If that turn's END arrives later, the turn is re-sent complete + with its original start. +- Every span carries `session.id`, `gen_ai.conversation.id`, `coding_agent.app`, + `coding_agent.scope`, `coding_agent.hook.event`, and `coding_agent.turn.id` when known. + Tool spans add `gen_ai.tool.name` and `gen_ai.tool.call.id`; session/turn/llm/subagent + spans add `llm.model_name` when the payload names a model (the root uses the last model + reported by a session, turn or llm event). +- Opted-in content keeps paths but replaces the user name in home-directory paths with + `` (`/Users//...`, `/home//...`, `C:\Users\\...`, `~/...`). + The name is the one directory component after the prefix; it stops at whitespace, + quotes, and punctuation no account name uses, so the rest of a command stays intact + (`cd /Users/ && ls src/x`). This machine's own user name (the last component of + `os.path.expanduser("~")`) is matched literally, so it is replaced in full even when it + contains spaces. Limitation: another account's name with spaces (for example from a + log pasted from another machine) keeps the part after the first space + (`/Users/ Smith/...`). + +## Writing an adapter + +Create `adapters/.py` (module name = app slug, `[a-z0-9_]+`): + +```python +from adapters.base import END, START, SESSION, TOOL, TURN, OK, ERROR, SpanEvent, compact, first, number + +APP = "myapp" +DISPLAY_NAME = "My App" # "My App session", "My App turn" +SERVICE_NAME = "myapp" # optional, resource service.name + +def parse(event: str, payload: dict) -> list: # required + sid = first(payload, "session_id") + if not sid: + return [] # never invent a session id + if event == "PostToolUse": + return [SpanEvent(END, TOOL, str(sid), key=payload.get("tool_use_id", ""), + name=payload.get("tool_name"), duration_ms=number(payload.get("duration_ms")), + input=payload.get("tool_input"), output=payload.get("tool_response"))] + return [] + +def respond(event: str, payload: dict) -> str: # optional, default "{}" + return "{}" # "" prints nothing (hosts that add stdout to the model context) + +def event_name(argv_event, payload): # optional + return argv_event or payload.get("hook_event_name") + +def resource_attributes(event: str, payload: dict) -> dict: # optional, metadata only + return compact({"coding_agent.app.version": payload.get("version")}) +``` + +`SpanEvent(phase, scope, session_id, key="", turn_id=None, name=None, kind=None, +attributes={}, model=None, duration_ms=None, status=None, status_message=None, +input=None, output=None, error_message=None)`: + +- `phase`: `START` opens, `END` closes and exports, `INSTANT` exports a complete span, + `UPDATE` merges into an open span (for example a response text into the turn). +- `scope`: `SESSION`, `TURN`, `TOOL`, `SUBAGENT`, `LLM`, `EVENT`. +- `key` pairs START/END of the same scope (tool call id, turn id). Empty for the session. + Apps without ids can leave it empty: a keyless TURN START opens `turn-1`, `turn-2`, ... + and a keyless TURN END/UPDATE applies to the open turn (ignored if none); a keyless + END of another scope pairs with the oldest open keyless START of the same scope and + `name` (FIFO), else it becomes a standalone span. +- `turn_id` parents a tool/subagent/llm/event span; None means the open turn, else the session. +- `status` is `OK`, `ERROR` or None. `status_message` is always exported, so keep it to + metadata (`"timeout"`), never free text. +- `input`, `output`, `error_message` are content: dropped unless `FI_HOOK_CAPTURE_CONTENT=1`. +- `attributes` are always exported: ids, names, counts, enums only. Namespace them + `.*`. Never emails, file paths, transcript paths, or text bodies. + +Do not wire the emitter to hooks whose output decides whether the agent may act +(permission hooks): the emitter observes, it does not approve. If an app requires an +answer on such a hook, `respond()` must not approve; return `""` (print nothing) when +the app treats no output as "no decision", as the Cursor adapter does. + +Add `tests/fixtures//session.jsonl` (`{"event": ..., "payload": ...}` per line, +shaped exactly like the vendor docs) and `tests/test__adapter.py`. The Cursor +adapter and its test are the reference. + +## Test + +```bash +python3 -m unittest discover -s tests -t . +# replay a recorded session into a real collector with a fresh session id +python3 replay.py cursor tests/fixtures/cursor/session.jsonl --replace fixture-conv-0001=run-$(date +%s) +# same, through the commands of an installed hooks file, payload as a trailing heredoc +python3 replay.py cursor tests/fixtures/cursor/session.jsonl --replace fixture-conv-0001=run-$(date +%s) \ + --hooks-json /path/to/project/.cursor/hooks.json --cwd /path/to/project --transport heredoc +``` diff --git a/examples/coding-agents/hook-to-otlp/adapters/__init__.py b/examples/coding-agents/hook-to-otlp/adapters/__init__.py new file mode 100644 index 000000000..1ed9cc212 --- /dev/null +++ b/examples/coding-agents/hook-to-otlp/adapters/__init__.py @@ -0,0 +1,7 @@ +"""Per-app adapters for the hook-to-OTLP emitter. + +Each module in this package maps one coding agent's hook payloads onto the +normalized ``SpanEvent`` records defined in ``adapters.base``. ``emit.py`` +loads an adapter by its module name (``python3 emit.py cursor`` imports +``adapters.cursor``). +""" diff --git a/examples/coding-agents/hook-to-otlp/adapters/base.py b/examples/coding-agents/hook-to-otlp/adapters/base.py new file mode 100644 index 000000000..f4f617368 --- /dev/null +++ b/examples/coding-agents/hook-to-otlp/adapters/base.py @@ -0,0 +1,124 @@ +"""Adapter contract for the hook-to-OTLP emitter. + +An adapter is a plain module ``adapters/.py``. It never does I/O: it +receives the hook event name and the parsed stdin payload and returns +normalized ``SpanEvent`` records. The emitter core (``emit.py``) owns state, +span ids, parenting, durations, privacy, and export. + +Required module attributes:: + + APP: str # slug, equal to the module name ("cursor") + DISPLAY_NAME: str # used in default span names ("Cursor session") + + def parse(event: str, payload: dict) -> list[SpanEvent]: + ... + +Optional module attributes:: + + SERVICE_NAME: str # OTel resource service.name (default: APP) + + def event_name(argv_event: str | None, payload: dict) -> str | None: + # default: argv_event, else payload["hook_event_name"] + + def respond(event: str, payload: dict) -> str: + # what the host app expects on stdout; default "{}" + + def resource_attributes(event: str, payload: dict) -> dict: + # extra resource attributes, e.g. the app version; metadata only + +Rules for adapters: + +* Return ``[]`` for events you do not map, and when the payload has no + session id. Never invent a session id. +* Put content (prompts, tool arguments, tool results, model text, error + text) ONLY in ``input``, ``output`` and ``error_message``. The core drops + them unless the user opts in with ``FI_HOOK_CAPTURE_CONTENT=1``. +* ``attributes`` and ``status_message`` are always exported: put only + metadata there (ids, names, counts, durations, enum-like statuses). Never + emails, paths, transcript locations, or free text. +* Never open files named in the payload (for example ``transcript_path``). +""" +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any, Dict, Optional + +# Scopes: what the span represents. The core parents them as +# session -> turn -> (tool | subagent | llm | event). +SESSION = "session" +TURN = "turn" +TOOL = "tool" +SUBAGENT = "subagent" +LLM = "llm" +EVENT = "event" +SCOPES = (SESSION, TURN, TOOL, SUBAGENT, LLM, EVENT) + +# Phases: how the hook relates to the span's lifetime. +START = "start" # opens a span; nothing is exported until it ends +END = "end" # closes (and exports) a span +INSTANT = "instant" # a complete span in one hook (start = end - duration_ms) +UPDATE = "update" # merges attributes/content into an open span +PHASES = (START, END, INSTANT, UPDATE) + +OK = "ok" +ERROR = "error" + + +@dataclass +class SpanEvent: + """One normalized hook observation. + + ``key`` pairs START/END/UPDATE events of the same scope inside a session + (turn id, tool call id, subagent id). Leave it empty for the session + scope, and for INSTANT/END events that have no stable id (the core then + makes the span unique). + + ``turn_id`` names the parent turn of tool/subagent/llm/event spans. When + it is None the core uses the session's currently open turn, or parents + the span directly under the session. + """ + + phase: str + scope: str + session_id: str + key: str = "" + turn_id: Optional[str] = None + name: Optional[str] = None + kind: Optional[str] = None # fi.span.kind; default derived from scope + attributes: Dict[str, Any] = field(default_factory=dict) + model: Optional[str] = None + duration_ms: Optional[float] = None + status: Optional[str] = None # OK | ERROR | None (unset) + status_message: Optional[str] = None # metadata only, e.g. "timeout" + # Content. Exported only when FI_HOOK_CAPTURE_CONTENT is on. + input: Any = None + output: Any = None + error_message: Any = None + + +def first(payload: Dict[str, Any], *keys: str) -> Any: + """First non-empty value among ``keys`` in ``payload``, else None.""" + for key in keys: + value = payload.get(key) + if value not in (None, ""): + return value + return None + + +def number(value: Any) -> Optional[float]: + """``value`` as a float when it is numeric (bools excluded), else None.""" + if isinstance(value, bool): + return None + if isinstance(value, (int, float)): + return float(value) + if isinstance(value, str): + try: + return float(value) + except ValueError: + return None + return None + + +def compact(attributes: Dict[str, Any]) -> Dict[str, Any]: + """Drop None/empty values so adapters can build attribute dicts inline.""" + return {k: v for k, v in attributes.items() if v not in (None, "", [], {})} diff --git a/examples/coding-agents/hook-to-otlp/adapters/cursor.py b/examples/coding-agents/hook-to-otlp/adapters/cursor.py new file mode 100644 index 000000000..5e5ef1e90 --- /dev/null +++ b/examples/coding-agents/hook-to-otlp/adapters/cursor.py @@ -0,0 +1,299 @@ +"""Cursor agent hooks -> normalized span events. + +Payload shapes follow https://cursor.com/docs/hooks (read 2026-10-14): +every agent hook carries ``conversation_id`` (session), ``generation_id`` +(changes with every user message, so it is the turn) and ``model`` / +``model_id``. ``sessionStart`` / ``sessionEnd`` carry ``session_id``, which +the docs define as the same value as ``conversation_id``. + +Mapped events (all non-permission hooks, so the emitter never decides +whether Cursor may run something): + + sessionStart -> session start (nothing exported until a child span is sent) + beforeSubmitPrompt -> turn start prompt = content + postToolUse -> tool span start = end - duration + postToolUseFailure -> tool span, ERROR status message = failure_type + subagentStop -> subagent span start = end - duration_ms + afterAgentThought -> llm span "Agent thought" + afterAgentResponse -> llm span "Agent response", turn output = content + preCompact -> event span "Context compaction" (counts only) + stop -> turn end, ERROR when status == "error" + sessionEnd -> session end, ERROR when reason == "error" + +Never exported: ``user_email``, ``transcript_path`` (never opened), +``workspace_roots``, ``cwd``, attachment paths. + +Observed on real Cursor 3.24.18 payloads (not in the docs): + +- ``afterAgentResponse`` and ``stop`` carry ``input_tokens``, ``output_tokens``, + ``cache_read_tokens``, ``cache_write_tokens`` for the turn. They go on the + "Agent response" LLM span as standard token counts (the turn keeps them as + ``cursor.usage.*`` so trace totals are not counted twice). +- ``tool_use_id`` can be two ids joined by a newline (Cursor's ``call-...`` + id, then the provider's function-call id). The first line is the call id; + the second is kept as ``cursor.tool.provider_call_id``. +- ``afterAgentThought`` fires twice per thinking block, once with a + ``generation_id`` suffixed ``--<4 chars>``. Suffixes are stripped and the + pair maps to one span id. +""" +from __future__ import annotations + +import hashlib +import re +from typing import Any, Dict, List, Optional + +from adapters.base import ( + END, ERROR, EVENT, INSTANT, LLM, OK, SESSION, START, SUBAGENT, TOOL, TURN, + UPDATE, SpanEvent, compact, first, number, +) + +APP = "cursor" +DISPLAY_NAME = "Cursor" +SERVICE_NAME = "cursor" + +# The emitter is not registered on these (it observes, it never decides). +# If it is wired to one by mistake it prints nothing: Cursor treats a hook +# with no output as failed and lets the action through (fail-open unless the +# hook sets failClosed), the same as a missing emitter. It never answers +# "allow" (that could approve an action) and never prints "{}" (invalid for a +# permission hook, which blocks the action). +PERMISSION_HOOKS = { + "preToolUse", "beforeShellExecution", "beforeMCPExecution", + "beforeReadFile", "beforeTabFileRead", "subagentStart", +} + + +def event_name(argv_event: Optional[str], payload: Dict[str, Any]) -> Optional[str]: + return argv_event or payload.get("hook_event_name") + + +def respond(event: str, payload: Dict[str, Any]) -> str: + if event == "beforeSubmitPrompt": + return '{"continue": true}' + if event in PERMISSION_HOOKS: + return "" + return "{}" + + +def resource_attributes(event: str, payload: Dict[str, Any]) -> Dict[str, Any]: + return compact({"coding_agent.app.version": payload.get("cursor_version")}) + + +def _model(payload: Dict[str, Any]) -> Optional[str]: + value = first(payload, "model_id", "model") + return str(value) if value is not None else None + + +def _model_params(payload: Dict[str, Any]) -> Dict[str, Any]: + params = payload.get("model_params") + out: Dict[str, Any] = {} + if isinstance(params, list): + for item in params: + if isinstance(item, dict) and item.get("id") and item.get("value") is not None: + out[f"cursor.model_params.{item['id']}"] = str(item["value"]) + if payload.get("model") and payload.get("model_id") and payload["model"] != payload["model_id"]: + out["cursor.model"] = str(payload["model"]) + return out + + +_SUFFIXED_GENERATION = re.compile( + r"^([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})-\d+-[a-z0-9]{4}$") + + +def _generation(payload: Dict[str, Any]) -> Optional[str]: + """The turn id: ``generation_id`` without Cursor's per-phase suffix.""" + gen = first(payload, "generation_id") + if gen is None: + return None + gen = str(gen) + match = _SUFFIXED_GENERATION.match(gen) + return match.group(1) if match else gen + + +def _usage(payload: Dict[str, Any], prefix: str) -> Dict[str, Any]: + tokens_in = payload.get("input_tokens") + tokens_out = payload.get("output_tokens") + if not isinstance(tokens_in, int) and not isinstance(tokens_out, int): + return {} + usage: Dict[str, Any] = {} + if prefix == "llm": + usage = { + "llm.token_count.prompt": tokens_in, + "llm.token_count.completion": tokens_out, + "llm.token_count.prompt_details.cache_read": payload.get("cache_read_tokens"), + "llm.token_count.prompt_details.cache_write": payload.get("cache_write_tokens"), + } + if isinstance(tokens_in, int) and isinstance(tokens_out, int): + usage["llm.token_count.total"] = tokens_in + tokens_out + else: + for key in ("input_tokens", "output_tokens", "cache_read_tokens", "cache_write_tokens"): + usage[f"cursor.usage.{key}"] = payload.get(key) + return compact(usage) + + +def _tool_call_ids(payload: Dict[str, Any]) -> "tuple[str, Optional[str]]": + """(call id, provider call id) from a possibly newline-joined ``tool_use_id``.""" + raw = first(payload, "tool_use_id") + if raw is None: + return "", None + parts = [part.strip() for part in str(raw).splitlines() if part.strip()] + if not parts: + return "", None + return parts[0], (parts[1] if len(parts) > 1 else None) + + +def _digest(*parts: Any) -> str: + raw = "\x1f".join("" if p is None else str(p) for p in parts) + return hashlib.sha256(raw.encode("utf-8", "replace")).hexdigest()[:24] + + +def parse(event: str, payload: Dict[str, Any]) -> List[SpanEvent]: + session = first(payload, "conversation_id", "session_id") + if not session: + return [] # R-01: never invent a session id + sid = str(session) + turn = _generation(payload) + model = _model(payload) + + if event == "sessionStart": + return [SpanEvent(START, SESSION, sid, model=model, attributes=compact({ + "cursor.composer_mode": payload.get("composer_mode"), + "cursor.is_background_agent": payload.get("is_background_agent"), + }))] + + if event == "sessionEnd": + reason = payload.get("reason") + failed = reason == "error" + return [SpanEvent( + END, SESSION, sid, + duration_ms=number(payload.get("duration_ms")), + status=ERROR if failed else OK, + status_message="session ended with error" if failed else None, + error_message=payload.get("error_message") if failed else None, + attributes=compact({ + "cursor.session.reason": reason, + "cursor.session.final_status": payload.get("final_status"), + "cursor.session.duration_ms": number(payload.get("duration_ms")), + "cursor.is_background_agent": payload.get("is_background_agent"), + }), + )] + + if event == "beforeSubmitPrompt": + if turn is None: + return [] + attachments = payload.get("attachments") + return [SpanEvent( + START, TURN, sid, key=turn, model=model, input=payload.get("prompt"), + attributes=compact({ + "cursor.generation_id": turn, + "cursor.attachments.count": len(attachments) if isinstance(attachments, list) else None, + **_model_params(payload), + }), + )] + + if event == "stop": + if turn is None: + return [] + status = payload.get("status") + failed = status == "error" + return [SpanEvent( + END, TURN, sid, key=turn, model=model, + status=ERROR if failed else OK, + status_message="agent loop ended with error" if failed else None, + attributes=compact({ + "cursor.generation_id": turn, + "cursor.stop.status": status, + "cursor.loop_count": payload.get("loop_count"), + **_model_params(payload), + **_usage(payload, "turn"), + }), + )] + + if event in ("postToolUse", "postToolUseFailure"): + tool_name = str(first(payload, "tool_name") or "tool") + failed = event == "postToolUseFailure" + failure_type = payload.get("failure_type") if failed else None + call_id, provider_call_id = _tool_call_ids(payload) + return [SpanEvent( + END, TOOL, sid, + key=call_id, + turn_id=turn, + name=tool_name, + duration_ms=number(payload.get("duration")), + status=ERROR if failed else OK, + status_message=str(failure_type or "error") if failed else None, + input=payload.get("tool_input"), + output=None if failed else payload.get("tool_output"), + error_message=payload.get("error_message") if failed else None, + attributes=compact({ + "cursor.tool.name": tool_name, + "cursor.tool.provider_call_id": provider_call_id, + "cursor.tool.duration_ms": number(payload.get("duration")), + "cursor.tool.failure_type": failure_type, + "cursor.tool.is_interrupt": payload.get("is_interrupt") if failed else None, + }), + )] + + if event == "subagentStop": + subagent_type = payload.get("subagent_type") or "subagent" + status = payload.get("status") + failed = status == "error" + modified = payload.get("modified_files") + return [SpanEvent( + END, SUBAGENT, sid, + key=_digest(subagent_type, turn, payload.get("duration_ms"), + payload.get("task"), payload.get("description")), + turn_id=turn, + name=f"Subagent: {subagent_type}", + duration_ms=number(payload.get("duration_ms")), + status=ERROR if failed else OK, + status_message="subagent ended with error" if failed else None, + input=payload.get("task"), + output=payload.get("summary"), + attributes=compact({ + "cursor.subagent.type": subagent_type, + "cursor.subagent.status": status, + "cursor.subagent.message_count": payload.get("message_count"), + "cursor.subagent.tool_call_count": payload.get("tool_call_count"), + "cursor.subagent.modified_files.count": len(modified) if isinstance(modified, list) else None, + "cursor.subagent.duration_ms": number(payload.get("duration_ms")), + }), + )] + + if event == "afterAgentThought": + text = payload.get("text") + return [SpanEvent( + INSTANT, LLM, sid, turn_id=turn, name="Agent thought", model=model, + key=_digest("thought", turn, payload.get("duration_ms"), + hashlib.sha256(str(text).encode("utf-8", "replace")).hexdigest()), + duration_ms=number(payload.get("duration_ms")), + output=payload.get("text"), + attributes=compact({"cursor.llm.part": "thought", **_model_params(payload)}), + )] + + if event == "afterAgentResponse": + events = [SpanEvent( + INSTANT, LLM, sid, turn_id=turn, name="Agent response", model=model, + output=payload.get("text"), + attributes=compact({"cursor.llm.part": "response", **_model_params(payload), + **_usage(payload, "llm")}), + )] + if turn is not None: + events.append(SpanEvent(UPDATE, TURN, sid, key=turn, output=payload.get("text"))) + return events + + if event == "preCompact": + return [SpanEvent( + INSTANT, EVENT, sid, turn_id=turn, name="Context compaction", + attributes=compact({ + "cursor.compaction.trigger": payload.get("trigger"), + "cursor.compaction.context_usage_percent": number(payload.get("context_usage_percent")), + "cursor.compaction.context_tokens": payload.get("context_tokens"), + "cursor.compaction.context_window_size": payload.get("context_window_size"), + "cursor.compaction.message_count": payload.get("message_count"), + "cursor.compaction.messages_to_compact": payload.get("messages_to_compact"), + "cursor.compaction.is_first": payload.get("is_first_compaction"), + }), + )] + + return [] diff --git a/examples/coding-agents/hook-to-otlp/emit.py b/examples/coding-agents/hook-to-otlp/emit.py new file mode 100755 index 000000000..6a6da4e63 --- /dev/null +++ b/examples/coding-agents/hook-to-otlp/emit.py @@ -0,0 +1,863 @@ +#!/usr/bin/env python3 +"""Coding-agent hook -> OTLP/HTTP spans for Future AGI. + +Usage (as a hook command): + + python3 emit.py [] [--env-file PATH] < payload.json + +```` selects ``adapters/.py``. ```` is the hook name when +the host does not put it in the payload. The script reads one JSON object +from stdin, prints what the host expects on stdout, and always exits 0. +If the adapter cannot be loaded or the script fails, it prints nothing +(the same as a missing emitter behind the guarded hook command). + +One agent session becomes one trace (the trace id is derived from the +session id). Turns, tools, subagents, and model responses become child +spans. Start/end hooks are paired through a small per-session state file. + +Privacy: prompts, tool arguments, tool results, model text, and error text +are dropped unless FI_HOOK_CAPTURE_CONTENT=1. Standard library only, +Python 3.9+. +""" +from __future__ import annotations + +import contextlib +import dataclasses +import functools +import hashlib +import importlib +import json +import os +import re +import secrets +import sys +import tempfile +import time +import urllib.error +import urllib.request +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Dict, Iterator, List, Mapping, Optional, Tuple + +HERE = Path(__file__).resolve().parent +if str(HERE) not in sys.path: + sys.path.insert(0, str(HERE)) + +from adapters.base import ( # noqa: E402 + END, ERROR, EVENT, INSTANT, LLM, OK, SESSION, START, SUBAGENT, TOOL, TURN, + UPDATE, SpanEvent, +) + +VERSION = "0.1.0" +SCOPE_NAME = "fi-hook-to-otlp" +DEFAULT_BASE_URL = "https://api.futureagi.com" +TRACES_PATH = "/tracer/v1/traces" +DEFAULT_ENV_FILE = Path("~/.config/fi-hook-to-otlp/env") + +DEFAULT_KIND = { + SESSION: "AGENT", TURN: "CHAIN", TOOL: "TOOL", + SUBAGENT: "AGENT", LLM: "LLM", EVENT: "CHAIN", +} +OTLP_STATUS = {None: 0, OK: 1, ERROR: 2} +SECRET_PATTERNS = ( + re.compile(r"-----BEGIN [A-Z ]*PRIVATE KEY-----[\s\S]*?-----END [A-Z ]*PRIVATE KEY-----"), + re.compile(r"\b(?:sk|pk|rk)-[A-Za-z0-9_-]{16,}"), + re.compile(r"\bAKIA[0-9A-Z]{16}\b"), + re.compile(r"\bgh[pousr]_[A-Za-z0-9]{20,}"), + re.compile(r"\bxox[abprs]-[A-Za-z0-9-]{10,}"), + re.compile(r"(?i)\b(bearer\s+)[A-Za-z0-9._~+/=-]{12,}"), + re.compile(r"(?i)\b((?:api[_-]?key|secret[_-]?key|access[_-]?token|password|passwd|token)\b[\"']?\s*[:=]\s*[\"']?)[^\s\"',}]+"), +) +# Opted-in content keeps paths but not the account name inside them: the one +# directory component after /Users/, /home/ or C:\Users\ (raw, JSON-escaped or +# with forward slashes). A component stops at whitespace, quotes and characters +# no macOS, Linux or Windows account name uses (shell and sentence punctuation), +# and does not end with a dot, so text after the path ("&& ls src/x", "),") +# stays as it was. Names with spaces are matched only for this machine's own +# home directory (see _own_home_pattern). +_HOME_PREFIX = r"(/Users/|/home/|[A-Za-z]:(?:\\\\|\\|/)+Users(?:\\\\|\\|/)+)" +_NAME_CHAR = r"[^\s/\\\"'`\[\]:;|=,+*?<>(){}&]" +_NAME = _NAME_CHAR + r"*(?:(?!\.)" + _NAME_CHAR + r")" +# Own name at a boundary: not followed by more name characters (al vs alice). +_NAME_END = r"(?!\.*(?:(?!\.)" + _NAME_CHAR + r"))" +HOME_PATTERNS = ( + re.compile(_HOME_PREFIX + _NAME), + re.compile(r"((? "Config": + base = env.get("FI_BASE_URL") or DEFAULT_BASE_URL + endpoint = env.get("FI_OTLP_ENDPOINT") or base.rstrip("/") + TRACES_PATH + state_root = env.get("FI_HOOK_STATE_DIR") or os.path.join( + tempfile.gettempdir(), f"fi-hook-to-otlp-{_uid()}") + return cls( + app=app, + endpoint=endpoint, + api_key=env.get("FI_API_KEY", ""), + secret_key=env.get("FI_SECRET_KEY", ""), + project_name=env.get("FI_PROJECT_NAME") or app, + capture_content=_truthy(env.get("FI_HOOK_CAPTURE_CONTENT")), + max_content_chars=_int(env.get("FI_HOOK_MAX_CONTENT_CHARS"), 4096), + # Background send (fork) by default; where there is no fork (Windows) + # the send is inline, so keep the host's wait short. + timeout=_float(env.get("FI_HOOK_TIMEOUT"), 10.0 if hasattr(os, "fork") else 3.0), + sync=_truthy(env.get("FI_HOOK_SYNC")), + state_dir=Path(state_root) / app, + state_ttl_s=_float(env.get("FI_HOOK_STATE_TTL_HOURS"), 24.0) * 3600, + log_file=env.get("FI_HOOK_LOG_FILE") or None, + dump_file=env.get("FI_HOOK_DUMP_FILE") or None, + disabled=_truthy(env.get("FI_HOOK_DISABLED")), + ) + + @property + def can_export(self) -> bool: + return bool(self.api_key and self.secret_key and not self.disabled) + + +def _truthy(value: Optional[str]) -> bool: + return (value or "").strip().lower() in {"1", "true", "yes", "on"} + + +def _int(value: Optional[str], default: int) -> int: + try: + return int(value) if value else default + except ValueError: + return default + + +def _float(value: Optional[str], default: float) -> float: + try: + return float(value) if value else default + except ValueError: + return default + + +def _uid() -> str: + getuid = getattr(os, "getuid", None) + return str(getuid()) if getuid else os.environ.get("USERNAME", "user") + + +# ---------------------------------------------------------------- ids + + +def _hex(*parts: str, length: int) -> str: + raw = "\x1f".join(parts).encode("utf-8", "replace") + return hashlib.sha256(raw).hexdigest()[:length] + + +def trace_id(app: str, session_id: str) -> str: + """Deterministic 32-hex OTLP trace id: one session, one trace.""" + return _hex("trace", app, session_id, length=32) + + +def span_id(app: str, session_id: str, scope: str, key: str) -> str: + """Deterministic 16-hex span id for (session, scope, key).""" + return _hex("span", app, session_id, scope, key, length=16) + + +# ---------------------------------------------------------------- content + + +def redact(text: str) -> str: + for pattern in SECRET_PATTERNS: + if pattern.groups: + text = pattern.sub(lambda m: m.group(1) + "[REDACTED]", text) + else: + text = pattern.sub("[REDACTED]", text) + return text + + +@functools.lru_cache(maxsize=4) +def _own_home_pattern(home: str) -> Optional["re.Pattern[str]"]: + """The basename of this machine's home directory after any home prefix. + + Matched literally, so a name the generic pattern cannot take whole (one + with spaces, quotes or parentheses) is still replaced in full. + """ + name = re.split(r"[/\\]", home.rstrip("/\\"))[-1] + if not name or name in (".", ".."): + return None + return re.compile(_HOME_PREFIX + re.escape(name) + _NAME_END) + + +def scrub_home(text: str) -> str: + """Replace the user name in home-directory paths with .""" + own = _own_home_pattern(os.path.expanduser("~")) + if own: # this machine's own name first: it may contain spaces + text = own.sub(lambda m: m.group(1) + "", text) + for pattern in HOME_PATTERNS: + text = pattern.sub(lambda m: m.group(1) + "", text) + return text + + +def render_content(value: Any, max_chars: int) -> Optional[str]: + if value is None: + return None + text = value if isinstance(value, str) else json.dumps(value, ensure_ascii=False, sort_keys=True, default=str) + text = scrub_home(redact(text)) + if max_chars > 0 and len(text) > max_chars: + text = text[:max_chars] + f"...[truncated {len(text) - max_chars} chars]" + return text + + +# ---------------------------------------------------------------- state + + +@contextlib.contextmanager +def session_state(cfg: Config, session_id: str) -> Iterator[Dict[str, Any]]: + """Lock and load the per-session state file; save it on exit. + + The state dir is created 0700. If the yielded dict gets ``_delete`` set, + the file is removed instead of saved. + """ + cfg.state_dir.mkdir(parents=True, exist_ok=True, mode=0o700) + path = cfg.state_dir / (_hex("state", cfg.app, session_id, length=32) + ".json") + fd = os.open(str(path), os.O_RDWR | os.O_CREAT, 0o600) + with os.fdopen(fd, "r+", encoding="utf-8") as handle: + _lock(handle) + try: + raw = handle.read() + try: + state = json.loads(raw) if raw.strip() else {} + except ValueError: + state = {} + if not isinstance(state, dict): + state = {} + yield state + if state.pop("_delete", False): + os.unlink(str(path)) + else: + handle.seek(0) + handle.truncate() + handle.write(json.dumps(state, separators=(",", ":"))) + handle.flush() + finally: + _unlock(handle) + + +def _lock(handle: Any) -> None: + try: + import fcntl + fcntl.flock(handle.fileno(), fcntl.LOCK_EX) + except ImportError: # Windows: best effort, no lock + pass + + +def _unlock(handle: Any) -> None: + try: + import fcntl + fcntl.flock(handle.fileno(), fcntl.LOCK_UN) + except ImportError: + pass + + +def sweep_state(cfg: Config, now_s: Optional[float] = None) -> None: + """Remove state files untouched for longer than FI_HOOK_STATE_TTL_HOURS.""" + now_s = time.time() if now_s is None else now_s + try: + for entry in cfg.state_dir.glob("*.json"): + with contextlib.suppress(OSError): + if now_s - entry.stat().st_mtime > cfg.state_ttl_s: + entry.unlink() + except OSError: + pass + + +# ---------------------------------------------------------------- spans + + +class SpanBuilder: + """Applies SpanEvents to one session's state and collects finished spans.""" + + def __init__(self, cfg: Config, adapter: Any, state: Dict[str, Any], session_id: str, now_ns: int): + self.cfg = cfg + self.adapter = adapter + self.state = state + self.sid = session_id + self.now = now_ns + self.spans: List[Dict[str, Any]] = [] + state.setdefault("open", {}) + state.setdefault("ended_turns", []) + state.setdefault("session", {}) + self.trace = trace_id(cfg.app, session_id) + self.root = span_id(cfg.app, session_id, SESSION, "") + + # -- helpers + + def _start_of(self, ev: SpanEvent) -> int: + if ev.phase in (END, INSTANT) and ev.duration_ms and ev.duration_ms > 0: + return self.now - int(ev.duration_ms * 1_000_000) + return self.now + + def _ensure_session(self, start_ns: int) -> Dict[str, Any]: + session = self.state["session"] + if "start_ns" not in session: + session["start_ns"] = start_ns + elif not session.get("exported") and start_ns < session["start_ns"]: + session["start_ns"] = start_ns + return session + + def _turn_key(self, ev: SpanEvent) -> Optional[str]: + if ev.scope == TURN: + return ev.key or None + return ev.turn_id or self.state.get("current_turn") + + def _ensure_turn(self, key: str, start_ns: int) -> Optional[Dict[str, Any]]: + """Open turn record for ``key`` (auto-opened on first sight).""" + if key in self.state["ended_turns"]: + return None + slot = f"{TURN}:{key}" + rec = self.state["open"].get(slot) + if rec is None: + rec = self.state["open"][slot] = { + "scope": TURN, "key": key, "start_ns": start_ns, "attrs": {}, + "parent": self.root, + } + self.state["current_turn"] = key + elif start_ns < rec["start_ns"]: + rec["start_ns"] = start_ns + return rec + + def _parent_for(self, ev: SpanEvent, start_ns: int) -> Tuple[str, Optional[str]]: + if ev.scope == SESSION: + return "", None + if ev.scope == TURN: + return self.root, ev.key or None + turn = self._turn_key(ev) + if turn: + self._ensure_turn(turn, start_ns) + return span_id(self.cfg.app, self.sid, TURN, turn), turn + return self.root, None + + def _merge(self, rec: Dict[str, Any], ev: SpanEvent) -> None: + rec.setdefault("attrs", {}).update(ev.attributes or {}) + for field in ("name", "kind", "model", "status", "status_message"): + value = getattr(ev, field) + if value is not None: + rec[field] = value + if self.cfg.capture_content: + for field in ("input", "output", "error_message"): + value = getattr(ev, field) + if value is not None: + rec[field] = render_content(value, self.cfg.max_content_chars) + + def _default_name(self, scope: str, rec: Dict[str, Any]) -> str: + display = getattr(self.adapter, "DISPLAY_NAME", self.cfg.app) + if scope == SESSION: + return f"{display} session" + if scope == TURN: + return f"{display} turn" + if scope == TOOL: + return "tool" + return scope + + def _emit(self, scope: str, sid: str, parent: str, start_ns: int, end_ns: int, + rec: Dict[str, Any], event: str, turn: Optional[str]) -> None: + session = self.state["session"] + if scope != SESSION and session.get("exported") and start_ns < session["start_ns"]: + # The root's start is fixed once exported (it keys the stored row); + # keep children inside it. + start_ns = session["start_ns"] + attrs: Dict[str, Any] = dict(rec.get("attrs") or {}) + kind = rec.get("kind") or DEFAULT_KIND.get(scope, "UNKNOWN") + name = rec.get("name") or self._default_name(scope, rec) + attrs.update({ + "fi.span.kind": kind, + "session.id": self.sid, + "gen_ai.conversation.id": self.sid, + "coding_agent.app": self.cfg.app, + "coding_agent.scope": scope, + "coding_agent.hook.event": event, + }) + if turn: + attrs["coding_agent.turn.id"] = turn + if scope == TOOL: + attrs.setdefault("gen_ai.tool.name", name) + if rec.get("key") and not rec.get("synthetic_key"): + attrs.setdefault("gen_ai.tool.call.id", rec["key"]) + model = rec.get("model") + if model and scope in (SESSION, TURN, LLM, SUBAGENT): + attrs["llm.model_name"] = model + attrs["gen_ai.request.model"] = model + if rec.get("incomplete"): + attrs["coding_agent.incomplete"] = True + status = rec.get("status") + message = rec.get("status_message") + if self.cfg.capture_content: + if rec.get("input") is not None: + attrs["input.value"] = rec["input"] + if rec.get("output") is not None: + attrs["output.value"] = rec["output"] + if rec.get("error_message") is not None: + attrs["exception.message"] = rec["error_message"] + message = rec["error_message"] + span: Dict[str, Any] = { + "traceId": self.trace, + "spanId": sid, + "name": str(name), + "kind": 1, + "startTimeUnixNano": str(start_ns), + "endTimeUnixNano": str(max(end_ns, start_ns)), + "attributes": otlp_attributes(attrs), + "status": {"code": OTLP_STATUS.get(status, 0)}, + } + if parent: + span["parentSpanId"] = parent + if status == ERROR and message: + span["status"]["message"] = str(message)[:512] + self.spans.append(span) + + def _emit_root(self, event: str) -> None: + session = self.state["session"] + rec = dict(session) + rec.setdefault("attrs", {}) + if not rec.get("model") and self.state.get("model"): + rec["model"] = self.state["model"] + self._emit(SESSION, self.root, "", session["start_ns"], self.now, rec, event, None) + session["exported"] = True + + def _close_children(self, turn: Optional[str], event: str) -> None: + """Export still-open spans under ``turn`` (or all) as incomplete.""" + for slot, rec in list(self.state["open"].items()): + if rec["scope"] == TURN: + continue + if turn is not None and rec.get("turn") != turn: + continue + del self.state["open"][slot] + rec["incomplete"] = True + sid = span_id(self.cfg.app, self.sid, rec["scope"], rec["key"]) + self._emit(rec["scope"], sid, rec["parent"], rec["start_ns"], self.now, rec, event, rec.get("turn")) + + def _end_turn(self, key: str, rec: Dict[str, Any], event: str) -> None: + self._close_children(key, event) + sid = span_id(self.cfg.app, self.sid, TURN, key) + self._emit(TURN, sid, self.root, rec["start_ns"], self.now, rec, event, key) + ended = self.state["ended_turns"] + ended.append(key) + del ended[:-MAX_ENDED_TURNS] + recs = self.state.setdefault("ended_recs", {}) + recs.pop(key, None) + if rec.get("incomplete"): + # Kept so a late END can re-send the turn complete; holds the turn's + # content only when content capture is on. + recs[key] = dict(rec) + while len(recs) > MAX_ENDED_RECS: + recs.pop(next(iter(recs))) + if self.state.get("current_turn") == key: + self.state["current_turn"] = None + + def _resolve_turn_key(self, ev: SpanEvent) -> Optional[str]: + """Key for a TURN event that has none (apps without turn ids). + + START mints the next per-session key (turn-1, turn-2, ...); END and + UPDATE apply to the currently open turn, if any. + """ + if ev.phase == START: + seq = int(self.state.get("turn_seq", 0)) + 1 + self.state["turn_seq"] = seq + return f"turn-{seq}" + return self.state.get("current_turn") + + def _oldest_open(self, scope: str, name: Optional[str]) -> Optional[str]: + """Key of the oldest open keyless START of ``scope`` with this name (FIFO pairing).""" + candidates = [rec for rec in self.state["open"].values() + if rec["scope"] == scope and rec.get("synthetic_key") and rec.get("name") == name] + if not candidates: + return None + return min(candidates, key=lambda rec: rec["start_ns"])["key"] + + # -- main entry + + def apply(self, ev: SpanEvent, event: str) -> None: + if ev.scope == TURN and not ev.key: + resolved = self._resolve_turn_key(ev) + if not resolved: + return + ev = dataclasses.replace(ev, key=resolved) + start_ns = self._start_of(ev) + session = self._ensure_session(start_ns) + if ev.model and ev.scope in (SESSION, TURN, LLM): + self.state["model"] = ev.model + parent, turn = self._parent_for(ev, start_ns) + # No turn id on the event: it was attached to the open turn. + inferred = ev.scope not in (SESSION, TURN) and not ev.turn_id and turn is not None + if turn: + turn_rec = self.state["open"].get(f"{TURN}:{turn}") + if turn_rec is not None and ev.model: + turn_rec["model"] = ev.model + + if ev.scope == SESSION: + self._apply_session(ev, event, session) + return + if ev.scope == TURN: + self._apply_turn(ev, event, start_ns) + return + + key = ev.key + synthetic = False + if not key and ev.phase == END: + key = self._oldest_open(ev.scope, ev.name) or "" + if not key: + key, synthetic = secrets.token_hex(8), True + slot = f"{ev.scope}:{key}" + if ev.phase == START: + rec = {"scope": ev.scope, "key": key, "start_ns": start_ns, "parent": parent, + "turn": turn, "attrs": {}, "synthetic_key": synthetic} + if inferred: + rec["attrs"]["coding_agent.turn.inferred"] = True + self._merge(rec, ev) + self.state["open"][slot] = rec + elif ev.phase == UPDATE: + rec = self.state["open"].get(slot) + if rec is not None: + self._merge(rec, ev) + else: # END / INSTANT + rec = self.state["open"].pop(slot, None) if ev.phase == END else None + if rec is None: + rec = {"scope": ev.scope, "key": key, "start_ns": start_ns, "parent": parent, + "turn": turn, "attrs": {}, "synthetic_key": synthetic} + if inferred: + rec["attrs"]["coding_agent.turn.inferred"] = True + self._merge(rec, ev) + sid = span_id(self.cfg.app, self.sid, ev.scope, key) + self._emit(ev.scope, sid, rec["parent"], rec["start_ns"], self.now, rec, event, rec.get("turn")) + + def _apply_session(self, ev: SpanEvent, event: str, session: Dict[str, Any]) -> None: + self._merge(session, ev) + if ev.phase in (END, INSTANT): + for slot, rec in list(self.state["open"].items()): + if rec["scope"] == TURN: + del self.state["open"][slot] + rec["incomplete"] = True + self._end_turn(rec["key"], rec, event) + self._close_children(None, event) + self.state["_delete"] = True + + def _apply_turn(self, ev: SpanEvent, event: str, start_ns: int) -> None: + key = ev.key + if not key: + return + slot = f"{TURN}:{key}" + if ev.phase == START: + # One turn at a time per session: a new turn closes any other open + # turn as incomplete (apps that sometimes skip their turn-end hook). + for other_slot, other in list(self.state["open"].items()): + if other["scope"] == TURN and other["key"] != key: + del self.state["open"][other_slot] + other["incomplete"] = True + self._end_turn(other["key"], other, event) + if key in self.state["ended_turns"]: + self.state["ended_turns"].remove(key) + rec = self._ensure_turn(key, start_ns) + if rec is not None: + self._merge(rec, ev) + self.state["current_turn"] = key + elif ev.phase == UPDATE: + rec = self._ensure_turn(key, start_ns) + if rec is not None: + self._merge(rec, ev) + else: + rec = self.state["open"].pop(slot, None) + if rec is None: + late = self.state.get("ended_recs", {}).pop(key, None) + if late is not None: + # The turn was closed as incomplete; re-send it complete with its original start. + rec = late + rec.pop("incomplete", None) + if key in self.state["ended_turns"]: + self.state["ended_turns"].remove(key) + elif key in self.state["ended_turns"]: + return # duplicate end of a turn already sent complete + else: + rec = {"scope": TURN, "key": key, "start_ns": start_ns, "attrs": {}, "parent": self.root} + self._merge(rec, ev) + self._end_turn(key, rec, event) + + def finish(self, event: str) -> None: + """Append the root session span to any batch that exported children. + + The root is re-sent with the same id and start time each time, so it + always spans the whole conversation. A session that never had any + activity (opened and closed without a message) exports nothing. + """ + session = self.state["session"] + if self.spans: + session["activity"] = True + if self.spans or (self.state.get("_delete") and session.get("activity")): + self._emit_root(event) + + +def otlp_attributes(attrs: Mapping[str, Any]) -> List[Dict[str, Any]]: + out = [] + for key, value in attrs.items(): + encoded = _otlp_value(value) + if encoded is not None: + out.append({"key": key, "value": encoded}) + return out + + +def _otlp_value(value: Any) -> Optional[Dict[str, Any]]: + if value is None: + return None + if isinstance(value, bool): + return {"boolValue": value} + if isinstance(value, int): + return {"intValue": str(value)} + if isinstance(value, float): + return {"doubleValue": value} + if isinstance(value, (list, tuple)): + values = [v for v in (_otlp_value(item) for item in value) if v is not None] + return {"arrayValue": {"values": values}} + return {"stringValue": str(value)} + + +# ---------------------------------------------------------------- pipeline + + +def load_adapter(app: str) -> Any: + if not re.fullmatch(r"[a-z0-9_]+", app): + raise ValueError("adapter name must match [a-z0-9_]+") + return importlib.import_module(f"adapters.{app}") + + +def resolve_event(adapter: Any, argv_event: Optional[str], payload: Dict[str, Any]) -> str: + resolver = getattr(adapter, "event_name", None) + if resolver is not None: + return resolver(argv_event, payload) or "" + return argv_event or str(payload.get("hook_event_name") or "") + + +def respond(adapter: Any, event: str, payload: Dict[str, Any]) -> str: + fn = getattr(adapter, "respond", None) + if fn is None: + return "{}" + try: + out = fn(event, payload) + except Exception: + # Print nothing, like a missing emitter: "{}" can be invalid output for + # a hook that decides something (Cursor blocks permission hooks on it). + return "" + # None means "use the default"; "" means "print nothing" (hosts that add + # hook stdout to the model context). + return "{}" if out is None else str(out) + + +def build_spans(adapter: Any, cfg: Config, event: str, payload: Dict[str, Any], + now_ns: Optional[int] = None) -> List[Dict[str, Any]]: + """Map one hook invocation to finished OTLP spans, updating state.""" + now_ns = time.time_ns() if now_ns is None else now_ns + events = [ev for ev in adapter.parse(event, payload) if ev.session_id] + by_session: Dict[str, List[SpanEvent]] = {} + for ev in events: + by_session.setdefault(str(ev.session_id), []).append(ev) + spans: List[Dict[str, Any]] = [] + for sid, evs in by_session.items(): + with session_state(cfg, sid) as state: + builder = SpanBuilder(cfg, adapter, state, sid, now_ns) + for ev in evs: + builder.apply(ev, event) + builder.finish(event) + spans.extend(builder.spans) + if any(ev.scope == SESSION for ev in evs): + sweep_state(cfg) + return spans + + +def build_body(adapter: Any, cfg: Config, event: str, payload: Dict[str, Any], + spans: List[Dict[str, Any]]) -> Dict[str, Any]: + resource = { + "service.name": getattr(adapter, "SERVICE_NAME", cfg.app), + "project_name": cfg.project_name, + "project_type": "observe", + "telemetry.sdk.name": SCOPE_NAME, + "telemetry.sdk.version": VERSION, + "coding_agent.app": cfg.app, + "coding_agent.content_captured": cfg.capture_content, + } + extra = getattr(adapter, "resource_attributes", None) + if extra is not None: + with contextlib.suppress(Exception): + for key, value in (extra(event, payload) or {}).items(): + resource.setdefault(key, value) + return {"resourceSpans": [{ + "resource": {"attributes": otlp_attributes(resource)}, + "scopeSpans": [{"scope": {"name": SCOPE_NAME, "version": VERSION}, "spans": spans}], + }]} + + +def post(cfg: Config, body: Dict[str, Any]) -> int: + """POST the OTLP/JSON body. Returns the HTTP status (raises on network error).""" + data = json.dumps(body, separators=(",", ":")).encode("utf-8") + request = urllib.request.Request(cfg.endpoint, data=data, method="POST", headers={ + "Content-Type": "application/json", + "X-Api-Key": cfg.api_key, + "X-Secret-Key": cfg.secret_key, + "User-Agent": f"{SCOPE_NAME}/{VERSION}", + }) + # Environment proxies only: the macOS system-proxy lookup is not fork-safe. + opener = urllib.request.build_opener(urllib.request.ProxyHandler(urllib.request.getproxies_environment())) + try: + with opener.open(request, timeout=cfg.timeout) as response: + return int(response.status) + except urllib.error.HTTPError as exc: + return int(exc.code) + + +def log(cfg: Config, message: str) -> None: + if not cfg.log_file: + return + with contextlib.suppress(OSError): + with open(cfg.log_file, "a", encoding="utf-8") as handle: + handle.write(f"{time.strftime('%Y-%m-%dT%H:%M:%S')} {cfg.app} {message}\n") + + +def deliver(cfg: Config, body: Dict[str, Any], label: str) -> None: + """Send in a detached child (default) or inline (FI_HOOK_SYNC=1).""" + if cfg.dump_file: + with contextlib.suppress(OSError): + with open(cfg.dump_file, "a", encoding="utf-8") as handle: + handle.write(json.dumps(body, separators=(",", ":")) + "\n") + if not cfg.sync and hasattr(os, "fork"): + try: + pid = os.fork() + except OSError: + pid = -1 + if pid > 0: + return + if pid == 0: + try: + os.setsid() + devnull = os.open(os.devnull, os.O_RDWR) + for fd in (0, 1, 2): + os.dup2(devnull, fd) + _send_logged(cfg, body, label) + finally: + os._exit(0) + _send_logged(cfg, body, label) + + +def _send_logged(cfg: Config, body: Dict[str, Any], label: str) -> None: + try: + status = post(cfg, body) + log(cfg, f"{label} http={status}") + except Exception as exc: # never surface payloads or headers + log(cfg, f"{label} error={type(exc).__name__}") + + +def load_env_file(path: Optional[str], env: Dict[str, str]) -> None: + """Fill FI_* keys missing from ``env`` from a KEY=VALUE file.""" + candidate = Path(path).expanduser() if path else DEFAULT_ENV_FILE.expanduser() + try: + lines = candidate.read_text(encoding="utf-8").splitlines() + except OSError: + return + for line in lines: + line = line.strip() + if not line or line.startswith("#") or "=" not in line: + continue + key, value = line.split("=", 1) + key = key.strip() + if key.startswith("export "): + key = key[len("export "):].strip() + if key.startswith("FI_") and key not in env: + env[key] = value.strip().strip("'\"") + + +def parse_args(argv: List[str]) -> Tuple[str, Optional[str], Optional[str]]: + app, event, env_file = "", None, None + rest = list(argv) + while rest: + arg = rest.pop(0) + if arg == "--env-file" and rest: + env_file = rest.pop(0) + elif arg.startswith("--env-file="): + env_file = arg.split("=", 1)[1] + elif not app: + app = arg + elif event is None: + event = arg + return app, event, env_file + + +def run(argv: List[str], stdin_text: str, env: Mapping[str, str]) -> str: + """Process one hook call. Returns stdout text; never raises.""" + app, argv_event, env_file = parse_args(argv) + env = dict(env) + load_env_file(env_file or env.get("FI_HOOK_ENV_FILE"), env) + try: + adapter = load_adapter(app) + except Exception: + # Without the adapter we cannot tell an observation hook from a + # permission hook, so print nothing (same as a missing emitter). + return "" + try: + payload = json.loads(stdin_text) if stdin_text.strip() else {} + except ValueError: + payload = {} + if not isinstance(payload, dict): + payload = {} + event = "" + try: + event = resolve_event(adapter, argv_event, payload) + except Exception: + pass + out = respond(adapter, event, payload) + try: + cfg = Config.from_env(app, env) + if not cfg.can_export: + return out + spans = build_spans(adapter, cfg, event, payload) + if spans: + body = build_body(adapter, cfg, event, payload, spans) + _write_stdout(out) + deliver(cfg, body, f"event={event} spans={len(spans)}") + return "" + except Exception as exc: + with contextlib.suppress(Exception): + log(Config.from_env(app, env), f"event={event} failed={type(exc).__name__}") + return out + + +def _write_stdout(text: str) -> None: + if text: + sys.stdout.write(text if text.endswith("\n") else text + "\n") + sys.stdout.flush() + + +def main() -> int: + try: + stdin_text = sys.stdin.read() + except Exception: + stdin_text = "" + try: + out = run(sys.argv[1:], stdin_text, os.environ) + except Exception: + out = "" + with contextlib.suppress(Exception): + _write_stdout(out) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/examples/coding-agents/hook-to-otlp/replay.py b/examples/coding-agents/hook-to-otlp/replay.py new file mode 100755 index 000000000..0bfe4bc2d --- /dev/null +++ b/examples/coding-agents/hook-to-otlp/replay.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python3 +"""Replay recorded hook payloads through emit.py, one subprocess per hook. + +Each line of the JSONL file is ``{"event": "", "payload": {...}}``. +The emitter runs exactly as a host app would run it (stdin payload, stdout +response), with the current environment. + + python3 replay.py cursor tests/fixtures/cursor/session.jsonl \ + --replace fixture-conv-0001=my-run-123 --sleep 0.2 + +``--replace OLD=NEW`` rewrites every occurrence of OLD inside string values +(use it to give each run a fresh session id; it also rewrites ids that carry +a suffix, such as Cursor's ``-0-ab12``). Prints one line per +hook: the event and the stdout the host would receive. + +``--hooks-json PATH`` runs the commands registered in an installed hooks file +instead (every ``"command"`` under ``hooks.``), through ``$SHELL -c`` +in ``--cwd``, with the payload on stdin or, with ``--transport heredoc``, +appended as `` <<'CURSOR_HOOK_EOF'`` the way Cursor's headless agent +host passes it. Events with no registered command are skipped. +""" +from __future__ import annotations + +import argparse +import json +import os +import subprocess +import sys +import time +from pathlib import Path +from typing import Any, Dict, List + +EMIT = Path(__file__).resolve().parent / "emit.py" + + +def rewrite(value: Any, mapping: Dict[str, str]) -> Any: + if isinstance(value, str): + for old, new in mapping.items(): + value = value.replace(old, new) + return value + if isinstance(value, list): + return [rewrite(v, mapping) for v in value] + if isinstance(value, dict): + return {k: rewrite(v, mapping) for k, v in value.items()} + return value + + +def registered_commands(node: Any) -> List[str]: + """Every "command" string under a hooks-file entry (flat or nested matcher lists).""" + if isinstance(node, dict): + found = [node["command"]] if isinstance(node.get("command"), str) else [] + return found + [c for k, v in node.items() if k != "command" for c in registered_commands(v)] + if isinstance(node, list): + return [c for item in node for c in registered_commands(item)] + return [] + + +def run_command(command: str, text: str, args: argparse.Namespace) -> subprocess.CompletedProcess: + if args.transport == "heredoc": + script = f"{command} <<'CURSOR_HOOK_EOF'\n{text}\nCURSOR_HOOK_EOF" + return subprocess.run([args.shell, "-c", script], cwd=args.cwd, stdin=subprocess.DEVNULL, + capture_output=True, text=True, timeout=30) + return subprocess.run([args.shell, "-c", command], cwd=args.cwd, input=text, + capture_output=True, text=True, timeout=30) + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("app") + ap.add_argument("jsonl") + ap.add_argument("--replace", action="append", default=[], metavar="OLD=NEW") + ap.add_argument("--sleep", type=float, default=0.0, help="seconds between hooks") + ap.add_argument("--pass-event", action="store_true", help="also pass the event name as argv") + ap.add_argument("--hooks-json", help="run the commands registered in this hooks file") + ap.add_argument("--cwd", help="working directory for --hooks-json commands") + ap.add_argument("--transport", choices=("stdin", "heredoc"), default="stdin", + help="how --hooks-json commands get the payload") + ap.add_argument("--shell", default=os.environ.get("SHELL") or "/bin/sh", + help="shell for --hooks-json commands (default $SHELL)") + args = ap.parse_args() + mapping = dict(item.split("=", 1) for item in args.replace) + hooks = json.loads(Path(args.hooks_json).read_text(encoding="utf-8"))["hooks"] if args.hooks_json else None + for line in Path(args.jsonl).read_text(encoding="utf-8").splitlines(): + if not line.strip(): + continue + record = json.loads(line) + event = record["event"] + payload = rewrite(record["payload"], mapping) + text = json.dumps(payload) + started = time.monotonic() + if hooks is not None: + cmds = registered_commands(hooks.get(event)) + if not cmds: + print(f"{event:22s} not registered, skipped") + continue + results = [run_command(cmd, text, args) for cmd in cmds] + else: + argv = [sys.executable, "-B", str(EMIT), args.app] + ([event] if args.pass_event else []) + results = [subprocess.run(argv, input=text, capture_output=True, text=True, timeout=30)] + took = (time.monotonic() - started) * 1000 + for result in results: + print(f"{event:22s} exit={result.returncode} {took:6.0f}ms stdout={result.stdout.strip()}") + if args.sleep: + time.sleep(args.sleep) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/examples/coding-agents/hook-to-otlp/tests/__init__.py b/examples/coding-agents/hook-to-otlp/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/examples/coding-agents/hook-to-otlp/tests/fixtures/cursor/real-3.24.18-turn.jsonl b/examples/coding-agents/hook-to-otlp/tests/fixtures/cursor/real-3.24.18-turn.jsonl new file mode 100644 index 000000000..2d007ed20 --- /dev/null +++ b/examples/coding-agents/hook-to-otlp/tests/fixtures/cursor/real-3.24.18-turn.jsonl @@ -0,0 +1,9 @@ +{"event": "beforeSubmitPrompt", "payload": {"conversation_id": "c590ab42-038b-4b93-97d1-4552af1df766", "generation_id": "d16f0adc-7404-4b88-adcd-e98f77fa8655", "model": "cursor-grok-4.6-medium", "model_id": "grok-4.6", "model_params": [{"id": "effort", "value": "medium"}, {"id": "fast", "value": "false"}], "composer_mode": "agent", "prompt": "List the files in this folder and read README.md PROMPT_MARKER_REAL", "attachments": [], "session_id": "c590ab42-038b-4b93-97d1-4552af1df766", "hook_event_name": "beforeSubmitPrompt", "cursor_version": "3.24.18", "workspace_roots": ["/Users/fixture/project"], "user_email": "fixture.user@example.com", "transcript_path": null}, "source": "Cursor 3.24.18 IDE, Hobby plan, 2026-10-14; email, paths and text replaced"} +{"event": "afterAgentThought", "payload": {"conversation_id": "c590ab42-038b-4b93-97d1-4552af1df766", "generation_id": "d16f0adc-7404-4b88-adcd-e98f77fa8655-0-phr2", "model": "cursor-grok-4.6-medium", "model_id": "grok-4.6", "model_params": [{"id": "effort", "value": "medium"}, {"id": "speed", "value": "default"}], "text": "thinking about the files THOUGHT_MARKER_REAL", "duration_ms": 2, "session_id": "c590ab42-038b-4b93-97d1-4552af1df766", "hook_event_name": "afterAgentThought", "cursor_version": "3.24.18", "workspace_roots": ["/Users/fixture/project"], "user_email": "fixture.user@example.com", "transcript_path": null}, "source": "Cursor 3.24.18 IDE, Hobby plan, 2026-10-14; email, paths and text replaced"} +{"event": "afterAgentThought", "payload": {"conversation_id": "c590ab42-038b-4b93-97d1-4552af1df766", "generation_id": "d16f0adc-7404-4b88-adcd-e98f77fa8655", "text": "thinking about the files THOUGHT_MARKER_REAL", "duration_ms": 2, "model": "cursor-grok-4.6-medium", "model_id": "grok-4.6", "model_params": [{"id": "effort", "value": "medium"}, {"id": "fast", "value": "false"}], "session_id": "c590ab42-038b-4b93-97d1-4552af1df766", "hook_event_name": "afterAgentThought", "cursor_version": "3.24.18", "workspace_roots": ["/Users/fixture/project"], "user_email": "fixture.user@example.com", "transcript_path": null}, "source": "Cursor 3.24.18 IDE, Hobby plan, 2026-10-14; email, paths and text replaced"} +{"event": "postToolUse", "payload": {"conversation_id": "c590ab42-038b-4b93-97d1-4552af1df766", "generation_id": "d16f0adc-7404-4b88-adcd-e98f77fa8655", "model": "grok-4.6", "tool_name": "Read", "tool_input": {"file_path": "/Users/fixture/project/README.md"}, "tool_output": "{\"file_path\": \"/Users/fixture/project/README.md\", \"content\": \"# Fixture TOOL_OUTPUT_MARKER_REAL\"}", "duration": 483.733, "tool_use_id": "call-f3e41504-16cd-4345-9614-dccb061cbdd4-1\nfc_cdbfcc58-3e58-9ca2-92f0-366509226caf_1", "session_id": "c590ab42-038b-4b93-97d1-4552af1df766", "hook_event_name": "postToolUse", "cursor_version": "3.24.18", "workspace_roots": ["/Users/fixture/project"], "user_email": "fixture.user@example.com", "transcript_path": null}, "source": "Cursor 3.24.18 IDE, Hobby plan, 2026-10-14; email, paths and text replaced"} +{"event": "postToolUse", "payload": {"conversation_id": "c590ab42-038b-4b93-97d1-4552af1df766", "generation_id": "d16f0adc-7404-4b88-adcd-e98f77fa8655", "model": "grok-4.6", "tool_name": "Grep", "tool_input": {"pattern": "", "file_path": "/Users/fixture/project", "glob": "**/*", "output_mode": "files_with_matches"}, "tool_output": "{\"pattern\":\"\",\"success\":true}", "duration": 574.717, "tool_use_id": "call-f3e41504-16cd-4345-9614-dccb061cbdd4-0\nfc_cdbfcc58-3e58-9ca2-92f0-366509226caf_0", "session_id": "c590ab42-038b-4b93-97d1-4552af1df766", "hook_event_name": "postToolUse", "cursor_version": "3.24.18", "workspace_roots": ["/Users/fixture/project"], "user_email": "fixture.user@example.com", "transcript_path": null}, "source": "Cursor 3.24.18 IDE, Hobby plan, 2026-10-14; email, paths and text replaced"} +{"event": "afterAgentThought", "payload": {"conversation_id": "c590ab42-038b-4b93-97d1-4552af1df766", "generation_id": "d16f0adc-7404-4b88-adcd-e98f77fa8655", "text": "thinking about the files THOUGHT_MARKER_REAL", "duration_ms": 1047, "model": "cursor-grok-4.6-medium", "model_id": "grok-4.6", "model_params": [{"id": "effort", "value": "medium"}, {"id": "fast", "value": "false"}], "session_id": "c590ab42-038b-4b93-97d1-4552af1df766", "hook_event_name": "afterAgentThought", "cursor_version": "3.24.18", "workspace_roots": ["/Users/fixture/project"], "user_email": "fixture.user@example.com", "transcript_path": null}, "source": "Cursor 3.24.18 IDE, Hobby plan, 2026-10-14; email, paths and text replaced"} +{"event": "afterAgentThought", "payload": {"conversation_id": "c590ab42-038b-4b93-97d1-4552af1df766", "generation_id": "d16f0adc-7404-4b88-adcd-e98f77fa8655-1-ty2w", "model": "cursor-grok-4.6-medium", "model_id": "grok-4.6", "model_params": [{"id": "effort", "value": "medium"}, {"id": "speed", "value": "default"}], "text": "thinking about the files THOUGHT_MARKER_REAL", "duration_ms": 1047, "session_id": "c590ab42-038b-4b93-97d1-4552af1df766", "hook_event_name": "afterAgentThought", "cursor_version": "3.24.18", "workspace_roots": ["/Users/fixture/project"], "user_email": "fixture.user@example.com", "transcript_path": null}, "source": "Cursor 3.24.18 IDE, Hobby plan, 2026-10-14; email, paths and text replaced"} +{"event": "afterAgentResponse", "payload": {"conversation_id": "c590ab42-038b-4b93-97d1-4552af1df766", "generation_id": "d16f0adc-7404-4b88-adcd-e98f77fa8655", "model": "cursor-grok-4.6-medium", "model_id": "grok-4.6", "model_params": [{"id": "effort", "value": "medium"}, {"id": "fast", "value": "false"}], "text": "This is a throwaway project RESPONSE_MARKER_REAL", "input_tokens": 38779, "output_tokens": 306, "cache_read_tokens": 24192, "cache_write_tokens": 0, "session_id": "c590ab42-038b-4b93-97d1-4552af1df766", "hook_event_name": "afterAgentResponse", "cursor_version": "3.24.18", "workspace_roots": ["/Users/fixture/project"], "user_email": "fixture.user@example.com", "transcript_path": "/tmp/fixture-transcript-CANARY.txt"}, "source": "Cursor 3.24.18 IDE, Hobby plan, 2026-10-14; email, paths and text replaced"} +{"event": "stop", "payload": {"conversation_id": "c590ab42-038b-4b93-97d1-4552af1df766", "generation_id": "d16f0adc-7404-4b88-adcd-e98f77fa8655", "model": "cursor-grok-4.6-medium", "model_id": "grok-4.6", "model_params": [{"id": "effort", "value": "medium"}, {"id": "fast", "value": "false"}], "status": "completed", "loop_count": 0, "input_tokens": 38779, "output_tokens": 306, "cache_read_tokens": 24192, "cache_write_tokens": 0, "session_id": "c590ab42-038b-4b93-97d1-4552af1df766", "hook_event_name": "stop", "cursor_version": "3.24.18", "workspace_roots": ["/Users/fixture/project"], "user_email": "fixture.user@example.com", "transcript_path": "/tmp/fixture-transcript-CANARY.txt"}, "source": "Cursor 3.24.18 IDE, Hobby plan, 2026-10-14; email, paths and text replaced"} diff --git a/examples/coding-agents/hook-to-otlp/tests/fixtures/cursor/real-3.24.18.jsonl b/examples/coding-agents/hook-to-otlp/tests/fixtures/cursor/real-3.24.18.jsonl new file mode 100644 index 000000000..30b26c47a --- /dev/null +++ b/examples/coding-agents/hook-to-otlp/tests/fixtures/cursor/real-3.24.18.jsonl @@ -0,0 +1 @@ +{"event": "sessionEnd", "payload": {"conversation_id": "99967ca3-7f9d-4fce-b67d-fe831e95efa0", "generation_id": "", "model": "default", "model_id": "default", "reason": "window_close", "duration_ms": 0, "is_background_agent": false, "final_status": "none", "session_id": "99967ca3-7f9d-4fce-b67d-fe831e95efa0", "hook_event_name": "sessionEnd", "cursor_version": "3.24.18", "workspace_roots": ["/Users/fixture/project"], "user_email": null, "transcript_path": null}, "source": "Cursor 3.24.18 hooks output channel, 2026-10-14, workspace path replaced"} diff --git a/examples/coding-agents/hook-to-otlp/tests/fixtures/cursor/session.jsonl b/examples/coding-agents/hook-to-otlp/tests/fixtures/cursor/session.jsonl new file mode 100644 index 000000000..7ac8d4f08 --- /dev/null +++ b/examples/coding-agents/hook-to-otlp/tests/fixtures/cursor/session.jsonl @@ -0,0 +1,11 @@ +{"event": "sessionStart", "payload": {"conversation_id": "fixture-conv-0001", "generation_id": "fixture-gen-0001", "model": "claude-opus-4-7-thinking-max", "model_id": "claude-opus-4-7", "model_params": [{"id": "thinking", "value": "true"}, {"id": "effort", "value": "max"}], "cursor_version": "3.24.18", "workspace_roots": ["/Users/fixture/project"], "user_email": "fixture.user@example.com", "transcript_path": "/tmp/fixture-transcript-CANARY.txt", "hook_event_name": "sessionStart", "session_id": "fixture-conv-0001", "is_background_agent": false, "composer_mode": "agent"}} +{"event": "beforeSubmitPrompt", "payload": {"conversation_id": "fixture-conv-0001", "generation_id": "fixture-gen-0001", "model": "claude-opus-4-7-thinking-max", "model_id": "claude-opus-4-7", "model_params": [{"id": "thinking", "value": "true"}, {"id": "effort", "value": "max"}], "cursor_version": "3.24.18", "workspace_roots": ["/Users/fixture/project"], "user_email": "fixture.user@example.com", "transcript_path": "/tmp/fixture-transcript-CANARY.txt", "hook_event_name": "beforeSubmitPrompt", "prompt": "List the files here and read README.md PROMPT_MARKER_CURSOR", "attachments": [{"type": "file", "file_path": "/Users/fixture/project/README.md"}]}} +{"event": "afterAgentThought", "payload": {"conversation_id": "fixture-conv-0001", "generation_id": "fixture-gen-0001", "model": "claude-opus-4-7-thinking-max", "model_id": "claude-opus-4-7", "model_params": [{"id": "thinking", "value": "true"}, {"id": "effort", "value": "max"}], "cursor_version": "3.24.18", "workspace_roots": ["/Users/fixture/project"], "user_email": "fixture.user@example.com", "transcript_path": "/tmp/fixture-transcript-CANARY.txt", "hook_event_name": "afterAgentThought", "text": "I should list files THOUGHT_MARKER_CURSOR", "duration_ms": 1200}} +{"event": "postToolUse", "payload": {"conversation_id": "fixture-conv-0001", "generation_id": "fixture-gen-0001", "model": "claude-opus-4-7-thinking-max", "model_id": "claude-opus-4-7", "model_params": [{"id": "thinking", "value": "true"}, {"id": "effort", "value": "max"}], "cursor_version": "3.24.18", "workspace_roots": ["/Users/fixture/project"], "user_email": "fixture.user@example.com", "transcript_path": "/tmp/fixture-transcript-CANARY.txt", "hook_event_name": "postToolUse", "tool_name": "Shell", "tool_input": {"command": "ls -la TOOL_INPUT_MARKER_CURSOR", "working_directory": "/Users/fixture/project"}, "tool_output": "{\"exitCode\":0,\"stdout\":\"README.md TOOL_OUTPUT_MARKER_CURSOR\"}", "tool_use_id": "toolu_fixture_1", "cwd": "/Users/fixture/project", "duration": 420}} +{"event": "postToolUse", "payload": {"conversation_id": "fixture-conv-0001", "generation_id": "fixture-gen-0001", "model": "claude-opus-4-7-thinking-max", "model_id": "claude-opus-4-7", "model_params": [{"id": "thinking", "value": "true"}, {"id": "effort", "value": "max"}], "cursor_version": "3.24.18", "workspace_roots": ["/Users/fixture/project"], "user_email": "fixture.user@example.com", "transcript_path": "/tmp/fixture-transcript-CANARY.txt", "hook_event_name": "postToolUse", "tool_name": "Read", "tool_input": {"path": "README.md"}, "tool_output": "{\"content\":\"# Fixture TOOL_OUTPUT_MARKER_CURSOR\"}", "tool_use_id": "toolu_fixture_2", "cwd": "/Users/fixture/project", "duration": 35}} +{"event": "postToolUseFailure", "payload": {"conversation_id": "fixture-conv-0001", "generation_id": "fixture-gen-0001", "model": "claude-opus-4-7-thinking-max", "model_id": "claude-opus-4-7", "model_params": [{"id": "thinking", "value": "true"}, {"id": "effort", "value": "max"}], "cursor_version": "3.24.18", "workspace_roots": ["/Users/fixture/project"], "user_email": "fixture.user@example.com", "transcript_path": "/tmp/fixture-transcript-CANARY.txt", "hook_event_name": "postToolUseFailure", "tool_name": "Shell", "tool_input": {"command": "npm test"}, "tool_use_id": "toolu_fixture_3", "cwd": "/Users/fixture/project", "error_message": "Command timed out after 30s ERROR_MARKER_CURSOR", "failure_type": "timeout", "duration": 5000, "is_interrupt": false}} +{"event": "subagentStop", "payload": {"conversation_id": "fixture-conv-0001", "generation_id": "fixture-gen-0001", "model": "claude-opus-4-7-thinking-max", "model_id": "claude-opus-4-7", "model_params": [{"id": "thinking", "value": "true"}, {"id": "effort", "value": "max"}], "cursor_version": "3.24.18", "workspace_roots": ["/Users/fixture/project"], "user_email": "fixture.user@example.com", "transcript_path": "/tmp/fixture-transcript-CANARY.txt", "hook_event_name": "subagentStop", "subagent_type": "explore", "status": "completed", "task": "Explore the docs SUBAGENT_MARKER_CURSOR", "description": "Exploring docs", "summary": "Found 2 files SUBAGENT_MARKER_CURSOR", "duration_ms": 4000, "message_count": 6, "tool_call_count": 3, "loop_count": 0, "modified_files": [], "agent_transcript_path": "/tmp/fixture-subagent-transcript.txt"}} +{"event": "preCompact", "payload": {"conversation_id": "fixture-conv-0001", "generation_id": "fixture-gen-0001", "model": "claude-opus-4-7-thinking-max", "model_id": "claude-opus-4-7", "model_params": [{"id": "thinking", "value": "true"}, {"id": "effort", "value": "max"}], "cursor_version": "3.24.18", "workspace_roots": ["/Users/fixture/project"], "user_email": "fixture.user@example.com", "transcript_path": "/tmp/fixture-transcript-CANARY.txt", "hook_event_name": "preCompact", "trigger": "auto", "context_usage_percent": 85, "context_tokens": 120000, "context_window_size": 128000, "message_count": 45, "messages_to_compact": 30, "is_first_compaction": true}} +{"event": "afterAgentResponse", "payload": {"conversation_id": "fixture-conv-0001", "generation_id": "fixture-gen-0001", "model": "claude-opus-4-7-thinking-max", "model_id": "claude-opus-4-7", "model_params": [{"id": "thinking", "value": "true"}, {"id": "effort", "value": "max"}], "cursor_version": "3.24.18", "workspace_roots": ["/Users/fixture/project"], "user_email": "fixture.user@example.com", "transcript_path": "/tmp/fixture-transcript-CANARY.txt", "hook_event_name": "afterAgentResponse", "text": "README.md says hello RESPONSE_MARKER_CURSOR"}} +{"event": "stop", "payload": {"conversation_id": "fixture-conv-0001", "generation_id": "fixture-gen-0001", "model": "claude-opus-4-7-thinking-max", "model_id": "claude-opus-4-7", "model_params": [{"id": "thinking", "value": "true"}, {"id": "effort", "value": "max"}], "cursor_version": "3.24.18", "workspace_roots": ["/Users/fixture/project"], "user_email": "fixture.user@example.com", "transcript_path": "/tmp/fixture-transcript-CANARY.txt", "hook_event_name": "stop", "status": "completed", "loop_count": 0}} +{"event": "sessionEnd", "payload": {"conversation_id": "fixture-conv-0001", "generation_id": "fixture-gen-0001", "model": "claude-opus-4-7-thinking-max", "model_id": "claude-opus-4-7", "model_params": [{"id": "thinking", "value": "true"}, {"id": "effort", "value": "max"}], "cursor_version": "3.24.18", "workspace_roots": ["/Users/fixture/project"], "user_email": "fixture.user@example.com", "transcript_path": "/tmp/fixture-transcript-CANARY.txt", "hook_event_name": "sessionEnd", "session_id": "fixture-conv-0001", "reason": "user_close", "duration_ms": 45000, "is_background_agent": false, "final_status": "completed"}} diff --git a/examples/coding-agents/hook-to-otlp/tests/helpers.py b/examples/coding-agents/hook-to-otlp/tests/helpers.py new file mode 100644 index 000000000..010dca144 --- /dev/null +++ b/examples/coding-agents/hook-to-otlp/tests/helpers.py @@ -0,0 +1,122 @@ +"""Shared helpers for emitter and adapter tests (stdlib only).""" +from __future__ import annotations + +import json +import os +import subprocess +import sys +import tempfile +import threading +import time +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from typing import Any, Dict, Iterable, List, Optional, Tuple + +ROOT = Path(__file__).resolve().parents[1] +EMIT = ROOT / "emit.py" +FIXTURES = Path(__file__).resolve().parent / "fixtures" +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +import emit # noqa: E402 + + +class Receiver: + """Loopback OTLP/HTTP receiver that records (path, headers, body).""" + + def __init__(self, status: int = 200): + self.requests: List[Tuple[str, Dict[str, str], Dict[str, Any]]] = [] + outer = self + + class Handler(BaseHTTPRequestHandler): + def do_POST(self): # noqa: N802 + size = int(self.headers.get("Content-Length", "0")) + body = json.loads(self.rfile.read(size) or b"{}") + outer.requests.append((self.path, dict(self.headers), body)) + self.send_response(status) + self.end_headers() + self.wfile.write(b"{}") + + def log_message(self, *args): # silence + return + + self.server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + self.port = self.server.server_address[1] + self.url = f"http://127.0.0.1:{self.port}" + threading.Thread(target=self.server.serve_forever, daemon=True).start() + + def wait_for(self, count: int, timeout: float = 10.0) -> None: + deadline = time.time() + timeout + while len(self.requests) < count and time.time() < deadline: + time.sleep(0.02) + + def spans(self) -> List[Dict[str, Any]]: + return [s for _, _, body in self.requests for s in all_spans(body)] + + def close(self) -> None: + self.server.shutdown() + self.server.server_close() + + +def all_spans(body: Dict[str, Any]) -> List[Dict[str, Any]]: + return [s for rs in body.get("resourceSpans", []) for ss in rs.get("scopeSpans", []) for s in ss.get("spans", [])] + + +def attrs(span_or_resource: Dict[str, Any]) -> Dict[str, Any]: + out: Dict[str, Any] = {} + for item in span_or_resource.get("attributes", []): + value = item["value"] + out[item["key"]] = next(iter(value.values())) + return out + + +def base_env(state_dir: str, endpoint: str = "http://127.0.0.1:9/v1/traces", **extra: str) -> Dict[str, str]: + env = { + "FI_API_KEY": "test-api-key", + "FI_SECRET_KEY": "test-secret-key", + "FI_PROJECT_NAME": "hook-test", + "FI_OTLP_ENDPOINT": endpoint, + "FI_HOOK_STATE_DIR": state_dir, + "FI_HOOK_SYNC": "1", + "FI_HOOK_TIMEOUT": "2", + } + env.update(extra) + return env + + +def config(app: str, state_dir: str, **extra: str) -> "emit.Config": + return emit.Config.from_env(app, base_env(state_dir, **extra)) + + +def load_jsonl(path: Path) -> List[Dict[str, Any]]: + return [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines() if line.strip()] + + +def replay(adapter: Any, cfg: "emit.Config", records: Iterable[Dict[str, Any]], + start_ns: int = 1_800_000_000_000_000_000, step_ms: int = 1000) -> List[Dict[str, Any]]: + """Run records through build_spans in-process with a fake clock.""" + spans: List[Dict[str, Any]] = [] + now = start_ns + for record in records: + now += step_ms * 1_000_000 + event = emit.resolve_event(adapter, record.get("argv_event"), record["payload"]) or record["event"] + spans.extend(emit.build_spans(adapter, cfg, event, record["payload"], now_ns=now)) + return spans + + +def run_cli(args: List[str], payload: Any, env: Dict[str, str], timeout: float = 20) -> subprocess.CompletedProcess: + full_env = {"PATH": os.environ.get("PATH", ""), "HOME": tempfile.gettempdir()} + full_env.update(env) + text = payload if isinstance(payload, str) else json.dumps(payload) + return subprocess.run([sys.executable, str(EMIT)] + args, input=text, capture_output=True, + text=True, env=full_env, timeout=timeout) + + +def by_name(spans: List[Dict[str, Any]], name: str) -> List[Dict[str, Any]]: + return [s for s in spans if s["name"] == name] + + +def latest(spans: List[Dict[str, Any]], span_id: str) -> Optional[Dict[str, Any]]: + """Last export of a span id (the root is re-exported as the session grows).""" + found = [s for s in spans if s["spanId"] == span_id] + return found[-1] if found else None diff --git a/examples/coding-agents/hook-to-otlp/tests/test_core.py b/examples/coding-agents/hook-to-otlp/tests/test_core.py new file mode 100644 index 000000000..ef2486c4d --- /dev/null +++ b/examples/coding-agents/hook-to-otlp/tests/test_core.py @@ -0,0 +1,519 @@ +"""Emitter core tests: ids, pairing, parenting, privacy, transport, failure modes.""" +from __future__ import annotations + +import json +import os +import shutil +import subprocess +import sys +import tempfile +import time +import types +import unittest +from pathlib import Path +from unittest import mock + +from tests.helpers import Receiver, all_spans, attrs, base_env, config, latest, run_cli + +import emit +from adapters.base import END, ERROR, INSTANT, LLM, OK, SESSION, START, TOOL, TURN, UPDATE, SpanEvent + +T0 = 1_800_000_000_000_000_000 +MS = 1_000_000 + + +def fake_adapter(handler): + module = types.ModuleType("adapters.fake") + module.APP = "fake" + module.DISPLAY_NAME = "Fake" + module.parse = handler + return module + + +def scripted(events_by_hook): + """Adapter whose parse() returns a fixed list of SpanEvents per hook name.""" + return fake_adapter(lambda event, payload: list(events_by_hook.get(event, []))) + + +class CoreTest(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.cfg = config("fake", self.tmp.name) + + def tearDown(self): + self.tmp.cleanup() + + def run_hooks(self, adapter, hooks, cfg=None): + cfg = cfg or self.cfg + spans = [] + for offset_ms, hook in hooks: + spans.extend(emit.build_spans(adapter, cfg, hook, {}, now_ns=T0 + offset_ms * MS)) + return spans + + def test_trace_and_span_ids_are_deterministic(self): + self.assertEqual(emit.trace_id("cursor", "s1"), emit.trace_id("cursor", "s1")) + self.assertNotEqual(emit.trace_id("cursor", "s1"), emit.trace_id("cursor", "s2")) + self.assertNotEqual(emit.trace_id("cursor", "s1"), emit.trace_id("codex", "s1")) + self.assertEqual(len(emit.trace_id("a", "b")), 32) + self.assertEqual(len(emit.span_id("a", "b", TOOL, "k")), 16) + + def test_session_turn_tool_hierarchy_and_pairing(self): + adapter = scripted({ + "s": [SpanEvent(START, SESSION, "S")], + "t0": [SpanEvent(START, TURN, "S", key="g1", model="m-1")], + "pre": [SpanEvent(START, TOOL, "S", key="call-1", turn_id="g1", name="Shell")], + "post": [SpanEvent(END, TOOL, "S", key="call-1", turn_id="g1", status=OK)], + "t1": [SpanEvent(END, TURN, "S", key="g1", status=OK)], + "e": [SpanEvent(END, SESSION, "S")], + }) + spans = self.run_hooks(adapter, [(0, "s"), (100, "t0"), (200, "pre"), (700, "post"), (900, "t1"), (1000, "e")]) + trace = emit.trace_id("fake", "S") + self.assertTrue(all(s["traceId"] == trace for s in spans)) + root_id = emit.span_id("fake", "S", SESSION, "") + turn_id = emit.span_id("fake", "S", TURN, "g1") + tool = [s for s in spans if s["name"] == "Shell"][0] + self.assertEqual(tool["parentSpanId"], turn_id) + self.assertEqual(int(tool["endTimeUnixNano"]) - int(tool["startTimeUnixNano"]), 500 * MS) + self.assertEqual(attrs(tool)["fi.span.kind"], "TOOL") + self.assertEqual(attrs(tool)["gen_ai.tool.call.id"], "call-1") + turn = [s for s in spans if s["spanId"] == turn_id][0] + self.assertEqual(turn["parentSpanId"], root_id) + self.assertEqual(int(turn["endTimeUnixNano"]) - int(turn["startTimeUnixNano"]), 800 * MS) + self.assertEqual(attrs(turn)["llm.model_name"], "m-1") + root = latest(spans, root_id) + self.assertNotIn("parentSpanId", root) + self.assertEqual(int(root["endTimeUnixNano"]) - int(root["startTimeUnixNano"]), 1000 * MS) + self.assertEqual(attrs(root)["session.id"], "S") + # Every root export reuses the same start, so the backend upserts it. + starts = {s["startTimeUnixNano"] for s in spans if s["spanId"] == root_id} + self.assertEqual(len(starts), 1) + # Session end removes the state file. + self.assertEqual(list(self.cfg.state_dir.glob("*.json")), []) + + def test_root_rides_along_with_every_child_export(self): + adapter = scripted({ + "s": [SpanEvent(START, SESSION, "S")], + "tool": [SpanEvent(END, TOOL, "S", key="c1", name="Shell")], + }) + self.assertEqual(self.run_hooks(adapter, [(0, "s")]), [], "session start alone exports nothing") + spans = self.run_hooks(adapter, [(500, "tool")]) + root_id = emit.span_id("fake", "S", SESSION, "") + self.assertEqual([s["spanId"] for s in spans][-1], root_id) + self.assertEqual(int(spans[-1]["startTimeUnixNano"]), T0) + + def test_session_without_activity_exports_nothing(self): + adapter = scripted({ + "s": [SpanEvent(START, SESSION, "S")], + "e": [SpanEvent(END, SESSION, "S", duration_ms=0)], + }) + self.assertEqual(self.run_hooks(adapter, [(0, "s"), (10, "e")]), []) + self.assertEqual(list(self.cfg.state_dir.glob("*.json")), []) + + def test_keyless_turns_get_per_session_keys(self): + # Apps without turn ids (Kiro, Antigravity): START mints turn-N, END/UPDATE use the open turn. + adapter = scripted({ + "prompt": [SpanEvent(START, TURN, "S", input="P")], + "tool": [SpanEvent(END, TOOL, "S", name="Shell", duration_ms=100)], + "reply": [SpanEvent(UPDATE, TURN, "S", output="R")], + "stop": [SpanEvent(END, TURN, "S", status=OK)], + }) + spans = self.run_hooks(adapter, [(0, "prompt"), (500, "tool"), (600, "reply"), (700, "stop"), + (800, "prompt"), (900, "stop"), (950, "stop")]) + turn1 = emit.span_id("fake", "S", TURN, "turn-1") + turn2 = emit.span_id("fake", "S", TURN, "turn-2") + tool = [s for s in spans if s["name"] == "Shell"][0] + self.assertEqual(tool["parentSpanId"], turn1) + t1 = latest(spans, turn1) + self.assertEqual(int(t1["endTimeUnixNano"]) - int(t1["startTimeUnixNano"]), 700 * MS) + self.assertEqual(attrs(t1)["coding_agent.turn.id"], "turn-1") + t2 = latest(spans, turn2) + self.assertEqual(int(t2["endTimeUnixNano"]) - int(t2["startTimeUnixNano"]), 100 * MS) + # The extra stop has no open turn and is ignored. + self.assertEqual(len([s for s in spans if s["spanId"] in (turn1, turn2)]), 2) + + def test_keyless_tool_end_pairs_with_oldest_start_of_same_name(self): + # Apps whose pre/post tool hooks carry no call id: FIFO by tool name. + adapter = scripted({ + "pre-read": [SpanEvent(START, TOOL, "S", turn_id="g", name="Read")], + "pre-shell": [SpanEvent(START, TOOL, "S", turn_id="g", name="Shell")], + "post-read": [SpanEvent(END, TOOL, "S", turn_id="g", name="Read")], + "post-shell": [SpanEvent(END, TOOL, "S", turn_id="g", name="Shell", status=ERROR)], + }) + spans = self.run_hooks(adapter, [(0, "pre-read"), (100, "pre-read"), (200, "pre-shell"), + (300, "post-read"), (400, "post-shell"), (450, "post-read")]) + reads = [s for s in spans if s["name"] == "Read"] + self.assertEqual([int(s["endTimeUnixNano"]) - int(s["startTimeUnixNano"]) for s in reads], + [300 * MS, 350 * MS]) + shell = [s for s in spans if s["name"] == "Shell"][0] + self.assertEqual(int(shell["endTimeUnixNano"]) - int(shell["startTimeUnixNano"]), 200 * MS) + self.assertEqual(shell["status"]["code"], 2) + self.assertNotIn("gen_ai.tool.call.id", attrs(shell)) + + def test_turn_start_closes_other_open_turn_and_late_end_corrects_it(self): + adapter = scripted({ + "t1": [SpanEvent(START, TURN, "S", key="g1")], + "tool1": [SpanEvent(START, TOOL, "S", key="c1", turn_id="g1", name="Shell")], + "t2": [SpanEvent(START, TURN, "S", key="g2")], + "end1": [SpanEvent(END, TURN, "S", key="g1", status=OK)], + "end2": [SpanEvent(END, TURN, "S", key="g2")], + }) + first = self.run_hooks(adapter, [(0, "t1"), (100, "tool1"), (500, "t2")]) + g1 = emit.span_id("fake", "S", TURN, "g1") + incomplete = latest(first, g1) + self.assertTrue(attrs(incomplete)["coding_agent.incomplete"]) + tool = [s for s in first if s["name"] == "Shell"][0] + self.assertTrue(attrs(tool)["coding_agent.incomplete"], "children of the closed turn are closed too") + # A late end for g1 re-sends it complete, same id and start, without the incomplete flag. + late = self.run_hooks(adapter, [(600, "end1")]) + corrected = latest(late, g1) + self.assertEqual(corrected["startTimeUnixNano"], incomplete["startTimeUnixNano"]) + self.assertNotIn("coding_agent.incomplete", attrs(corrected)) + self.assertEqual(corrected["status"]["code"], 1) + # g2 is still the open turn. + rest = self.run_hooks(adapter, [(900, "end2")]) + g2 = latest(rest, emit.span_id("fake", "S", TURN, "g2")) + self.assertEqual(int(g2["endTimeUnixNano"]) - int(g2["startTimeUnixNano"]), 400 * MS) + + def test_empty_response_prints_nothing(self): + module = types.ModuleType("adapters.fake_quiet") + module.APP, module.DISPLAY_NAME = "fake_quiet", "Quiet" + module.parse = lambda event, payload: [] + module.respond = lambda event, payload: "" + sys.modules["adapters.fake_quiet"] = module + try: + self.assertEqual(emit.run(["fake_quiet", "UserPromptSubmit"], "{}", {}), "") + module.respond = lambda event, payload: None + self.assertEqual(emit.run(["fake_quiet", "UserPromptSubmit"], "{}", {}), "{}") + module.respond = lambda event, payload: 1 / 0 + self.assertEqual(emit.run(["fake_quiet", "UserPromptSubmit"], "{}", {}), "", + "a failing respond() prints nothing, never a default answer") + finally: + del sys.modules["adapters.fake_quiet"] + + def test_children_never_start_before_the_exported_root(self): + adapter = scripted({ + "tool1": [SpanEvent(END, TOOL, "S", key="c1", turn_id="g", name="Read", duration_ms=10)], + "tool2": [SpanEvent(END, TOOL, "S", key="c2", turn_id="g", name="Shell", duration_ms=5000)], + "stop": [SpanEvent(END, TURN, "S", key="g")], + }) + spans = self.run_hooks(adapter, [(1000, "tool1"), (2000, "tool2"), (2100, "stop")]) + root_start = int(latest(spans, emit.span_id("fake", "S", SESSION, ""))["startTimeUnixNano"]) + self.assertTrue(all(int(s["startTimeUnixNano"]) >= root_start for s in spans)) + + def test_root_model_comes_from_session_turn_and_llm_events_only(self): + adapter = scripted({ + "t": [SpanEvent(START, TURN, "S", key="g", model="turn-model")], + "tool": [SpanEvent(END, TOOL, "S", key="c", turn_id="g", name="Read", model="tool-model")], + "stop": [SpanEvent(END, TURN, "S", key="g")], + }) + spans = self.run_hooks(adapter, [(0, "t"), (10, "tool"), (20, "stop")]) + root = latest(spans, emit.span_id("fake", "S", SESSION, "")) + self.assertEqual(attrs(root)["llm.model_name"], "turn-model") + + def test_tool_without_turn_id_is_marked_as_attached_to_open_turn(self): + adapter = scripted({ + "t": [SpanEvent(START, TURN, "S", key="g1")], + "tool": [SpanEvent(END, TOOL, "S", key="c", name="Shell")], + "tool-known": [SpanEvent(END, TOOL, "S", key="d", turn_id="g1", name="Read")], + }) + spans = self.run_hooks(adapter, [(0, "t"), (10, "tool"), (20, "tool-known")]) + shell = [s for s in spans if s["name"] == "Shell"][0] + self.assertEqual(shell["parentSpanId"], emit.span_id("fake", "S", TURN, "g1")) + self.assertTrue(attrs(shell)["coding_agent.turn.inferred"]) + read = [s for s in spans if s["name"] == "Read"][0] + self.assertNotIn("coding_agent.turn.inferred", attrs(read)) + + def test_state_keeps_no_content_of_completed_turns_and_ignores_duplicate_ends(self): + cfg = config("fake", self.tmp.name, FI_HOOK_CAPTURE_CONTENT="1") + adapter = scripted({ + "t": [SpanEvent(START, TURN, "S", key="g", input="PROMPT_IN_STATE")], + "reply": [SpanEvent(UPDATE, TURN, "S", key="g", output="REPLY_IN_STATE")], + "stop": [SpanEvent(END, TURN, "S", key="g")], + }) + self.run_hooks(adapter, [(0, "t"), (10, "reply"), (20, "stop")], cfg=cfg) + state_text = "".join(p.read_text() for p in cfg.state_dir.glob("*.json")) + self.assertNotIn("PROMPT_IN_STATE", state_text) + self.assertNotIn("REPLY_IN_STATE", state_text) + self.assertEqual(self.run_hooks(adapter, [(30, "stop")], cfg=cfg), [], "duplicate end is ignored") + + def test_opted_in_content_drops_the_user_name_from_home_paths(self): + text = emit.render_content({"file_path": "/Users/alice/p/a.md", "cwd": "/home/bob/x", + "win": "C:\\Users\\carol\\y"}, 0) + for name in ("alice", "bob", "carol"): + self.assertNotIn(name, text) + self.assertIn("/Users//p/a.md", text) + + def assert_scrubs(self, cases): + for raw, want in cases.items(): + self.assertEqual(emit.render_content(raw, 0), want) + self.assertEqual(json.loads(emit.render_content({"p": raw}, 0))["p"], want, "JSON-escaped") + + def test_home_path_scrub_replaces_only_the_name_component(self): + # Regression: a name must not run on across spaces and swallow the command after it. + with mock.patch.dict(os.environ, {"HOME": "/Users/someone"}): + self.assert_scrubs({ + "cd /Users/alice && ls src/foo": "cd /Users/ && ls src/foo", + "grep -rn TODO /home/bob --include=*.py && cat docs/readme.md": + "grep -rn TODO /home/ --include=*.py && cat docs/readme.md", + "ls /Users/alice;echo hi": "ls /Users/;echo hi", + "(see /Users/alice), then /home/bob.": "(see /Users/), then /home/.", + "see /Users/alice and /tmp": "see /Users/ and /tmp", + "/Users/john.smith/x and /Users/ann@corp/y": "/Users//x and /Users//y", + "C:\\Users\\carol\\y and d:/Users/dave/z": "C:\\Users\\\\y and d:/Users//z", + "~bob/x and ~/y": "~/x and ~/y", + "/Users//x": "/Users//x", + }) + + def test_spaced_user_name_is_scrubbed_for_this_machines_home(self): + with mock.patch.dict(os.environ, {"HOME": "/Users/Jo Ann"}): + self.assert_scrubs({ + "error at /Users/Jo Ann": "error at /Users/", + "failed in /Users/Jo Ann.": "failed in /Users/.", + "at /Users/Jo Ann, ok": "at /Users/, ok", + "cd /Users/Jo Ann && ls src/x": "cd /Users/ && ls src/x", + "/Users/Jo Ann/notes.txt": "/Users//notes.txt", + # Another account that only starts with the same name is not this home. + "/Users/Jo Annabel/x": "/Users/ Annabel/x", + }) + with mock.patch.dict(os.environ, {"HOME": "C:\\Users\\Jo Ann"}): + self.assert_scrubs({ + "C:\\Users\\Jo Ann\\x": "C:\\Users\\\\x", + "c:/Users/Jo Ann/x": "c:/Users//x", + "dir C:\\Users\\Jo Ann": "dir C:\\Users\\", + }) + with mock.patch.dict(os.environ, {"HOME": "/Users/al"}): + self.assert_scrubs({"/Users/alice/x /Users/al/y /Users/al.": "/Users//x /Users//y /Users/."}) + + def test_another_accounts_spaced_name_keeps_the_part_after_the_space(self): + # Documented limitation: without this machine's home to go by, a space ends the name. + with mock.patch.dict(os.environ, {"HOME": "/Users/someone"}): + self.assert_scrubs({ + "/Users/Hank Smith/notes.txt": "/Users/ Smith/notes.txt", + "C:\\Users\\John Smith\\x": "C:\\Users\\ Smith\\x", + }) + + def test_default_timeout_is_short_when_the_send_is_inline(self): + fork = os.fork + try: + del os.fork + self.assertEqual(emit.Config.from_env("fake", {}).timeout, 3.0) + finally: + os.fork = fork + self.assertEqual(emit.Config.from_env("fake", {}).timeout, 10.0) + + def test_end_without_start_uses_vendor_duration(self): + adapter = scripted({"post": [SpanEvent(END, TOOL, "S", key="c", name="Read", duration_ms=250)]}) + spans = self.run_hooks(adapter, [(1000, "post")]) + tool = spans[0] + self.assertEqual(int(tool["endTimeUnixNano"]) - int(tool["startTimeUnixNano"]), 250 * MS) + # No turn known -> parented to the session root. + self.assertEqual(tool["parentSpanId"], emit.span_id("fake", "S", SESSION, "")) + + def test_tool_uses_current_turn_when_turn_id_missing(self): + adapter = scripted({ + "t0": [SpanEvent(START, TURN, "S", key="g7")], + "tool": [SpanEvent(INSTANT, TOOL, "S", name="Grep")], + }) + spans = self.run_hooks(adapter, [(0, "t0"), (10, "tool")]) + self.assertEqual(spans[0]["parentSpanId"], emit.span_id("fake", "S", TURN, "g7")) + + def test_turn_auto_opens_and_root_upserts_on_turn_end(self): + adapter = scripted({ + "tool": [SpanEvent(END, TOOL, "S", key="c1", turn_id="g1", name="Shell", duration_ms=300)], + "stop": [SpanEvent(END, TURN, "S", key="g1")], + }) + spans = self.run_hooks(adapter, [(1000, "tool"), (2000, "stop")]) + turn = [s for s in spans if s["spanId"] == emit.span_id("fake", "S", TURN, "g1")][0] + # Turn starts when its first child started. + self.assertEqual(int(turn["startTimeUnixNano"]), T0 + 700 * MS) + root = latest(spans, emit.span_id("fake", "S", SESSION, "")) + self.assertIsNotNone(root, "root must be exported at turn end even without a session hook") + self.assertEqual(int(root["startTimeUnixNano"]), T0 + 700 * MS) + + def test_open_children_are_closed_as_incomplete_at_turn_end(self): + adapter = scripted({ + "pre": [SpanEvent(START, TOOL, "S", key="c9", turn_id="g1", name="Shell")], + "stop": [SpanEvent(END, TURN, "S", key="g1")], + }) + spans = self.run_hooks(adapter, [(0, "pre"), (50, "stop")]) + tool = [s for s in spans if s["name"] == "Shell"][0] + self.assertTrue(attrs(tool)["coding_agent.incomplete"]) + + def test_error_status_and_message_privacy(self): + adapter = scripted({"fail": [SpanEvent(END, TOOL, "S", key="c", name="Shell", status=ERROR, + status_message="timeout", error_message="secret detail")]}) + span = self.run_hooks(adapter, [(0, "fail")])[0] + self.assertEqual(span["status"], {"code": 2, "message": "timeout"}) + self.assertNotIn("secret detail", json.dumps(span)) + + def test_content_dropped_by_default_and_kept_on_opt_in(self): + events = { + "t0": [SpanEvent(START, TURN, "S", key="g", input="PROMPT_MARK")], + "tool": [SpanEvent(END, TOOL, "S", key="c", turn_id="g", name="Shell", + input={"command": "echo ARG_MARK"}, output="OUT_MARK")], + "resp": [SpanEvent(UPDATE, TURN, "S", key="g", output="RESP_MARK")], + "stop": [SpanEvent(END, TURN, "S", key="g")], + } + hooks = [(0, "t0"), (10, "tool"), (20, "resp"), (30, "stop")] + off = json.dumps(self.run_hooks(scripted(events), hooks)) + for marker in ("PROMPT_MARK", "ARG_MARK", "OUT_MARK", "RESP_MARK", "input.value", "output.value"): + self.assertNotIn(marker, off) + with tempfile.TemporaryDirectory() as other: + cfg = config("fake", other, FI_HOOK_CAPTURE_CONTENT="1") + on = self.run_hooks(scripted(events), hooks, cfg=cfg) + turn = [s for s in on if s["spanId"] == emit.span_id("fake", "S", TURN, "g")][0] + self.assertEqual(attrs(turn)["input.value"], "PROMPT_MARK") + self.assertEqual(attrs(turn)["output.value"], "RESP_MARK") + tool = [s for s in on if s["name"] == "Shell"][0] + self.assertIn("ARG_MARK", attrs(tool)["input.value"]) + + def test_redaction_and_truncation(self): + text = "key sk-abcdefghijklmnopqrstuv and Bearer abcdefghijklmnop api_key=xyz123 " + "x" * 50 + rendered = emit.render_content(text, max_chars=60) + self.assertNotIn("sk-abcdefghijklmnopqrstuv", rendered) + self.assertNotIn("abcdefghijklmnop api", rendered) + self.assertIn("[truncated", rendered) + self.assertIn("[REDACTED]", emit.render_content({"password": "hunter22"}, 0)) + + def test_events_without_session_are_dropped(self): + adapter = scripted({"x": [SpanEvent(INSTANT, LLM, "", name="nope")]}) + self.assertEqual(self.run_hooks(adapter, [(0, "x")]), []) + + def test_state_sweep_removes_old_files(self): + self.cfg.state_dir.mkdir(parents=True, exist_ok=True) + old = self.cfg.state_dir / "old.json" + old.write_text("{}") + os.utime(old, (time.time() - 3 * 86400,) * 2) + emit.sweep_state(self.cfg) + self.assertFalse(old.exists()) + + def test_attribute_encoding(self): + encoded = {a["key"]: a["value"] for a in emit.otlp_attributes( + {"b": True, "i": 3, "f": 1.5, "s": "x", "l": ["a", 1], "n": None})} + self.assertEqual(encoded["b"], {"boolValue": True}) + self.assertEqual(encoded["i"], {"intValue": "3"}) + self.assertEqual(encoded["f"], {"doubleValue": 1.5}) + self.assertEqual(encoded["l"]["arrayValue"]["values"][1], {"intValue": "1"}) + self.assertNotIn("n", encoded) + + def test_env_file_fills_missing_keys_only(self): + with tempfile.NamedTemporaryFile("w", suffix=".env", delete=False) as handle: + handle.write("# comment\nexport FI_API_KEY='from-file'\nFI_PROJECT_NAME=p\nOTHER=ignored\n") + env = {"FI_API_KEY": "from-env"} + emit.load_env_file(handle.name, env) + os.unlink(handle.name) + self.assertEqual(env, {"FI_API_KEY": "from-env", "FI_PROJECT_NAME": "p"}) + + +class CliTest(unittest.TestCase): + """End-to-end through the real CLI with the Cursor adapter and a loopback receiver.""" + + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.rx = Receiver() + + def tearDown(self): + self.rx.close() + self.tmp.cleanup() + + def payload(self, **extra): + p = {"conversation_id": "cli-1", "generation_id": "g1", "hook_event_name": "postToolUse", + "tool_name": "Shell", "tool_use_id": "t1", "duration": 10, "tool_input": {"command": "ls"}} + p.update(extra) + return p + + def test_sync_export_headers_resource_and_stdout(self): + env = base_env(self.tmp.name, endpoint=self.rx.url + "/tracer/v1/traces") + result = run_cli(["cursor", "postToolUse"], self.payload(), env) + self.assertEqual(result.returncode, 0) + self.assertEqual(result.stdout.strip(), "{}") + self.assertEqual(len(self.rx.requests), 1) + path, headers, body = self.rx.requests[0] + self.assertEqual(path, "/tracer/v1/traces") + self.assertEqual(headers["X-Api-Key"], "test-api-key") + self.assertEqual(headers["X-Secret-Key"], "test-secret-key") + resource = attrs(body["resourceSpans"][0]["resource"]) + self.assertEqual(resource["project_name"], "hook-test") + self.assertEqual(resource["service.name"], "cursor") + self.assertFalse(resource["coding_agent.content_captured"]) + + def test_base_url_gets_traces_path(self): + env = base_env(self.tmp.name, FI_BASE_URL=self.rx.url) + del env["FI_OTLP_ENDPOINT"] + run_cli(["cursor", "postToolUse"], self.payload(), env) + self.assertEqual(self.rx.requests[0][0], "/tracer/v1/traces") + + def test_background_send_returns_before_network(self): + env = base_env(self.tmp.name, endpoint=self.rx.url + "/v1/traces") + del env["FI_HOOK_SYNC"] + result = run_cli(["cursor", "postToolUse"], self.payload(), env) + self.assertEqual((result.returncode, result.stdout.strip()), (0, "{}")) + self.rx.wait_for(1) + self.assertEqual(sorted(sp["name"] for sp in self.rx.spans()), ["Cursor session", "Shell"]) + + def test_unreachable_collector_is_fast_and_silent(self): + env = base_env(self.tmp.name, endpoint="http://127.0.0.1:9/v1/traces", + FI_HOOK_LOG_FILE=os.path.join(self.tmp.name, "hook.log")) + started = time.monotonic() + result = run_cli(["cursor", "postToolUse"], self.payload(), env) + self.assertLess(time.monotonic() - started, 5) + self.assertEqual((result.returncode, result.stdout.strip(), result.stderr), (0, "{}", "")) + log = Path(self.tmp.name, "hook.log").read_text() + self.assertIn("error=", log) + self.assertNotIn("test-api-key", log) + + def test_collector_error_status_is_swallowed(self): + bad = Receiver(status=500) + try: + env = base_env(self.tmp.name, endpoint=bad.url + "/v1/traces") + result = run_cli(["cursor", "postToolUse"], self.payload(), env) + self.assertEqual((result.returncode, result.stdout.strip()), (0, "{}")) + finally: + bad.close() + + def test_garbage_input_still_answers_host(self): + env = base_env(self.tmp.name, endpoint=self.rx.url + "/v1/traces") + for stdin in ("", "not json", "[1,2]"): + result = run_cli(["cursor", "beforeSubmitPrompt"], stdin, env) + self.assertEqual((result.returncode, result.stdout.strip()), (0, '{"continue": true}')) + result = run_cli(["no-such-app"], "{}", env) + self.assertEqual((result.returncode, result.stdout), (0, "")) + self.assertEqual(self.rx.requests, []) + + def test_adapter_that_fails_to_load_prints_nothing_even_on_a_permission_hook(self): + # A broken install (adapter file missing) must not answer "{}": Cursor + # blocks a permission hook on that. No output is a failed hook (fail-open). + broken = Path(self.tmp.name, "broken") + shutil.copytree(Path(emit.__file__).parent, broken, + ignore=shutil.ignore_patterns("tests", "__pycache__", "cursor.py")) + env = {"PATH": os.environ.get("PATH", ""), "HOME": self.tmp.name, + **base_env(self.tmp.name, endpoint=self.rx.url + "/v1/traces")} + for event in ("preToolUse", "beforeSubmitPrompt", "stop"): + result = subprocess.run([sys.executable, "-B", str(broken / "emit.py"), "cursor", event], + input=json.dumps(self.payload()), capture_output=True, text=True, + env=env, timeout=20) + self.assertEqual((result.returncode, result.stdout, result.stderr), (0, "", ""), event) + self.assertEqual(self.rx.requests, []) + + def test_missing_credentials_is_a_noop(self): + env = base_env(self.tmp.name, endpoint=self.rx.url + "/v1/traces") + del env["FI_API_KEY"] + result = run_cli(["cursor", "postToolUse"], self.payload(), env) + self.assertEqual((result.returncode, result.stdout.strip()), (0, "{}")) + self.assertEqual(self.rx.requests, []) + + def test_disabled_switch(self): + env = base_env(self.tmp.name, endpoint=self.rx.url + "/v1/traces", FI_HOOK_DISABLED="1") + run_cli(["cursor", "postToolUse"], self.payload(), env) + self.assertEqual(self.rx.requests, []) + + def test_dump_file_matches_what_was_sent(self): + dump = os.path.join(self.tmp.name, "dump.jsonl") + env = base_env(self.tmp.name, endpoint=self.rx.url + "/v1/traces", FI_HOOK_DUMP_FILE=dump) + run_cli(["cursor", "postToolUse"], self.payload(), env) + dumped = [json.loads(line) for line in Path(dump).read_text().splitlines()] + self.assertEqual(all_spans(dumped[0]), self.rx.spans()) + + +if __name__ == "__main__": + unittest.main() diff --git a/examples/coding-agents/hook-to-otlp/tests/test_cursor_adapter.py b/examples/coding-agents/hook-to-otlp/tests/test_cursor_adapter.py new file mode 100644 index 000000000..b983b2d9f --- /dev/null +++ b/examples/coding-agents/hook-to-otlp/tests/test_cursor_adapter.py @@ -0,0 +1,241 @@ +"""Cursor adapter contract tests against payloads shaped as https://cursor.com/docs/hooks documents them.""" +from __future__ import annotations + +import builtins +import io +import json +import os +import tempfile +import unittest +from pathlib import Path + +from tests.helpers import FIXTURES, attrs, by_name, config, latest, load_jsonl, replay + +import emit +from adapters import cursor +from adapters.base import SESSION, TURN + +SESSION_FIXTURE = FIXTURES / "cursor" / "session.jsonl" +MARKERS = ("PROMPT_MARKER_CURSOR", "THOUGHT_MARKER_CURSOR", "TOOL_INPUT_MARKER_CURSOR", + "TOOL_OUTPUT_MARKER_CURSOR", "ERROR_MARKER_CURSOR", "SUBAGENT_MARKER_CURSOR", + "RESPONSE_MARKER_CURSOR") +# Never exported, even with content capture on. +NEVER = ("fixture.user@example.com", "fixture-transcript-CANARY", "fixture-subagent-transcript") +# Paths only appear inside opted-in content (tool input), never as metadata, and +# even then with the user name replaced by . +DEFAULT_ONLY = ("/Users/fixture/project",) + + +class CursorAdapterTest(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.records = load_jsonl(SESSION_FIXTURE) + + def tearDown(self): + self.tmp.cleanup() + + def spans(self, capture=False): + extra = {"FI_HOOK_CAPTURE_CONTENT": "1"} if capture else {} + return replay(cursor, config("cursor", self.tmp.name, **extra), self.records) + + def test_one_trace_with_session_turn_tool_tree(self): + spans = self.spans() + self.assertEqual({s["traceId"] for s in spans}, {emit.trace_id("cursor", "fixture-conv-0001")}) + root_id = emit.span_id("cursor", "fixture-conv-0001", SESSION, "") + turn_id = emit.span_id("cursor", "fixture-conv-0001", TURN, "fixture-gen-0001") + root = latest(spans, root_id) + self.assertEqual(root["name"], "Cursor session") + self.assertEqual(attrs(root)["fi.span.kind"], "AGENT") + turn = latest(spans, turn_id) + self.assertEqual(turn["name"], "Cursor turn") + self.assertEqual(turn["parentSpanId"], root_id) + self.assertEqual(attrs(turn)["llm.model_name"], "claude-opus-4-7") + self.assertEqual(attrs(turn)["cursor.model"], "claude-opus-4-7-thinking-max") + self.assertEqual(attrs(turn)["cursor.model_params.effort"], "max") + names = sorted(s["name"] for s in spans if s.get("parentSpanId") == turn_id) + self.assertEqual(names, ["Agent response", "Agent thought", "Context compaction", + "Read", "Shell", "Shell", "Subagent: explore"]) + # Session end ran last; the root is final and ended after the turn. + self.assertGreaterEqual(int(root["endTimeUnixNano"]), int(turn["endTimeUnixNano"])) + self.assertEqual(attrs(root)["cursor.session.reason"], "user_close") + + def test_tools_use_vendor_duration_and_ids(self): + shell_ok, shell_failed = by_name(self.spans(), "Shell") + self.assertEqual(int(shell_ok["endTimeUnixNano"]) - int(shell_ok["startTimeUnixNano"]), 420_000_000) + a = attrs(shell_ok) + self.assertEqual((a["fi.span.kind"], a["gen_ai.tool.name"], a["gen_ai.tool.call.id"]), + ("TOOL", "Shell", "toolu_fixture_1")) + self.assertEqual(shell_ok["status"]["code"], 1) + self.assertEqual(shell_failed["status"], {"code": 2, "message": "timeout"}) + self.assertEqual(attrs(shell_failed)["cursor.tool.failure_type"], "timeout") + + def test_subagent_and_compaction_metadata(self): + spans = self.spans() + sub = by_name(spans, "Subagent: explore")[0] + self.assertEqual(attrs(sub)["cursor.subagent.tool_call_count"], "3") + self.assertEqual(int(sub["endTimeUnixNano"]) - int(sub["startTimeUnixNano"]), 4_000_000_000) + compaction = attrs(by_name(spans, "Context compaction")[0]) + self.assertEqual(compaction["cursor.compaction.context_tokens"], "120000") + + def test_default_export_has_no_content_or_pii(self): + encoded = json.dumps(self.spans()) + for marker in MARKERS + NEVER + DEFAULT_ONLY: + self.assertNotIn(marker, encoded, marker) + + def test_opt_in_exports_content_but_never_pii(self): + spans = self.spans(capture=True) + encoded = json.dumps(spans) + for marker in MARKERS: + self.assertIn(marker, encoded, marker) + for marker in NEVER: + self.assertNotIn(marker, encoded, marker) + turn = latest(spans, emit.span_id("cursor", "fixture-conv-0001", TURN, "fixture-gen-0001")) + self.assertIn("PROMPT_MARKER_CURSOR", attrs(turn)["input.value"]) + self.assertIn("RESPONSE_MARKER_CURSOR", attrs(turn)["output.value"]) + failed = by_name(spans, "Shell")[1] + self.assertIn("ERROR_MARKER_CURSOR", failed["status"]["message"]) + + def test_transcript_path_is_never_opened(self): + # PRD V-NEG: point transcript_path at an unreadable canary and spy on every open path. + canary = Path(self.tmp.name, "canary-transcript.txt") + canary.write_text("TRANSCRIPT_CANARY_TEXT") + canary.chmod(0) + records = [{**r, "payload": {**r["payload"], "transcript_path": str(canary)}} for r in self.records] + opened = [] + real_open, real_io_open, real_os_open = builtins.open, io.open, os.open + + def spy(path, *args, **kwargs): + opened.append(str(path)) + return real_open(path, *args, **kwargs) + + def os_spy(path, *args, **kwargs): + opened.append(str(path)) + return real_os_open(path, *args, **kwargs) + + builtins.open = io.open = spy + os.open = os_spy + try: + spans = replay(cursor, config("cursor", self.tmp.name, FI_HOOK_CAPTURE_CONTENT="1"), records) + finally: + builtins.open, io.open, os.open = real_open, real_io_open, real_os_open + canary.chmod(0o600) + self.assertTrue(spans) + self.assertNotIn(str(canary), opened) + self.assertNotIn("TRANSCRIPT_CANARY_TEXT", json.dumps(spans)) + self.assertNotIn("canary-transcript", json.dumps(spans)) + + def test_tool_without_generation_id_attaches_to_open_turn_and_says_so(self): + payloads = [r["payload"] for r in self.records] + prompt = {**payloads[1]} + tool = {k: v for k, v in payloads[3].items() if k != "generation_id"} + spans = replay(cursor, config("cursor", self.tmp.name), [ + {"event": "beforeSubmitPrompt", "payload": prompt}, + {"event": "postToolUse", "payload": tool}, + ]) + shell = by_name(spans, "Shell")[0] + self.assertEqual(shell["parentSpanId"], + emit.span_id("cursor", "fixture-conv-0001", TURN, "fixture-gen-0001")) + self.assertTrue(attrs(shell)["coding_agent.turn.inferred"]) + + def test_same_subagent_task_in_two_turns_keeps_two_spans(self): + stop = [r for r in self.records if r["event"] == "subagentStop"][0]["payload"] + events = cursor.parse("subagentStop", stop) + cursor.parse( + "subagentStop", {**stop, "generation_id": "another-gen"}) + self.assertNotEqual(events[0].key, events[1].key) + + def test_missing_conversation_id_drops_event(self): + payload = dict(self.records[3]["payload"]) + del payload["conversation_id"] + self.assertEqual(cursor.parse("postToolUse", payload), []) + + def test_real_cursor_3_24_18_session_end_payload(self): + # Captured from the Cursor 3.24.18 Hooks output channel (empty chat closed with the window). + record = load_jsonl(FIXTURES / "cursor" / "real-3.24.18.jsonl")[0] + events = cursor.parse(record["event"], record["payload"]) + self.assertEqual(len(events), 1) + self.assertEqual((events[0].phase, events[0].scope), ("end", SESSION)) + self.assertEqual(events[0].attributes["cursor.session.reason"], "window_close") + # An empty conversation is not exported. + self.assertEqual(replay(cursor, config("cursor", self.tmp.name), [record]), []) + + def test_real_cursor_3_24_18_agent_turn(self): + # One real IDE agent turn on the Hobby plan (beforeSubmitPrompt, thoughts, Read, Grep, response, stop). + records = load_jsonl(FIXTURES / "cursor" / "real-3.24.18-turn.jsonl") + spans = replay(cursor, config("cursor", self.tmp.name), records) + sid = records[0]["payload"]["conversation_id"] + gen = records[0]["payload"]["generation_id"] + self.assertEqual({s["traceId"] for s in spans}, {emit.trace_id("cursor", sid)}) + turn_id = emit.span_id("cursor", sid, TURN, gen) + turns = {s["spanId"] for s in spans if attrs(s)["coding_agent.scope"] == "turn"} + self.assertEqual(turns, {turn_id}, "suffixed generation ids must not open extra turns") + children = {} + for span in spans: + if span.get("parentSpanId") == turn_id: + children[span["spanId"]] = span["name"] + self.assertEqual(sorted(children.values()), ["Agent response", "Agent thought", "Agent thought", "Grep", "Read"]) + response = latest(spans, [k for k, v in children.items() if v == "Agent response"][0]) + a = attrs(response) + self.assertEqual((a["llm.token_count.prompt"], a["llm.token_count.completion"], a["llm.token_count.total"]), + ("38779", "306", "39085")) + self.assertEqual(a["llm.token_count.prompt_details.cache_read"], "24192") + self.assertEqual(a["llm.model_name"], "grok-4.6") + read = [s for s in spans if s["name"] == "Read"][0] + self.assertEqual(attrs(read)["gen_ai.tool.call.id"], "call-f3e41504-16cd-4345-9614-dccb061cbdd4-1") + self.assertEqual(attrs(read)["cursor.tool.provider_call_id"], "fc_cdbfcc58-3e58-9ca2-92f0-366509226caf_1") + self.assertFalse(any("\n" in str(v) for s in spans for v in attrs(s).values())) + turn = latest(spans, turn_id) + self.assertEqual(attrs(turn)["cursor.usage.input_tokens"], "38779") + self.assertNotIn("llm.token_count.prompt", attrs(turn)) + encoded = json.dumps(spans) + for marker in ("PROMPT_MARKER_REAL", "THOUGHT_MARKER_REAL", "RESPONSE_MARKER_REAL", + "TOOL_OUTPUT_MARKER_REAL") + NEVER + DEFAULT_ONLY: + self.assertNotIn(marker, encoded, marker) + + def test_stdout_contract(self): + self.assertEqual(json.loads(cursor.respond("beforeSubmitPrompt", {})), {"continue": True}) + self.assertEqual(json.loads(cursor.respond("stop", {})), {}) + self.assertEqual(json.loads(cursor.respond("sessionStart", {})), {}) + # Misregistered on a permission hook: print nothing (Cursor fails open on no output), + # never "allow" (could approve) and never "{}" (schema-invalid, blocks). + for hook in cursor.PERMISSION_HOOKS: + self.assertEqual(cursor.respond(hook, {}), "", hook) + env = {"FI_HOOK_ENV_FILE": str(Path(self.tmp.name, "no-env-file"))} + self.assertEqual(emit.run(["cursor", hook], json.dumps({"conversation_id": "c"}), env), "", hook) + + def test_error_text_with_paths_is_content_only(self): + # Cross-cutting rule: error text is content; a path with a user name never leaks by default. + leak = "ENOENT: open '/Users/alice/secret-proj/.env' ERRPATH_MARKER" + records = [r for r in self.records if r["event"] in ("beforeSubmitPrompt", "postToolUseFailure")] + records = [{**r, "payload": {**r["payload"], "error_message": leak}} for r in records] + records.append({"event": "sessionEnd", "payload": {**self.records[-1]["payload"], + "reason": "error", "error_message": leak}}) + off = replay(cursor, config("cursor", self.tmp.name), records) + encoded = json.dumps(off) + for marker in ("alice", "secret-proj", "ERRPATH_MARKER", "/Users/"): + self.assertNotIn(marker, encoded, marker) + self.assertEqual(by_name(off, "Shell")[0]["status"], {"code": 2, "message": "timeout"}) + with tempfile.TemporaryDirectory() as other: + on = json.dumps(replay(cursor, config("cursor", other, FI_HOOK_CAPTURE_CONTENT="1"), records)) + self.assertIn("/Users//secret-proj/.env' ERRPATH_MARKER", on) + self.assertNotIn("alice", on) + + def test_opt_in_paths_keep_no_user_name(self): + records = load_jsonl(FIXTURES / "cursor" / "real-3.24.18-turn.jsonl") + encoded = json.dumps(replay(cursor, config("cursor", self.tmp.name, FI_HOOK_CAPTURE_CONTENT="1"), records)) + self.assertIn("/Users//project/README.md", encoded) + self.assertNotIn("/Users/fixture", encoded) + + def test_unmapped_hooks_are_ignored(self): + for hook in ("afterShellExecution", "afterFileEdit", "beforeTabFileRead", "workspaceOpen"): + self.assertEqual(cursor.parse(hook, {"conversation_id": "c", "generation_id": "g"}), []) + + def test_resource_carries_cursor_version(self): + cfg = config("cursor", self.tmp.name) + body = emit.build_body(cursor, cfg, "stop", self.records[0]["payload"], []) + resource = attrs(body["resourceSpans"][0]["resource"]) + self.assertEqual(resource["coding_agent.app.version"], "3.24.18") + self.assertNotIn("fixture.user@example.com", json.dumps(body)) + + +if __name__ == "__main__": + unittest.main() diff --git a/examples/coding-agents/hook-to-otlp/tests/test_cursor_hooks_config.py b/examples/coding-agents/hook-to-otlp/tests/test_cursor_hooks_config.py new file mode 100644 index 000000000..004ea5426 --- /dev/null +++ b/examples/coding-agents/hook-to-otlp/tests/test_cursor_hooks_config.py @@ -0,0 +1,161 @@ +"""The shipped Cursor hooks.json must never block Cursor (exit 2 blocks), with or without the emitter. + +Cursor runs a hook command as `$SHELL -c ` (falling back to /bin/sh) and passes the +payload in one of two ways, depending on the surface (both read from the Cursor 3.24.18 bundle): + +- stdin: the IDE agent pipes the JSON to the command's stdin. +- heredoc: the bundled headless agent host (commandHookPayloadTransport "argv_heredoc") runs + ` <<'CURSOR_HOOK_EOF'\n\nCURSOR_HOOK_EOF` with stdin ignored. + +Every shipped command is a subshell, `(python3 -B ... 2>/dev/null || true)`, so the heredoc +reaches python3. Without the parentheses the heredoc binds to `true` and the emitter reads nothing. +""" +from __future__ import annotations + +import json +import os +import re +import subprocess +import tempfile +import unittest +from pathlib import Path + +from tests.helpers import Receiver, attrs + +EXAMPLE = Path(__file__).resolve().parents[2] / "cursor" +PERMISSION_HOOKS = {"preToolUse", "beforeShellExecution", "beforeMCPExecution", "beforeReadFile", + "beforeTabFileRead", "subagentStart"} +EXPECTED_STDOUT = {"beforeSubmitPrompt": {"continue": True}} +CONFIGS = {"hooks.json": ".cursor/hooks/fi-hook-to-otlp/emit.py", "hooks.user.json": "./hooks/fi-hook-to-otlp/emit.py"} +TRANSPORTS = ("stdin", "heredoc") +SHELLS = [sh for sh in ("/bin/sh", "/bin/zsh", "/bin/bash", "/bin/dash") if os.path.exists(sh)] + + +def commands(config_name): + config = json.loads((EXAMPLE / config_name).read_text()) + return [(event, hook["command"]) for event, hooks in config["hooks"].items() for hook in hooks] + + +def run_hook(command, cwd, payload, env, shell="/bin/sh", transport="stdin"): + """Run a hook command the way Cursor does for the given payload transport.""" + text = json.dumps(payload) + if transport == "stdin": + return subprocess.run([shell, "-c", command], cwd=cwd, input=text, + capture_output=True, text=True, env=env, timeout=30) + script = f"{command} <<'CURSOR_HOOK_EOF'\n{text}\nCURSOR_HOOK_EOF" + return subprocess.run([shell, "-c", script], cwd=cwd, stdin=subprocess.DEVNULL, + capture_output=True, text=True, env=env, timeout=30) + + +class HooksConfigTest(unittest.TestCase): + payload = {"conversation_id": "cfg-1", "generation_id": "g1", "tool_name": "Shell", + "tool_use_id": "t1", "duration": 5, "status": "completed", + "prompt": "it's $HOME `id` \"quoted\" PROMPT_MARK"} + + def base_env(self, home=None, **extra): + env = {"PATH": os.environ.get("PATH", "/usr/bin:/bin"), "HOME": home or tempfile.gettempdir()} + env.update(extra) + return env + + def test_observation_hooks_only_and_every_command_is_the_guarded_subshell(self): + for name, path in CONFIGS.items(): + for event, command in commands(name): + self.assertNotIn(event, PERMISSION_HOOKS, f"{name}: {event}") + self.assertEqual(command, f"(python3 -B {path} cursor {event} 2>/dev/null || true)") + + def test_missing_emitter_exits_0_with_empty_stdout(self): + with tempfile.TemporaryDirectory() as empty: + for shell in SHELLS: + for transport in TRANSPORTS: + for name in CONFIGS: + for event, command in commands(name): + result = run_hook(command, empty, {**self.payload, "hook_event_name": event}, + self.base_env(), shell, transport) + self.assertEqual((result.returncode, result.stdout), (0, ""), + f"{shell} {transport} {name} {event}") + + def test_missing_python_exits_0_with_empty_stdout(self): + with tempfile.TemporaryDirectory() as empty: + for shell in SHELLS: + for transport in TRANSPORTS: + for event, command in commands("hooks.json"): + result = run_hook(command, empty, self.payload, self.base_env(PATH=empty), shell, transport) + self.assertEqual((result.returncode, result.stdout), (0, ""), f"{shell} {transport} {event}") + + def run_installed(self, name, install_args, cwd_of, shells): + """Install, then run every shipped command per shell and transport; return spans per run.""" + results = {} + with tempfile.TemporaryDirectory() as tmp: + home = Path(tmp, "home") + project = Path(tmp, "project") + home.mkdir() + project.mkdir() + subprocess.run([str(EXAMPLE / "install.sh"), *install_args(project)], check=True, + capture_output=True, env=self.base_env(str(home))) + for shell in shells: + for transport in TRANSPORTS: + rx = Receiver() + try: + env = self.base_env(str(home), FI_API_KEY="k", FI_SECRET_KEY="s", FI_HOOK_SYNC="1", + FI_HOOK_CAPTURE_CONTENT="1", + FI_OTLP_ENDPOINT=rx.url + "/tracer/v1/traces", + FI_HOOK_STATE_DIR=str(Path(tmp, f"state-{Path(shell).name}-{transport}"))) + sid = f"cfg-{Path(shell).name}-{transport}" + for event, command in commands(name): + result = run_hook(command, cwd_of(project, home), + {**self.payload, "conversation_id": sid, "hook_event_name": event}, + env, shell, transport) + label = f"{name} {shell} {transport} {event}" + self.assertEqual(result.returncode, 0, label) + self.assertEqual(json.loads(result.stdout), EXPECTED_STDOUT.get(event, {}), label) + results[(shell, transport)] = (sid, rx.spans()) + finally: + rx.close() + self.assertEqual(list(project.rglob("__pycache__")) + list(home.rglob("__pycache__")), [], + "-B keeps bytecode out of the repo") + return results + + def assert_payload_arrived(self, results): + for (shell, transport), (sid, spans) in results.items(): + label = f"{shell} {transport}" + shell_spans = [s for s in spans if s["name"] == "Shell"] + self.assertTrue(shell_spans, f"{label}: the payload reached the emitter") + self.assertEqual(attrs(shell_spans[-1])["session.id"], sid, label) + turn = [s for s in spans if s["name"] == "Cursor turn"][-1] + # The heredoc is quoted: no expansion of $HOME, backticks or quotes on the way in. + self.assertEqual(attrs(turn)["input.value"], self.payload["prompt"], label) + + def test_installed_project_hooks_receive_the_payload_on_stdin_and_as_heredoc(self): + results = self.run_installed("hooks.json", lambda project: [str(project)], + lambda project, home: project, SHELLS) + self.assertEqual(len(results), len(SHELLS) * len(TRANSPORTS)) + self.assert_payload_arrived(results) + + def test_installed_user_hooks_receive_the_payload_on_stdin_and_as_heredoc(self): + results = self.run_installed("hooks.user.json", lambda project: ["--user"], + lambda project, home: home / ".cursor", SHELLS[:1]) + self.assert_payload_arrived(results) + + def test_unparenthesized_guard_would_lose_a_heredoc_payload(self): + # Why the parentheses are required: `cmd || true < ./check_pin.sh # any other commit of ClickHouse/ai-sdk-cpp +``` + +## Re-run the wire probe + +Needs git, cmake 3.16+, a C++20 compiler, OpenSSL, python3, GitHub access (the build downloads zlib) and an +OpenAI-compatible endpoint for the model call (the probe makes one tool call). + +```bash +export PROBE_API_KEY=... # key for the endpoint, never committed +export PROBE_OPENAI_BASE=http://127.0.0.1:8317 # base URL without /v1 +export PROBE_MODEL=gpt-6-luna +export OPENSSL_ROOT_DIR=/opt/homebrew/opt/openssl@3 # macOS/Homebrew; cmake finds it elsewhere +./wire_probe/run_wire_probe.sh # about two minutes the first time +``` + +It runs five modes (`run`, `fail`, `noend`, `strict-down`, `best-down`) and prints one PASS or FAIL line per fact that the +page states. A FAIL means upstream behaviour changed and the page needs a new look. + +## Run the tests + +```bash +python3 -m unittest discover -s tests -v +``` + +(`pytest tests` also works.) diff --git a/examples/frameworks/ai-sdk-cpp/check_pin.sh b/examples/frameworks/ai-sdk-cpp/check_pin.sh new file mode 100755 index 000000000..452f61cc2 --- /dev/null +++ b/examples/frameworks/ai-sdk-cpp/check_pin.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash +# TH-8251: does ClickHouse/ai-sdk-cpp contain any OTLP / OpenTelemetry export at the pinned commit? +# +# usage: check_pin.sh shallow-clone the pinned commit into a fresh temp dir and search it +# check_pin.sh search an existing git checkout (the tests use this with tiny trees) +# +# exit 0 none of the OTLP terms is present: the claim on the docs page holds +# exit 1 at least one term is present: reopen TH-8251, the page is out of date +# exit 2 the search could not be trusted (clone failed, wrong commit, or a positive control found nothing) +# +# Positive controls are terms that MUST be present (the Langfuse client and the Anthropic header). A search +# that cannot find them proves nothing when it finds zero OTLP files. +set -u +PIN="${CHECK_PIN_SHA:-9d8cd18200a33b64460c9101ed0dd6d4988bea61}" +REPO="${CHECK_PIN_REPO:-https://github.com/ClickHouse/ai-sdk-cpp.git}" +read -r -a TERMS <<< "${CHECK_PIN_TERMS:-otlp opentelemetry exporter traceparent gen_ai /v1/traces}" +read -r -a CONTROLS <<< "${CHECK_PIN_CONTROLS:-langfuse x-api-key}" + +dir="${1:-}" +if [ -z "$dir" ]; then + dir="$(mktemp -d)/ai-sdk-cpp" + mkdir -p "$dir" && git -C "$dir" init -q \ + && git -C "$dir" remote add origin "$REPO" \ + && git -C "$dir" fetch -q --depth 1 origin "$PIN" \ + && git -C "$dir" checkout -q FETCH_HEAD || { echo "clone of $REPO at $PIN failed"; exit 2; } + upstream_main="$(git ls-remote "$REPO" refs/heads/main 2>/dev/null | cut -f1)" + echo "upstream main : ${upstream_main:-unknown}$([ "$upstream_main" = "$PIN" ] && echo ' (same as the pin)')" +fi + +head="$(git -C "$dir" rev-parse HEAD 2>/dev/null)" || { echo "$dir is not a git checkout"; exit 2; } +[ -z "${1:-}" ] && [ "$head" != "$PIN" ] && { echo "checked out $head, expected $PIN"; exit 2; } +echo "commit : $head" +echo "tracked files : $(git -C "$dir" ls-files | wc -l | tr -d ' ')" +echo +count() { git -C "$dir" grep -i -l -F -e "$1" | wc -l | tr -d ' '; } + +found=0 +echo "OTLP terms (case-insensitive, all tracked files; 0 is the expected result)" +for t in "${TERMS[@]}"; do + n="$(count "$t")"; printf ' %-14s %s files\n' "$t" "$n" + [ "$n" != "0" ] && found=1 +done +echo "positive controls (must be above 0 or the search is not trusted)" +bad=0 +for t in "${CONTROLS[@]}"; do + n="$(count "$t")"; printf ' %-14s %s files\n' "$t" "$n" + [ "$n" = "0" ] && bad=1 +done +if [ -f "$dir/src/langfuse/tracer.cpp" ]; then + echo + echo "Langfuse transport:" + grep -n -E 'kIngestionPath = |"Authorization"|X-Langfuse-Sdk' "$dir/src/langfuse/tracer.cpp" | sed 's/^/ /' +fi +echo +[ "$bad" = "1" ] && { echo "RESULT: search not trusted (a positive control found nothing)"; exit 2; } +[ "$found" = "1" ] && { echo "RESULT: OTLP term present, reopen TH-8251"; exit 1; } +echo "RESULT: no OTLP or OpenTelemetry export in this tree" +exit 0 diff --git a/examples/frameworks/ai-sdk-cpp/tests/fixtures/probe_out/best-down.stdout.txt b/examples/frameworks/ai-sdk-cpp/tests/fixtures/probe_out/best-down.stdout.txt new file mode 100644 index 000000000..f4a5a3894 --- /dev/null +++ b/examples/frameworks/ai-sdk-cpp/tests/fixtures/probe_out/best-down.stdout.txt @@ -0,0 +1,4 @@ +tracer.is_valid=1 +model_output=Paris is currently clear and 21°C. +[ERROR] Langfuse ingestion failed: Could not establish connection +end#1=1 end#2=1 trace_id=3d8cc1ab-6222-42c6-b0f3-951cb3fb48c7 diff --git a/examples/frameworks/ai-sdk-cpp/tests/fixtures/probe_out/fail.jsonl b/examples/frameworks/ai-sdk-cpp/tests/fixtures/probe_out/fail.jsonl new file mode 100644 index 000000000..bd424f548 --- /dev/null +++ b/examples/frameworks/ai-sdk-cpp/tests/fixtures/probe_out/fail.jsonl @@ -0,0 +1 @@ +{"ts": 1792136759.594816, "method": "POST", "path": "/api/public/ingestion", "headers": {"Accept-Encoding": "gzip, deflate", "User-Agent": "ai-sdk-cpp-langfuse/0.1", "Content-Type": "application/json", "Connection": "close", "Host": "127.0.0.1:18251", "Content-Length": "1634", "Authorization": "", "Accept": "*/*", "X-Langfuse-Sdk-Variant": "ai-sdk-cpp", "X-Langfuse-Sdk-Name": "ai-sdk-cpp"}, "basic_auth_user": "pk-lf-probe-fake", "content_type": "application/json", "body": {"batch": [{"body": {"environment": "ai-sdk-cpp-probe", "id": "e03a9142-e043-4304-ab8c-8347024879cd", "input": "What is the weather in Paris? Context tag: MARKER-fail", "metadata": {"marker": "MARKER-fail", "probe": "fail"}, "name": "probe_fail", "output": {"error": "Failed after 1 attempts with non-retryable error: {\"error\":{\"message\":\"unknown provider for model no-such-model-probe\",\"type\":\"invalid_request_error\",\"code\":\"model_not_found\",\"param\":\"model\"}}"}, "sessionId": "probe-session", "timestamp": "2026-10-16T07:45:59.575Z", "userId": "probe-user"}, "id": "1ba0a253-057d-410c-9731-130d9c326a82", "timestamp": "2026-10-16T07:45:59.575Z", "type": "trace-create"}, {"body": {"endTime": "2026-10-16T07:45:59.592Z", "environment": "ai-sdk-cpp-probe", "id": "4dddb82c-b76c-4f95-b175-2c74ca8a85f9", "input": [{"content": "You are a concise assistant. Always call get_weather before answering.", "role": "system"}, {"content": "What is the weather in Paris? Context tag: MARKER-fail", "role": "user"}], "level": "ERROR", "metadata": {"finish_reason": "error"}, "model": "no-such-model-probe", "modelParameters": {"max_steps": 4, "temperature": 0.0}, "name": "generate_text", "output": "", "startTime": "2026-10-16T07:45:59.575Z", "statusMessage": "Failed after 1 attempts with non-retryable error: {\"error\":{\"message\":\"unknown provider for model no-such-model-probe\",\"type\":\"invalid_request_error\",\"code\":\"model_not_found\",\"param\":\"model\"}}", "traceId": "e03a9142-e043-4304-ab8c-8347024879cd", "usage": {"input": 0, "output": 0, "total": 0, "unit": "TOKENS"}}, "id": "396ad9b0-d93f-4e70-9255-851f6494e491", "timestamp": "2026-10-16T07:45:59.592Z", "type": "generation-create"}]}} diff --git a/examples/frameworks/ai-sdk-cpp/tests/fixtures/probe_out/fail.stdout.txt b/examples/frameworks/ai-sdk-cpp/tests/fixtures/probe_out/fail.stdout.txt new file mode 100644 index 000000000..ddb77e6de --- /dev/null +++ b/examples/frameworks/ai-sdk-cpp/tests/fixtures/probe_out/fail.stdout.txt @@ -0,0 +1,4 @@ +tracer.is_valid=1 +[ERROR] Request failed after retries: Failed after 1 attempts with non-retryable error: {"error":{"message":"unknown provider for model no-such-model-probe","type":"invalid_request_error","code":"model_not_found","param":"model"}} +generation_failed=Failed after 1 attempts with non-retryable error: {"error":{"message":"unknown provider for model no-such-model-probe","type":"invalid_request_error","code":"model_not_found","param":"model"}} +end#1=1 end#2=1 trace_id=e03a9142-e043-4304-ab8c-8347024879cd diff --git a/examples/frameworks/ai-sdk-cpp/tests/fixtures/probe_out/noend.stdout.txt b/examples/frameworks/ai-sdk-cpp/tests/fixtures/probe_out/noend.stdout.txt new file mode 100644 index 000000000..c35b42ec1 --- /dev/null +++ b/examples/frameworks/ai-sdk-cpp/tests/fixtures/probe_out/noend.stdout.txt @@ -0,0 +1,3 @@ +tracer.is_valid=1 +model_output=Paris is currently clear and 21°C. +exiting without end() trace_id=a5b65c35-d2e1-4220-8551-3ac71c157a02 diff --git a/examples/frameworks/ai-sdk-cpp/tests/fixtures/probe_out/run.jsonl b/examples/frameworks/ai-sdk-cpp/tests/fixtures/probe_out/run.jsonl new file mode 100644 index 000000000..29a9efad1 --- /dev/null +++ b/examples/frameworks/ai-sdk-cpp/tests/fixtures/probe_out/run.jsonl @@ -0,0 +1 @@ +{"ts": 1792136758.840935, "method": "POST", "path": "/api/public/ingestion", "headers": {"Accept-Encoding": "gzip, deflate", "User-Agent": "ai-sdk-cpp-langfuse/0.1", "Content-Type": "application/json", "Connection": "close", "Host": "127.0.0.1:18251", "Content-Length": "1920", "Authorization": "", "Accept": "*/*", "X-Langfuse-Sdk-Variant": "ai-sdk-cpp", "X-Langfuse-Sdk-Name": "ai-sdk-cpp"}, "basic_auth_user": "pk-lf-probe-fake", "content_type": "application/json", "body": {"batch": [{"body": {"environment": "ai-sdk-cpp-probe", "id": "f3ce4181-8877-4214-b129-bd905610530f", "input": "What is the weather in Paris? Context tag: MARKER-run", "metadata": {"marker": "MARKER-run", "probe": "run"}, "name": "probe_run", "output": "Paris: clear skies, 21\u00b0C.", "sessionId": "probe-session", "timestamp": "2026-10-16T07:45:47.946Z", "userId": "probe-user"}, "id": "9a0c128a-b98e-406d-875e-9ebe559ffc93", "timestamp": "2026-10-16T07:45:47.946Z", "type": "trace-create"}, {"body": {"environment": "ai-sdk-cpp-probe", "id": "423f2bb7-9aa7-48b1-a79a-db0b00dae866", "input": {"location": "Paris"}, "name": "get_weather", "parentObservationId": "5d3d2810-a7e6-456c-8542-34095f7fa5bb", "startTime": "2026-10-16T07:45:53.217Z", "traceId": "f3ce4181-8877-4214-b129-bd905610530f"}, "id": "b40e7ceb-e893-420e-9855-13da4ca48238", "timestamp": "2026-10-16T07:45:53.217Z", "type": "span-create"}, {"body": {"endTime": "2026-10-16T07:45:53.219Z", "id": "423f2bb7-9aa7-48b1-a79a-db0b00dae866", "output": {"location": "Paris", "sky": "clear", "temperature_c": 21}, "traceId": "f3ce4181-8877-4214-b129-bd905610530f"}, "id": "f7f34393-896e-41e1-bc34-c6c6863fb02f", "timestamp": "2026-10-16T07:45:53.219Z", "type": "span-update"}, {"body": {"endTime": "2026-10-16T07:45:58.835Z", "environment": "ai-sdk-cpp-probe", "id": "5d3d2810-a7e6-456c-8542-34095f7fa5bb", "input": [{"content": "You are a concise assistant. Always call get_weather before answering.", "role": "system"}, {"content": "What is the weather in Paris? Context tag: MARKER-run", "role": "user"}], "metadata": {"finish_reason": "stop", "steps": 2}, "model": "gpt-6-luna", "modelParameters": {"max_steps": 4, "temperature": 0.0}, "name": "generate_text", "output": "Paris: clear skies, 21\u00b0C.", "startTime": "2026-10-16T07:45:47.946Z", "traceId": "f3ce4181-8877-4214-b129-bd905610530f", "usage": {"input": 783, "output": 43, "total": 826, "unit": "TOKENS"}}, "id": "049dec30-cf6b-4454-8f73-b0e2ea3ab0af", "timestamp": "2026-10-16T07:45:58.835Z", "type": "generation-create"}]}} diff --git a/examples/frameworks/ai-sdk-cpp/tests/fixtures/probe_out/run.stdout.txt b/examples/frameworks/ai-sdk-cpp/tests/fixtures/probe_out/run.stdout.txt new file mode 100644 index 000000000..f4da85b4f --- /dev/null +++ b/examples/frameworks/ai-sdk-cpp/tests/fixtures/probe_out/run.stdout.txt @@ -0,0 +1,3 @@ +tracer.is_valid=1 +model_output=Paris: clear skies, 21°C. +end#1=1 end#2=1 trace_id=f3ce4181-8877-4214-b129-bd905610530f diff --git a/examples/frameworks/ai-sdk-cpp/tests/fixtures/probe_out/strict-down.stdout.txt b/examples/frameworks/ai-sdk-cpp/tests/fixtures/probe_out/strict-down.stdout.txt new file mode 100644 index 000000000..d9fd5f9b8 --- /dev/null +++ b/examples/frameworks/ai-sdk-cpp/tests/fixtures/probe_out/strict-down.stdout.txt @@ -0,0 +1,4 @@ +tracer.is_valid=1 +model_output=Paris is clear and 21°C. +[ERROR] Langfuse ingestion failed: Could not establish connection +end#1=0 end#2=1 trace_id=d4a7a454-942b-4f2c-93c9-5a2a3d969641 diff --git a/examples/frameworks/ai-sdk-cpp/tests/test_capture_stub.py b/examples/frameworks/ai-sdk-cpp/tests/test_capture_stub.py new file mode 100644 index 000000000..0920aab86 --- /dev/null +++ b/examples/frameworks/ai-sdk-cpp/tests/test_capture_stub.py @@ -0,0 +1,43 @@ +"""The loopback capture stub records requests, redacts Authorization, and binds to loopback only.""" +import base64 +import json +import os +import sys +import tempfile +import threading +import unittest +import urllib.request + +HERE = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, os.path.join(HERE, "..", "wire_probe")) +import capture_stub # noqa: E402 + + +class CaptureStubTest(unittest.TestCase): + def test_records_post_and_redacts_authorization(self): + out = os.path.join(tempfile.mkdtemp(), "cap.jsonl") + server = capture_stub.make_server(out, 0) + self.assertEqual(server.server_address[0], "127.0.0.1") + t = threading.Thread(target=server.handle_request) + t.start() + auth = base64.b64encode(b"pk-lf-fake:sk-lf-fake").decode() + req = urllib.request.Request( + f"http://127.0.0.1:{server.server_address[1]}/api/public/ingestion", + data=json.dumps({"batch": [{"type": "trace-create"}]}).encode(), + headers={"Content-Type": "application/json", "Authorization": "Basic " + auth}, + ) + with urllib.request.urlopen(req) as resp: + self.assertEqual(resp.status, 207) + t.join() + server.server_close() + with open(out) as f: + rec = json.loads(f.readline()) + self.assertEqual(rec["path"], "/api/public/ingestion") + self.assertEqual(rec["basic_auth_user"], "pk-lf-fake") + self.assertEqual(rec["headers"]["Authorization"], "") + self.assertNotIn("sk-lf-fake", json.dumps(rec)) + self.assertEqual(rec["body"]["batch"][0]["type"], "trace-create") + + +if __name__ == "__main__": + unittest.main() diff --git a/examples/frameworks/ai-sdk-cpp/tests/test_check_pin.py b/examples/frameworks/ai-sdk-cpp/tests/test_check_pin.py new file mode 100644 index 000000000..0506a7bbf --- /dev/null +++ b/examples/frameworks/ai-sdk-cpp/tests/test_check_pin.py @@ -0,0 +1,60 @@ +"""check_pin.sh against tiny synthetic git trees: it must say yes only when it should (no network).""" +import os +import subprocess +import tempfile +import unittest + +HERE = os.path.dirname(os.path.abspath(__file__)) +SCRIPT = os.path.join(HERE, "..", "check_pin.sh") + + +def make_tree(files): + d = tempfile.mkdtemp() + subprocess.run(["git", "init", "-q", d], check=True) + for path, text in files.items(): + full = os.path.join(d, path) + os.makedirs(os.path.dirname(full), exist_ok=True) + with open(full, "w") as f: + f.write(text) + subprocess.run(["git", "-C", d, "add", "-A"], check=True) + subprocess.run(["git", "-C", d, "-c", "user.name=t", "-c", "user.email=t@example.com", "commit", "-q", "-m", "x"], check=True) + return d + + +def run(tree): + p = subprocess.run(["bash", SCRIPT, tree], capture_output=True, text=True) + return p.returncode, p.stdout + + +CONTROLS = {"include/langfuse.h": "// Langfuse client\n", "src/anthropic.cpp": 'headers["x-api-key"] = key;\n'} + + +class CheckPinTest(unittest.TestCase): + def test_tree_without_otlp_passes(self): + code, out = run(make_tree(dict(CONTROLS))) + self.assertEqual(code, 0, out) + self.assertIn("RESULT: no OTLP or OpenTelemetry export", out) + self.assertIn("otlp 0 files", out) + + def test_positive_control_otlp_in_tree_is_detected(self): + for term, text in [("otlp", "// OTLP exporter\n"), ("opentelemetry", "#include \n"), + ("path", 'auto p = "/v1/traces";\n'), ("exp", "class SpanExporter {};\n")]: + with self.subTest(term=term): + files = dict(CONTROLS) + files["src/tracing.cpp"] = text + code, out = run(make_tree(files)) + self.assertEqual(code, 1, out) + self.assertIn("reopen TH-8251", out) + + def test_search_that_cannot_find_its_controls_is_not_trusted(self): + code, out = run(make_tree({"README.md": "nothing relevant\n"})) + self.assertEqual(code, 2, out) + self.assertIn("search not trusted", out) + + def test_not_a_git_checkout(self): + code, _ = run(tempfile.mkdtemp()) + self.assertEqual(code, 2) + + +if __name__ == "__main__": + unittest.main() diff --git a/examples/frameworks/ai-sdk-cpp/tests/test_check_wire.py b/examples/frameworks/ai-sdk-cpp/tests/test_check_wire.py new file mode 100644 index 000000000..817b72d8e --- /dev/null +++ b/examples/frameworks/ai-sdk-cpp/tests/test_check_wire.py @@ -0,0 +1,84 @@ +"""check_wire.py against the recorded probe output (a real run of the pinned SDK on 2026-10-16), plus mutations.""" +import copy +import json +import os +import shutil +import sys +import tempfile +import unittest + +HERE = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, os.path.join(HERE, "..", "wire_probe")) +import check_wire # noqa: E402 + +FIXTURE = os.path.join(HERE, "fixtures", "probe_out") + + +def fresh_copy(): + d = tempfile.mkdtemp() + for name in os.listdir(FIXTURE): + shutil.copy(os.path.join(FIXTURE, name), d) + return d + + +def read(path): + with open(path) as f: + return f.read() + + +def write(path, text): + with open(path, "w") as f: + f.write(text) + + +def failed(results): + return [name for name, ok, _ in results if not ok] + + +class CheckWireTest(unittest.TestCase): + def test_recorded_run_satisfies_every_expectation(self): + results = check_wire.check(FIXTURE) + self.assertGreaterEqual(len(results), 19) + self.assertEqual(failed(results), []) + + def test_otlp_shaped_body_is_caught(self): + d = fresh_copy() + path = os.path.join(d, "run.jsonl") + rec = json.loads(read(path).splitlines()[0]) + rec["body"] = {"resourceSpans": []} + rec["path"] = "/v1/traces" + write(path, json.dumps(rec) + "\n") + bad = failed(check_wire.check(d)) + self.assertIn("run: path is /api/public/ingestion", bad) + self.assertIn("run: body is JSON with a single 'batch' key (not OTLP)", bad) + + def test_future_agi_style_headers_are_caught(self): + d = fresh_copy() + path = os.path.join(d, "run.jsonl") + rec = json.loads(read(path).splitlines()[0]) + rec["headers"]["X-Api-Key"] = "x" + write(path, json.dumps(rec) + "\n") + self.assertIn("run: no X-Api-Key / X-Secret-Key header", failed(check_wire.check(d))) + + def test_second_post_after_second_end_is_caught(self): + d = fresh_copy() + path = os.path.join(d, "run.jsonl") + line = read(path).splitlines()[0] + "\n" + write(path, line + line) + self.assertIn("run: exactly one POST although end() was called twice", failed(check_wire.check(d))) + + def test_a_flush_without_end_is_caught(self): + d = fresh_copy() + shutil.copy(os.path.join(FIXTURE, "run.jsonl"), os.path.join(d, "noend.jsonl")) + self.assertIn("noend: no POST at all", failed(check_wire.check(d))) + + def test_retry_after_failed_strict_flush_is_caught(self): + d = fresh_copy() + path = os.path.join(d, "strict-down.stdout.txt") + text = read(path).replace("end#1=0 end#2=1", "end#1=0 end#2=0") + write(path, text) + self.assertIn("strict-down: first end() false, second end() true (no retry)", failed(check_wire.check(d))) + + +if __name__ == "__main__": + unittest.main() diff --git a/examples/frameworks/ai-sdk-cpp/tests/test_page.py b/examples/frameworks/ai-sdk-cpp/tests/test_page.py new file mode 100644 index 000000000..1bf966d8e --- /dev/null +++ b/examples/frameworks/ai-sdk-cpp/tests/test_page.py @@ -0,0 +1,60 @@ +"""The docs page is a stop notice. These checks keep it one: pinned facts match the scripts, no setup steps.""" +import os +import re +import unittest + +HERE = os.path.dirname(os.path.abspath(__file__)) +ROOT = os.path.abspath(os.path.join(HERE, "..", "..", "..", "..")) +PAGE = os.path.join(ROOT, "src", "pages", "docs", "integrations", "frameworks", "ai-sdk-cpp.mdx") +SCRIPT = os.path.join(HERE, "..", "check_pin.sh") + + +def read(path): + with open(path) as f: + return f.read() + + +class PageTest(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.page = read(PAGE) + cls.body = cls.page.split("---", 2)[2] + + def test_pin_in_page_matches_the_script(self): + pin = re.search(r'CHECK_PIN_SHA:-([0-9a-f]{40})', read(SCRIPT)).group(1) + self.assertIn(pin, self.page) + + def test_first_heading_is_about(self): + self.assertEqual(re.findall(r"^#{2,3} (.+)$", self.body, re.M)[0], "About") + + def test_says_not_supported(self): + self.assertIn("Not supported", self.page) + self.assertIn("has no setup steps", self.page) + + def test_no_setup_steps_no_code_blocks(self): + self.assertNotIn("```", self.page) + for word in ("pip install", "npm install", "cmake -", "export FI_", "FI_API_KEY", "FI_SECRET_KEY"): + self.assertNotIn(word, self.page) + + def test_makes_no_claim_that_a_trace_arrived(self): + self.assertIn("No AI SDK C++ trace has been sent to Future AGI", self.page) + + def test_no_em_dashes(self): + self.assertNotIn("—", self.page) + + def test_internal_links_resolve_to_pages(self): + pages = os.path.join(ROOT, "src", "pages") + for href in re.findall(r"\]\((/docs[^)#]*)\)", self.page): + rel = href.rstrip("/") + candidates = [os.path.join(pages, rel.lstrip("/") + ".mdx"), os.path.join(pages, rel.lstrip("/"), "index.mdx")] + self.assertTrue(any(os.path.exists(c) for c in candidates), href) + + def test_every_search_in_the_table_is_run_by_the_script(self): + script = read(SCRIPT) + for term in ("otlp", "opentelemetry", "exporter", "traceparent", "gen_ai", "/v1/traces", "langfuse", "x-api-key"): + self.assertIn(f"`{term}`", self.page) + self.assertIn(term, script) + + +if __name__ == "__main__": + unittest.main() diff --git a/examples/frameworks/ai-sdk-cpp/wire_probe/CMakeLists.txt b/examples/frameworks/ai-sdk-cpp/wire_probe/CMakeLists.txt new file mode 100644 index 000000000..8b65ad094 --- /dev/null +++ b/examples/frameworks/ai-sdk-cpp/wire_probe/CMakeLists.txt @@ -0,0 +1,10 @@ +cmake_minimum_required(VERSION 3.16) +project(ai_sdk_cpp_wire_probe CXX) +set(CMAKE_CXX_STANDARD 20) +set(CMAKE_CXX_STANDARD_REQUIRED ON) +# AI_SDK_SRC: a checkout of ClickHouse/ai-sdk-cpp (run_wire_probe.sh passes the pinned commit). +set(BUILD_EXAMPLES OFF CACHE BOOL "" FORCE) +set(BUILD_TESTS OFF CACHE BOOL "" FORCE) +add_subdirectory(${AI_SDK_SRC} ${CMAKE_BINARY_DIR}/ai-sdk-cpp) +add_executable(probe probe.cpp) +target_link_libraries(probe PRIVATE ai::openai ai::langfuse) diff --git a/examples/frameworks/ai-sdk-cpp/wire_probe/capture_stub.py b/examples/frameworks/ai-sdk-cpp/wire_probe/capture_stub.py new file mode 100755 index 000000000..686faeadb --- /dev/null +++ b/examples/frameworks/ai-sdk-cpp/wire_probe/capture_stub.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python3 +"""Loopback stand-in for a Langfuse host. Records every POST to a JSONL file and answers 207. + +usage: capture_stub.py + +Each line: ts, method, path, headers (Authorization value redacted), basic_auth_user, content_type, body. +Binds to 127.0.0.1 only. The keys it sees are the probe's fake ones; the Authorization value is not stored. +""" +import base64 +import json +import sys +import time +from http.server import BaseHTTPRequestHandler, HTTPServer + + +def make_server(out_path, port): + class Handler(BaseHTTPRequestHandler): + def log_message(self, *args): + pass + + def do_POST(self): + raw = self.rfile.read(int(self.headers.get("Content-Length", 0))) + try: + body = json.loads(raw) + except ValueError: + body = {"_unparsed_bytes": len(raw)} + auth = self.headers.get("Authorization", "") + user = "" + if auth.startswith("Basic "): + try: + user = base64.b64decode(auth[6:]).decode().split(":")[0] + except Exception: + pass + record = { + "ts": time.time(), + "method": "POST", + "path": self.path, + "headers": {k: ("" if k.lower() == "authorization" else v) for k, v in self.headers.items()}, + "basic_auth_user": user, + "content_type": self.headers.get("Content-Type"), + "body": body, + } + with open(out_path, "a") as f: + f.write(json.dumps(record) + "\n") + payload = json.dumps({"successes": [], "errors": []}).encode() + self.send_response(207) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(payload))) + self.end_headers() + self.wfile.write(payload) + + return HTTPServer(("127.0.0.1", port), Handler) + + +if __name__ == "__main__": + make_server(sys.argv[1], int(sys.argv[2])).serve_forever() diff --git a/examples/frameworks/ai-sdk-cpp/wire_probe/check_wire.py b/examples/frameworks/ai-sdk-cpp/wire_probe/check_wire.py new file mode 100755 index 000000000..bd7c28436 --- /dev/null +++ b/examples/frameworks/ai-sdk-cpp/wire_probe/check_wire.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python3 +"""Check what the AI SDK C++ Langfuse module put on the wire, from the probe's output directory. + +usage: check_wire.py + + holds, per mode (run, fail, noend, strict-down, best-down): + .stdout.txt the probe's stdout + .jsonl the capture stub's records (absent when no request arrived) +Prints one PASS/FAIL line per expectation and exits 1 if any fails. Expectations are the facts stated on the +docs page; if upstream changes behaviour this fails and the page needs a new look. +""" +import json +import os +import re +import sys + + +def load(out, mode): + path = os.path.join(out, mode + ".jsonl") + if not os.path.exists(path): + return [] + with open(path) as f: + return [json.loads(line) for line in f if line.strip()] + + +def stdout(out, mode): + with open(os.path.join(out, mode + ".stdout.txt")) as f: + return f.read() + + +def end_returns(text): + m = re.search(r"end#1=(\d) end#2=(\d)", text) + return (m.group(1), m.group(2)) if m else None + + +def check(out): + results = [] + + def ok(name, cond, detail=""): + results.append((name, bool(cond), detail)) + + # run: one tool call, one model call chain, end() twice + reqs = load(out, "run") + ok("run: exactly one POST although end() was called twice", len(reqs) == 1, f"{len(reqs)} requests") + if reqs: + r = reqs[0] + batch = r["body"].get("batch", []) + types = [e.get("type") for e in batch] + ok("run: path is /api/public/ingestion", r["path"] == "/api/public/ingestion", r["path"]) + ok("run: body is JSON with a single 'batch' key (not OTLP)", list(r["body"].keys()) == ["batch"], str(list(r["body"].keys()))) + ok("run: Langfuse event types", types == ["trace-create", "span-create", "span-update", "generation-create"], str(types)) + ok("run: Authorization header is HTTP Basic", r["basic_auth_user"] != "", r["basic_auth_user"]) + names = {k.lower() for k in r["headers"]} + ok("run: no X-Api-Key / X-Secret-Key header", not ({"x-api-key", "x-secret-key"} & names), str(sorted(names))) + ok("run: Langfuse SDK marker headers", r["headers"].get("X-Langfuse-Sdk-Name") == "ai-sdk-cpp") + ok("run: no OTLP resourceSpans anywhere in the body", "resourceSpans" not in json.dumps(r["body"])) + gen = next((e["body"] for e in batch if e.get("type") == "generation-create"), {}) + span = next((e["body"] for e in batch if e.get("type") == "span-create"), {}) + usage = gen.get("usage", {}) + ok("run: generation carries model and token usage", bool(gen.get("model")) and usage.get("total", 0) > 0, str(usage)) + ok("run: tool span is a child of the generation", span.get("parentObservationId") == gen.get("id")) + ret = end_returns(stdout(out, "run")) + ok("run: both end() calls return true", ret == ("1", "1"), str(ret)) + + # fail: API error still produces a batch with an ERROR generation + reqs = load(out, "fail") + ok("fail: one POST", len(reqs) == 1, f"{len(reqs)} requests") + if reqs: + gen = next((e["body"] for e in reqs[0]["body"].get("batch", []) if e.get("type") == "generation-create"), {}) + ok("fail: generation level is ERROR", gen.get("level") == "ERROR", str(gen.get("level"))) + ok("fail: statusMessage carries the provider's raw error text", bool(gen.get("statusMessage"))) + + # noend: nothing is sent when the process exits without end() + ok("noend: no POST at all", len(load(out, "noend")) == 0) + ok("noend: probe reached the exit without end()", "exiting without end()" in stdout(out, "noend")) + + # unreachable host + s = stdout(out, "strict-down") + ok("strict-down: first end() false, second end() true (no retry)", end_returns(s) == ("0", "1"), str(end_returns(s))) + b = stdout(out, "best-down") + ok("best-down: both end() true", end_returns(b) == ("1", "1"), str(end_returns(b))) + ok("no mode threw out of end()", not any("end() threw" in stdout(out, m) for m in ("run", "fail", "strict-down", "best-down"))) + return results + + +def main(): + results = check(sys.argv[1]) + for name, passed, detail in results: + print(("PASS " if passed else "FAIL ") + name + (f" [{detail}]" if detail and not passed else "")) + failed = [r for r in results if not r[1]] + print(f"{len(results) - len(failed)}/{len(results)} expectations hold") + return 1 if failed else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/examples/frameworks/ai-sdk-cpp/wire_probe/probe.cpp b/examples/frameworks/ai-sdk-cpp/wire_probe/probe.cpp new file mode 100644 index 000000000..f4235de72 --- /dev/null +++ b/examples/frameworks/ai-sdk-cpp/wire_probe/probe.cpp @@ -0,0 +1,115 @@ +// Wire probe for TH-8251. Runs the real `ai::langfuse` module of ClickHouse/ai-sdk-cpp against a LOOPBACK +// capture stub, with the model call going to a local OpenAI-compatible endpoint, to record what the module +// puts on the wire and how Trace::end() behaves. It is not an integration and it never talks to Future AGI: +// the program refuses any Langfuse host that is not loopback. +// +// probe +// run one tool call + generate_text, end() called twice +// fail unknown model -> API error, trace still ended +// noend like run, but the process exits without end() +// strict-down Langfuse host unreachable, ErrorPolicy::kStrict +// best-down Langfuse host unreachable, ErrorPolicy::kBestEffort +// +// env: PROBE_LF_HOST capture stub, loopback only (default http://127.0.0.1:18251) +// PROBE_OPENAI_BASE OpenAI-compatible base URL without /v1 (default http://127.0.0.1:8317) +// PROBE_API_KEY key for that endpoint +// PROBE_MODEL model id (default gpt-6-luna) +// PROBE_MARKER string placed in the prompt and trace metadata +#include +#include +#include + +#include +#include +#include + +namespace { +std::string env(const char* k, const char* d = "") { + const char* v = std::getenv(k); + return (v && *v) ? v : d; +} + +bool is_loopback(const std::string& url) { + return url.rfind("http://127.0.0.1", 0) == 0 || url.rfind("http://localhost", 0) == 0; +} + +ai::JsonValue get_weather(const ai::JsonValue& args, const ai::ToolExecutionContext&) { + return ai::JsonValue{{"location", args.value("location", std::string{"unknown"})}, + {"temperature_c", 21}, + {"sky", "clear"}}; +} +} // namespace + +int main(int argc, char** argv) { + const std::string mode = argc > 1 ? argv[1] : "run"; + const std::string marker = env("PROBE_MARKER", "MARKER-NONE"); + + std::string host = env("PROBE_LF_HOST", "http://127.0.0.1:18251"); + auto policy = ai::langfuse::Config::ErrorPolicy::kStrict; + if (mode == "strict-down") host = "http://127.0.0.1:1"; + if (mode == "best-down") { + host = "http://127.0.0.1:1"; + policy = ai::langfuse::Config::ErrorPolicy::kBestEffort; + } + if (!is_loopback(host)) { + std::cerr << "refusing Langfuse host " << host << ": the probe only talks to a loopback capture stub\n"; + return 3; + } + + // Fake keys on purpose: the stub accepts anything. + ai::langfuse::Tracer tracer({ + .host = host, + .public_key = "pk-lf-probe-fake", + .secret_key = "sk-lf-probe-fake", + .environment = "ai-sdk-cpp-probe", + .error_policy = policy, + }); + std::cout << "tracer.is_valid=" << tracer.is_valid() << "\n"; + + auto client = ai::openai::create_client(env("PROBE_API_KEY"), env("PROBE_OPENAI_BASE", "http://127.0.0.1:8317")); + if (!client.is_valid()) { + std::cerr << "openai client invalid\n"; + return 1; + } + + ai::ToolSet tools; + tools["get_weather"] = ai::create_tool("Get the current weather for a location", + ai::create_object_schema({{"location", "string"}}), get_weather); + + ai::GenerateOptions options; + options.model = mode == "fail" ? "no-such-model-probe" : env("PROBE_MODEL", "gpt-6-luna"); + options.system = "You are a concise assistant. Always call get_weather before answering."; + options.prompt = "What is the weather in Paris? Context tag: " + marker; + options.tools = std::move(tools); + options.max_steps = 4; + options.temperature = 0.0; + + auto trace = tracer.start_trace("probe_" + mode); + trace->set_input(options.prompt); + trace->set_metadata({{"probe", mode}, {"marker", marker}}); + trace->set_session_id("probe-session"); + trace->set_user_id("probe-user"); + + auto result = ai::langfuse::generate_text(client, std::move(options), *trace); + if (result) { + std::cout << "model_output=" << result.text << "\n"; + trace->set_output(result.text); + } else { + std::cout << "generation_failed=" << result.error_message() << "\n"; + trace->set_output(ai::JsonValue{{"error", result.error_message()}}); + } + + if (mode == "noend") { + std::cout << "exiting without end() trace_id=" << trace->id() << "\n"; + return 0; + } + bool first = false; + try { + first = trace->end(); + } catch (const std::exception& e) { + std::cout << "end() threw: " << e.what() << "\n"; + } + const bool second = trace->end(); + std::cout << "end#1=" << first << " end#2=" << second << " trace_id=" << trace->id() << "\n"; + return 0; +} diff --git a/examples/frameworks/ai-sdk-cpp/wire_probe/run_wire_probe.sh b/examples/frameworks/ai-sdk-cpp/wire_probe/run_wire_probe.sh new file mode 100755 index 000000000..f581fce77 --- /dev/null +++ b/examples/frameworks/ai-sdk-cpp/wire_probe/run_wire_probe.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +# Build the PINNED ClickHouse/ai-sdk-cpp, run its ai::langfuse module against a loopback capture stub, and +# check what it puts on the wire. Nothing is sent to Future AGI or to any Langfuse server. +# +# needs : git, cmake >= 3.16, a C++20 compiler, OpenSSL (set OPENSSL_ROOT_DIR if cmake cannot find it), +# python3, network access to GitHub (the build downloads zlib), and an OpenAI-compatible endpoint +# for the model call. +# env : PROBE_API_KEY (key for the endpoint), PROBE_OPENAI_BASE (default http://127.0.0.1:8317, no /v1), +# PROBE_MODEL (default gpt-6-luna), WORK_DIR (default: a fresh temp dir), STUB_PORT (default 18251) +set -euo pipefail +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PIN="${CHECK_PIN_SHA:-9d8cd18200a33b64460c9101ed0dd6d4988bea61}" +WORK="${WORK_DIR:-$(mktemp -d)}" +PORT="${STUB_PORT:-18251}" +OUT="$WORK/out"; mkdir -p "$OUT" "$WORK/src" + +if [ ! -d "$WORK/src/ai-sdk-cpp/.git" ]; then + mkdir -p "$WORK/src/ai-sdk-cpp" + git -C "$WORK/src/ai-sdk-cpp" init -q + git -C "$WORK/src/ai-sdk-cpp" remote add origin https://github.com/ClickHouse/ai-sdk-cpp.git + git -C "$WORK/src/ai-sdk-cpp" fetch -q --depth 1 origin "$PIN" + git -C "$WORK/src/ai-sdk-cpp" checkout -q FETCH_HEAD +fi +test "$(git -C "$WORK/src/ai-sdk-cpp" rev-parse HEAD)" = "$PIN" || { echo "wrong commit"; exit 2; } + +cmake -S "$HERE" -B "$WORK/build" -DAI_SDK_SRC="$WORK/src/ai-sdk-cpp" -DCMAKE_BUILD_TYPE=Release -Wno-dev >/dev/null +cmake --build "$WORK/build" -j4 --target probe >/dev/null + +export PROBE_LF_HOST="http://127.0.0.1:$PORT" +for mode in run fail noend strict-down best-down; do + rm -f "$OUT/$mode.jsonl" + python3 -I "$HERE/capture_stub.py" "$OUT/$mode.jsonl" "$PORT" & stub=$! + sleep 0.7 + PROBE_MARKER="MARKER-$mode" "$WORK/build/probe" "$mode" > "$OUT/$mode.stdout.txt" 2>&1 || true + kill "$stub" 2>/dev/null || true; wait "$stub" 2>/dev/null || true +done +echo "output in $OUT" +python3 -I "$HERE/check_wire.py" "$OUT" diff --git a/examples/providers/z-ai/.gitignore b/examples/providers/z-ai/.gitignore new file mode 100644 index 000000000..77ac75498 --- /dev/null +++ b/examples/providers/z-ai/.gitignore @@ -0,0 +1,3 @@ +.venv/ +__pycache__/ +*.pyc diff --git a/examples/providers/z-ai/README.md b/examples/providers/z-ai/README.md new file mode 100644 index 000000000..3c12fd6e9 --- /dev/null +++ b/examples/providers/z-ai/README.md @@ -0,0 +1,84 @@ +# Z.ai tracing example + +Runnable companion to the docs page `src/pages/docs/integrations/providers/z-ai.mdx` +(`/docs/integrations/providers/z-ai`). It traces Z.ai general-platform chat completions made with the +stock `openai` Python client and the existing `traceAI-openai` instrumentor. There is no Z.ai +package, no `traceai-zai` and no `zai-sdk` dependency. + +Status: alpha. Verified against the real `api.z.ai` general platform on 2026-10-17 with the free +`glm-4.7-flash` model (see "Live vendor verification" on the docs page). The local stand-in is still +used by the tests and by the default `./run_e2e.sh`, and is not Z.ai. + +## Contents + +| Path | What it is | +| --- | --- | +| `app.py` | The recipe (about 10 lines of integration) plus scenarios: `chat`, `stream`, `tools`, `thinking`, `error` | +| `run_e2e.sh` | Creates a venv, runs the tests, runs `app.py` against a collector, optionally runs the privacy comparison | +| `standin/zai_stub.py` | Local STAND-IN shaped like the Z.ai chat API (`/api/paas/v4/chat/completions`). Not Z.ai | +| `standin/connect_probe.py` | Refusing HTTP proxy that records which host the unmodified client dials. Opens no tunnel | +| `tests/` | pytest suite (see below) | +| `requirements.txt` | Pinned versions the example was tested with | + +## Run it + +Python 3.10 to 3.13 (traceAI-openai 0.1.10 does not support 3.14). + +```bash +export FI_API_KEY=... FI_SECRET_KEY=... +export FI_BASE_URL=http://localhost:4318 # your Future AGI collector (run_e2e.sh defaults to this) +./run_e2e.sh # tests, then app.py against the stand-in. No Z.ai needed +PRIVACY=1 ./run_e2e.sh # also: content capture off vs default, with a unique marker +``` + +Against the real Z.ai API (needs your own account and key; a paid model costs money, and the script +never creates an account): + +```bash +export ZAI_API_KEY=... # Z.ai general platform key, goes to the OpenAI client only +export ZAI_MODEL=... # a current model code from Z.ai's model list +./run_e2e.sh # calls https://api.z.ai/api/paas/v4/ +``` + +Any OpenAI-compatible server can stand in for the target: set `ZAI_BASE_URL`, `ZAI_API_KEY` and +`ZAI_MODEL`. A `ZAI_BASE_URL` that is one of the three GLM Coding Plan URLs +(`https://api.z.ai/api/anthropic`, `https://api.z.ai/api/coding/paas/v4`, `https://api.z.ai/api/v1`, or +anything under them) is refused: that is a different Z.ai product, not this recipe. + +Run a single scenario with `APP_ARGS="--scenario stream"`. `python app.py --help` lists the flags. + +## What the tests cover + +| File | Covers | +| --- | --- | +| `tests/test_recipe.py` | The base URL constant; the three Coding Plan URLs refused (and nothing exported or sent); trace shape (AGENT, LLM, TOOL, one session); model and usage; streaming usage from the last chunk with no `stream_options`; a stream that ends with `finish_reason: network_error`; `thinking` in `extra_body`; error span; provider label `openai`; keys go only where they belong; uncompressed OTLP with `project_name`; an unreachable collector never breaks the request. Runs against both stand-in shapes (documented fields only, and with the OpenAI `object` field) | +| `tests/test_host.py` | The fixture check: the real client built with the documented base URL, no network (mock transport), asserts the URL, the bearer header, the model, `project_name` and the provider label `openai`; the URL with and without the trailing slash; the async client; a Coding Plan URL is traced the same way, which is why the page warns | +| `tests/test_hosts_dialed.py` | With the documented URL, the unmodified client dials exactly `api.z.ai:443` (refusing proxy, nothing leaves the machine), and the span does not record the host | +| `tests/test_privacy.py` | A unique marker in every prompt, answer, tool argument and tool result: present by default, absent from every exported byte with `FI_HIDE_*`; what the hide flags do not remove | +| `tests/test_page.py` | The page has the required sections, URLs and flags; the Python snippet on the page runs as written (base URL swapped for the stand-in) and produces the span the page describes; no credentials in the page or the example | +| `tests/test_stub.py` | The stand-in only claims fields Z.ai documents, and answers the documented error codes | +| `tests/test_zai_sdk.py` | Optional (`pip install zai-sdk`): Z.ai's own `ZaiClient` is not traced by traceAI-openai | + +```bash +.venv/bin/python -m pytest -q +``` + +## What is real and what is a stand-in + +| Piece | Real | Stand-in | +| --- | --- | --- | +| `openai` SDK, `traceAI-openai`, `fi-instrumentation-otel`, OTLP export | yes | | +| Future AGI collector | yours | the tests use a recording collector | +| The Z.ai API | `ZAI_API_KEY` without `ZAI_BASE_URL`; run live 2026-10-17 (chat, stream, tools, thinking, 401, 429) | `standin/zai_stub.py`, shaped from Z.ai's chat completion reference, streaming guide and error table (read 2026-10-15). Nothing in it was captured from a real Z.ai response. Error messages come from the documented table; the body returned for each case on the live API was not checked | +| `api.z.ai` hostname | the client dials it in `tests/test_hosts_dialed.py` | a refusing proxy answers 403; no tunnel opens | + +Proven live 2026-10-17: the `model` value returned, usage on the last stream chunk, reasoning tokens inside +`usage.completion_tokens`, the 401 and 429 error bodies. Still not proven live: `finish_reason: network_error` +streams, cost for GLM ids, paid models, other Z.ai APIs. + +## Versions + +Tested 2026-10-15: traceAI-openai 0.1.10, fi-instrumentation-otel 1.1.0, opentelemetry-sdk 1.45.1, +Python 3.13 with openai 3.31.0 (latest), and Python 3.10 with openai 1.109.1. The suite also passes with +traceAI PR 249 (open, head 88ceb5a) overlaid. zai-sdk 0.2.3 +was used only for `tests/test_zai_sdk.py`. diff --git a/examples/providers/z-ai/app.py b/examples/providers/z-ai/app.py new file mode 100644 index 000000000..f5ea42c2d --- /dev/null +++ b/examples/providers/z-ai/app.py @@ -0,0 +1,206 @@ +#!/usr/bin/env python3 +"""Trace Z.ai general-platform chat completions with traceAI-openai. + +This is the recipe from docs/integrations/providers/z-ai, plus a few small +scenarios so it can be run end to end: + + chat one non-streaming chat completion + stream one streaming call (Z.ai sends usage on the last chunk by itself) + tools a tool round trip: model asks for a tool, the app runs it, the model answers + thinking a call that passes Z.ai's `thinking` field through extra_body + error a call with a wrong API key (the failure case) + +The integration is only these lines (everything else is demo scaffolding): + + trace_provider = register(project_type=ProjectType.OBSERVE, project_name=...) + OpenAIInstrumentor().instrument(tracer_provider=trace_provider) + client = OpenAI(api_key=, base_url="https://api.z.ai/api/paas/v4/") + +Environment + ZAI_API_KEY Z.ai API key. Goes to the OpenAI client only. + ZAI_MODEL A current Z.ai model code. Not frozen here. + ZAI_BASE_URL Override of the base URL. Use it only to point at a local + stand-in; a real setup leaves it unset. The three Z.ai Coding + Plan URLs are refused (see docs: "Coding Plan is not this page"). + FI_API_KEY, FI_SECRET_KEY, FI_BASE_URL Future AGI. Go to the tracer only. + FI_PROJECT_NAME Observe project (default: z-ai). + FI_HIDE_INPUTS, FI_HIDE_OUTPUTS, FI_HIDE_LLM_INVOCATION_PARAMETERS + Set to true to keep prompts, completions and request + parameters out of the spans (traceAI TraceConfig). +""" +import argparse +import json +import os +import sys +import uuid + +import openai +from fi_instrumentation import FITracer, TraceConfig, register, using_session +from fi_instrumentation.fi_types import FiSpanKindValues, ProjectType +from openai import OpenAI +from traceai_openai import OpenAIInstrumentor + +# The one base URL this recipe supports: the general platform OpenAI-compatible endpoint, +# written exactly as Z.ai's OpenAI Python guide prints it (trailing slash included). +ZAI_BASE_URL = "https://api.z.ai/api/paas/v4/" + +# Z.ai's GLM Coding Plan endpoints (docs.z.ai/devpack/quick-start). Different product, not this recipe. +CODING_PLAN_URLS = ( + "https://api.z.ai/api/anthropic", + "https://api.z.ai/api/coding/paas/v4", + "https://api.z.ai/api/v1", +) + +TOOLS = [ + { + "type": "function", + "function": { + "name": "lookup_record", + "description": "Look up a record by a free-text query.", + "parameters": { + "type": "object", + "properties": {"query": {"type": "string"}}, + "required": ["query"], + }, + }, + } +] + + +def lookup_record(query): + return f"record for {query}" + + +def is_coding_plan_url(url): + """True for the three Coding Plan base URLs, or anything under them (any case, trailing slash or not).""" + normal = url.strip().rstrip("/").lower() + return any(normal == u.rstrip("/").lower() or normal.startswith(u.rstrip("/").lower() + "/") for u in CODING_PLAN_URLS) + + +def resolve_base_url(env=None): + """The base URL to use, or (None, message) when it must be refused.""" + env = os.environ if env is None else env + base_url = env.get("ZAI_BASE_URL") or ZAI_BASE_URL + if is_coding_plan_url(base_url): + return None, ( + "refusing a Z.ai Coding Plan URL. The GLM Coding Plan is a different product with its own endpoints " + "and is not covered by this recipe; use the general platform URL " + ZAI_BASE_URL + ) + return base_url, None + + +def _messages(prompt, system): + return ([{"role": "system", "content": system}] if system else []) + [{"role": "user", "content": prompt}] + + +def run_chat(client, model, prompt, system): + reply = client.chat.completions.create(model=model, messages=_messages(prompt, system)) + return reply.choices[0].message.content + + +def run_stream(client, model, prompt, system): + # Z.ai documents `usage` on the last chunk of a stream; the app does not ask for it. + stream = client.chat.completions.create(model=model, messages=_messages(prompt, system), stream=True) + return "".join(part.choices[0].delta.content or "" for part in stream if part.choices) + + +def run_thinking(client, model, prompt, system, mode="enabled"): + # `thinking` is a Z.ai field the OpenAI SDK does not know; extra_body carries it. + reply = client.chat.completions.create( + model=model, + messages=_messages(prompt, system), + extra_body={"thinking": {"type": mode}}, + ) + return reply.choices[0].message.content + + +def run_tools(client, model, prompt, system, tracer): + messages = _messages(prompt, system) + first = client.chat.completions.create(model=model, messages=messages, tools=TOOLS, tool_choice="auto") + message = first.choices[0].message + if not message.tool_calls: + return message.content + call = message.tool_calls[0] + # Your own spans carry only what you put on them. This one records no arguments. + with tracer.start_as_current_span("tool lookup_record", fi_span_kind=FiSpanKindValues.TOOL): + result = lookup_record(**json.loads(call.function.arguments)) + messages += [ + {"role": "assistant", "content": message.content or "", "tool_calls": [c.model_dump() for c in message.tool_calls]}, + {"role": "tool", "tool_call_id": call.id, "content": result}, + ] + final = client.chat.completions.create(model=model, messages=messages, tools=TOOLS) + return final.choices[0].message.content + + +def run_error(base_url, model, prompt): + """A deliberately wrong API key: Z.ai answers 401 and the span is ERROR.""" + bad = OpenAI(api_key="not-a-real-zai-key", base_url=base_url) + try: + bad.chat.completions.create(model=model, messages=[{"role": "user", "content": prompt}]) + except openai.APIStatusError as exc: + return f"HTTP {exc.status_code}" + return "no error (unexpected)" + + +def main(argv=None): + ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + ap.add_argument("--scenario", choices=["all", "chat", "stream", "tools", "thinking", "error"], default="all") + ap.add_argument("--prompt", default="Say hello to Z.ai in five words.") + ap.add_argument("--system", default="") + ap.add_argument("--thinking", choices=["disabled", "enabled"], default="enabled", help="value for the thinking scenario") + ap.add_argument("--session-id", default=None) + a = ap.parse_args(argv) + + api_key = os.environ.get("ZAI_API_KEY") + model = os.environ.get("ZAI_MODEL") + if not api_key or not model: + print("set ZAI_API_KEY and ZAI_MODEL (a current model code from Z.ai's model list)", file=sys.stderr) + return 2 + base_url, problem = resolve_base_url() + if problem: + print(problem, file=sys.stderr) + return 2 + + # 1. Future AGI tracing. The Future AGI key pair is read from FI_API_KEY / FI_SECRET_KEY. + trace_provider = register( + project_type=ProjectType.OBSERVE, + project_name=os.environ.get("FI_PROJECT_NAME", "z-ai"), + ) + OpenAIInstrumentor().instrument(tracer_provider=trace_provider) + + # 2. The OpenAI client, pointed at the Z.ai general platform endpoint. + client = OpenAI(api_key=api_key, base_url=base_url) + + tracer = FITracer(trace_provider.get_tracer("z-ai-example"), config=TraceConfig()) + session_id = a.session_id or f"z-ai-{uuid.uuid4()}" + wanted = ["chat", "stream", "tools", "thinking", "error"] if a.scenario == "all" else [a.scenario] + # Optional app-level parent span and session, so the calls are grouped in Observe. + with using_session(session_id), tracer.start_as_current_span( + "z-ai-demo", fi_span_kind=FiSpanKindValues.AGENT + ) as root: + trace_id = str(uuid.UUID(int=root.get_span_context().trace_id)) + for name in wanted: + try: + if name == "chat": + out = run_chat(client, model, a.prompt, a.system) + elif name == "stream": + out = run_stream(client, model, a.prompt, a.system) + elif name == "tools": + out = run_tools(client, model, a.prompt, a.system, tracer) + elif name == "thinking": + out = run_thinking(client, model, a.prompt, a.system, a.thinking) + else: + out = run_error(base_url, model, a.prompt) + except openai.OpenAIError as exc: # keep the demo going; the span already records the failure + out = f"{type(exc).__name__}" + print(f"RESULT {name}: {out}") + trace_provider.force_flush() + trace_provider.shutdown() + print(f"base_url={base_url}") + print(f"trace_id={trace_id}") + print(f"session_id={session_id}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/examples/providers/z-ai/pytest.ini b/examples/providers/z-ai/pytest.ini new file mode 100644 index 000000000..0e02926da --- /dev/null +++ b/examples/providers/z-ai/pytest.ini @@ -0,0 +1,3 @@ +[pytest] +testpaths = tests +addopts = -p no:cacheprovider diff --git a/examples/providers/z-ai/requirements.txt b/examples/providers/z-ai/requirements.txt new file mode 100644 index 000000000..6152ddbc8 --- /dev/null +++ b/examples/providers/z-ai/requirements.txt @@ -0,0 +1,6 @@ +# Tested 2026-10-15 on Python 3.13 (traceAI-openai 0.1.10 requires Python >=3.9,<3.14). +traceAI-openai==0.1.10 +fi-instrumentation-otel==1.1.0 +openai==3.31.0 +# tests +pytest>=8 diff --git a/examples/providers/z-ai/run_e2e.sh b/examples/providers/z-ai/run_e2e.sh new file mode 100755 index 000000000..4d5a754f9 --- /dev/null +++ b/examples/providers/z-ai/run_e2e.sh @@ -0,0 +1,88 @@ +#!/usr/bin/env bash +# End-to-end run of the Z.ai recipe against a Future AGI collector. +# +# FI_API_KEY=... FI_SECRET_KEY=... FI_BASE_URL=http://localhost:4318 ./run_e2e.sh +# +# What it does +# 1. creates ./.venv (Python 3.10 to 3.13: traceAI-openai needs < 3.14) and installs requirements.txt +# (set VENV_DIR to reuse an existing environment) +# 2. runs the tests (they use the local stand-in, a recording collector and a refusing +# CONNECT probe; no Future AGI and no Z.ai needed) +# 3. runs app.py (chat, stream, tools, thinking, error) with tracing on to FI_BASE_URL +# 4. PRIVACY=1: runs app.py twice more with a unique marker in the prompts, once with content +# capture off (FI_HIDE_*) and once with the default, into -private and -optin, +# and prints the markers so you can search your own Future AGI store for them +# +# Which API it calls +# default the local stand-in in standin/ (NOT Z.ai) +# ZAI_API_KEY + ZAI_MODEL the real Z.ai general platform API (https://api.z.ai/api/paas/v4/). +# Needs your own Z.ai account and key; this script never creates one, +# and a call to a paid model costs money. +# ZAI_BASE_URL=... any OpenAI-compatible base URL (for example a local proxy running a +# real model); combine with ZAI_API_KEY and ZAI_MODEL. +# +# Other knobs: FI_PROJECT_NAME (default z-ai-e2e), SKIP_TESTS=1, APP_ARGS="--scenario chat". +set -euo pipefail + +HERE="$(cd "$(dirname "$0")" && pwd)" +cd "$HERE" +: "${FI_API_KEY:?set FI_API_KEY}" "${FI_SECRET_KEY:?set FI_SECRET_KEY}" +export FI_BASE_URL="${FI_BASE_URL:-http://localhost:4318}" +export FI_PROJECT_NAME="${FI_PROJECT_NAME:-z-ai-e2e}" + +VENV_DIR="${VENV_DIR:-$HERE/.venv}" +if [ ! -x "$VENV_DIR/bin/python" ]; then + PY="${PYTHON:-$(command -v python3.13 || command -v python3.12 || command -v python3.11 || command -v python3.10 || true)}" + [ -n "$PY" ] || { echo "need Python 3.10 to 3.13 on PATH (or set PYTHON)"; exit 2; } + "$PY" -m venv "$VENV_DIR" + "$VENV_DIR/bin/python" -m pip install --quiet --upgrade pip + "$VENV_DIR/bin/python" -m pip install --quiet -r requirements.txt +fi +PYBIN="$VENV_DIR/bin/python" +"$PYBIN" - <<'PY' +import importlib.metadata as m, sys +names = ["traceai-openai", "fi-instrumentation-otel", "openai", "opentelemetry-sdk"] +print("versions: python", sys.version.split()[0], *(f"{n}=={m.version(n)}" for n in names)) +PY + +if [ "${SKIP_TESTS:-0}" != "1" ]; then + "$PYBIN" -m pytest -q +fi + +STUB_PID="" +cleanup() { [ -n "$STUB_PID" ] && kill "$STUB_PID" 2>/dev/null || true; } +trap cleanup EXIT + +if [ -n "${ZAI_API_KEY:-}" ]; then + : "${ZAI_MODEL:?set ZAI_MODEL (a current model code from the Z.ai model list)}" + if [ -n "${ZAI_BASE_URL:-}" ]; then + echo "target: custom base URL $ZAI_BASE_URL (not necessarily Z.ai)" + else + echo "target: REAL Z.ai general platform, https://api.z.ai/api/paas/v4/" + fi +else + PORT="$("$PYBIN" -c 'import socket; s=socket.socket(); s.bind(("127.0.0.1",0)); print(s.getsockname()[1])')" + "$PYBIN" standin/zai_stub.py --port "$PORT" --api-key e2e-stand-in-key >/dev/null 2>&1 & + STUB_PID=$! + for _ in $(seq 1 30); do curl -fs "http://127.0.0.1:$PORT/healthz" >/dev/null 2>&1 && break; sleep 0.2; done + export ZAI_BASE_URL="http://127.0.0.1:$PORT/api/paas/v4/" ZAI_API_KEY=e2e-stand-in-key ZAI_MODEL="${ZAI_MODEL:-glm-example-model}" + echo "target: STAND-IN (not Z.ai) at $ZAI_BASE_URL" +fi + +echo "== app.py -> API, traces -> $FI_BASE_URL project=$FI_PROJECT_NAME" +"$PYBIN" app.py ${APP_ARGS:-} | grep -E '^(RESULT|base_url|trace_id|session_id)' + +if [ "${PRIVACY:-0}" = "1" ]; then + for mode in private optin; do + MARK="PRIV-$(date +%s)-$RANDOM$RANDOM" + if [ "$mode" = "private" ]; then + export FI_HIDE_INPUTS=true FI_HIDE_OUTPUTS=true FI_HIDE_LLM_INVOCATION_PARAMETERS=true + else + unset FI_HIDE_INPUTS FI_HIDE_OUTPUTS FI_HIDE_LLM_INVOCATION_PARAMETERS + fi + echo "== privacy run ($mode), marker $MARK, project ${FI_PROJECT_NAME}-$mode" + FI_PROJECT_NAME="${FI_PROJECT_NAME}-$mode" "$PYBIN" app.py --prompt "find $MARK" --system "system note $MARK" ${APP_ARGS:-} \ + | grep -E '^(RESULT|trace_id)' + echo "MARKER_${mode}=$MARK" + done +fi diff --git a/examples/providers/z-ai/standin/connect_probe.py b/examples/providers/z-ai/standin/connect_probe.py new file mode 100644 index 000000000..85756cbfc --- /dev/null +++ b/examples/providers/z-ai/standin/connect_probe.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python3 +"""A refusing HTTP proxy that records where a client tries to connect. + +Point a process at it with HTTPS_PROXY=http://127.0.0.1:. An HTTPS client +that is configured with the real Z.ai base URL asks the proxy for +`CONNECT api.z.ai:443`. This server writes that target down and answers 403, so +no tunnel is opened and nothing leaves the machine. + +It proves one thing: which host and port the unmodified recipe client dials. +It does not speak to Z.ai and it does not forward anything. +""" +import socketserver +import threading + + +class _Handler(socketserver.StreamRequestHandler): + def handle(self): + line = self.rfile.readline(65536).decode("latin-1").strip() + while True: # drain the header block + header = self.rfile.readline(65536) + if header in (b"\r\n", b"\n", b""): + break + method, _, rest = line.partition(" ") + target = rest.rsplit(" ", 1)[0] + self.server.seen.append({"method": method, "target": target}) + body = b"refused by connect_probe (no tunnel is opened)\n" + self.wfile.write( + b"HTTP/1.1 403 Forbidden\r\nContent-Type: text/plain\r\nConnection: close\r\n" + + f"Content-Length: {len(body)}\r\n\r\n".encode() + + body + ) + + +class ConnectProbe(socketserver.ThreadingTCPServer): + allow_reuse_address = True + daemon_threads = True + + def __init__(self, host="127.0.0.1", port=0): + super().__init__((host, port), _Handler) + self.seen = [] + + @property + def url(self): + return f"http://127.0.0.1:{self.server_address[1]}" + + def start(self): + threading.Thread(target=self.serve_forever, daemon=True).start() + return self + + +if __name__ == "__main__": + probe = ConnectProbe(port=0).start() + print(probe.url, flush=True) + threading.Event().wait() diff --git a/examples/providers/z-ai/standin/zai_stub.py b/examples/providers/z-ai/standin/zai_stub.py new file mode 100644 index 000000000..37ffb91ca --- /dev/null +++ b/examples/providers/z-ai/standin/zai_stub.py @@ -0,0 +1,280 @@ +#!/usr/bin/env python3 +"""Local STAND-IN for the Z.ai general platform Chat Completions API. + +This is NOT Z.ai. It is a small HTTP server shaped like the API that Z.ai +documents at https://docs.z.ai/api-reference/llm/chat-completion +(`POST /api/paas/v4/chat/completions`, `Authorization: Bearer `), +https://docs.z.ai/guides/capabilities/streaming (SSE chunks, `usage` on the last +chunk, `data: [DONE]`) and https://docs.z.ai/api-reference/api-code (error codes +and the `{"error": {"code", "message"}}` shape), all read 2026-10-15. It exists +so the recipe in ../app.py can be exercised without a Z.ai account, a key or any +spend. Nothing here was captured from a real Z.ai response. Fields the +documentation lists are produced; fields it does not list are not. + +Behaviour (deterministic, no model is called): + +* The reply echoes the last user or tool message, so a unique marker in a prompt + shows up in the response, in tool-call arguments and in tool results. +* A request that carries "tools" and no "tool" message yet returns one tool call + whose arguments are {"query": }. Z.ai documents only + `tool_choice: "auto"`; any other value is answered with error 1214. +* `thinking.type == "enabled"` adds `reasoning_content` to the message (or to the + stream deltas), as the Z.ai docs describe. +* A wrong bearer token returns 401 with business code 1000; no Authorization + header returns 401 with code 1001; a model name that does not start with + "glm-" returns 400 with code 1211. The messages are the ones in Z.ai's error + table; the exact body for each case on the live API was not checked. +* A streamed request whose last user message contains `[[abnormal-stream]]` stops after two words + and ends with `finish_reason: "network_error"` instead of an HTTP error, which is how Z.ai + documents an inference that terminates abnormally during a stream (docs.z.ai/api-reference/api-code). +* "stream": true returns SSE exactly like the Z.ai example: no `object` field, + `finish_reason` and `usage` on the final chunk (no `stream_options` needed), + then `data: [DONE]`. `--with-object` adds the OpenAI `object` field to every + response and chunk, in case the live API includes it. + +The server records what it received and serves it at GET /__requests so tests +can assert that Future AGI keys never reach Z.ai. +""" +import argparse +import json +import threading +import time +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +CHAT_PATH = "/api/paas/v4/chat/completions" +ERRORS = { + "1000": (401, "Authentication Failed"), + "1001": (401, "Authentication parameter not received in Header, unable to authenticate"), + "1211": (400, "Unknown Model, please check the model code."), + "1214": (400, "Parameter `tool_choice` is invalid. Please check the documentation."), +} + + +def _count_tokens(text): + return len(str(text).split()) + + +def _message_text(message): + content = message.get("content") + if isinstance(content, list): + return " ".join(p.get("text", "") for p in content if isinstance(p, dict)) + return content or "" + + +class Stub: + def __init__(self, api_key, with_object=False): + self.api_key = api_key + self.with_object = with_object + self.requests = [] + self.lock = threading.Lock() + self.counter = 0 + + def next_id(self): + with self.lock: + self.counter += 1 + return self.counter + + def record(self, path, headers, body): + lowered = {k.lower(): v for k, v in headers.items()} + auth = lowered.get("authorization") or "" + with self.lock: + self.requests.append( + { + "path": path, + "header_names": sorted(lowered), + "authorization_scheme": auth.split(" ", 1)[0] if auth else None, + "authorization_matches": auth == f"Bearer {self.api_key}", + "body": body, + } + ) + + +def _plan(body): + """What the stub answers: ('tool', args) or ('text', reply, reasoning).""" + messages = body.get("messages") or [] + last = messages[-1] if messages else {} + text = _message_text(last) + has_tool_result = any(m.get("role") == "tool" for m in messages) + thinking = (body.get("thinking") or {}).get("type") == "enabled" + if body.get("tools") and not has_tool_result: + return ("tool", {"query": text}, None) + if last.get("role") == "tool": + reply = f"The lookup returned: {text}" + else: + reply = f"Z.ai stand-in reply to: {text}" + reasoning = f"Stand-in reasoning about: {text}" if thinking else None + return ("text", reply, reasoning) + + +def _usage(body, plan): + prompt = sum(_count_tokens(_message_text(m)) for m in body.get("messages") or []) or 1 + if plan[0] == "tool": + completion = _count_tokens(json.dumps(plan[1])) + else: + completion = _count_tokens(plan[1]) + (_count_tokens(plan[2]) if plan[2] else 0) + return { + "prompt_tokens": prompt, + "completion_tokens": completion, + "prompt_tokens_details": {"cached_tokens": 0}, + "total_tokens": prompt + completion, + } + + +def build_response(stub, body, n): + plan = _plan(body) + message = {"role": "assistant"} + if plan[0] == "tool": + message["content"] = "" + message["tool_calls"] = [ + { + "id": f"call_{n}", + "type": "function", + "index": 0, + "function": {"name": body["tools"][0]["function"]["name"], "arguments": json.dumps(plan[1])}, + } + ] + finish = "tool_calls" + else: + message["content"] = plan[1] + if plan[2]: + message["reasoning_content"] = plan[2] + finish = "stop" + out = { + "id": str(n), + "request_id": f"stub-req-{n:06d}", + "created": int(time.time()), + "model": body.get("model"), + "choices": [{"index": 0, "message": message, "finish_reason": finish}], + "usage": _usage(body, plan), + } + if stub.with_object: + out["object"] = "chat.completion" + return out + + +def build_stream(stub, body, n): + plan = _plan(body) + base = {"id": str(n), "created": int(time.time()), "model": body.get("model")} + if stub.with_object: + base["object"] = "chat.completion.chunk" + chunks = [] + + def chunk(delta, finish=None, usage=None): + c = dict(base) + c["choices"] = [{"index": 0, "delta": delta, "finish_reason": finish}] if finish is None else [ + {"index": 0, "finish_reason": finish, "delta": delta} + ] + if usage is not None: + c["usage"] = usage + return c + + if plan[0] == "tool": + call = { + "id": f"call_{n}", + "type": "function", + "index": 0, + "function": {"name": body["tools"][0]["function"]["name"], "arguments": json.dumps(plan[1])}, + } + chunks.append(chunk({"role": "assistant", "content": "", "tool_calls": [call]})) + final_delta, finish = {"role": "assistant", "content": ""}, "tool_calls" + else: + if plan[2]: + for word in plan[2].split(" "): + chunks.append(chunk({"reasoning_content": word + " "})) + words = plan[1].split(" ") + abnormal = "[[abnormal-stream]]" in _message_text((body.get("messages") or [{}])[-1]) + if abnormal: + words = words[:2] + for i, word in enumerate(words): + chunks.append(chunk({"content": word + (" " if i < len(words) - 1 else "")})) + final_delta, finish = {"role": "assistant", "content": ""}, ("network_error" if abnormal else "stop") + chunks.append(chunk(final_delta, finish=finish, usage=_usage(body, plan))) + return chunks + + +def make_handler(stub): + class Handler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def log_message(self, *args): + pass + + def _send(self, status, payload, content_type="application/json"): + data = payload if isinstance(payload, bytes) else json.dumps(payload).encode() + self.send_response(status) + self.send_header("Content-Type", content_type) + self.send_header("Content-Length", str(len(data))) + self.end_headers() + self.wfile.write(data) + + def _error(self, code): + status, message = ERRORS[code] + self._send(status, {"error": {"code": code, "message": message}}) + + def do_GET(self): + if self.path == "/healthz": + return self._send(200, {"ok": True}) + if self.path == "/__requests": + with stub.lock: + return self._send(200, list(stub.requests)) + self._send(404, {"error": {"code": "1222", "message": "API `x` does not exist"}}) + + def do_POST(self): + raw = self.rfile.read(int(self.headers.get("Content-Length") or 0)) + try: + body = json.loads(raw or b"{}") + except ValueError: + body = {} + stub.record(self.path, self.headers, body) + if self.path != CHAT_PATH: + return self._send(404, {"error": {"code": "1222", "message": f"API `{self.path}` does not exist"}}) + auth = self.headers.get("Authorization") + if not auth: + return self._error("1001") + if auth != f"Bearer {stub.api_key}": + return self._error("1000") + if not str(body.get("model", "")).startswith("glm-"): + return self._error("1211") + if body.get("tool_choice") not in (None, "auto"): + return self._error("1214") + n = stub.next_id() + if not body.get("stream"): + return self._send(200, build_response(stub, body, n)) + self.send_response(200) + self.send_header("Content-Type", "text/event-stream; charset=UTF-8") + self.send_header("Cache-Control", "no-cache") + self.send_header("Transfer-Encoding", "chunked") + self.end_headers() + + def write(data): + self.wfile.write(f"{len(data):x}\r\n".encode() + data + b"\r\n") + + for c in build_stream(stub, body, n): + write(f"data: {json.dumps(c)}\n\n".encode()) + write(b"data: [DONE]\n\n") + self.wfile.write(b"0\r\n\r\n") + + return Handler + + +def serve(api_key, host="127.0.0.1", port=0, with_object=False): + stub = Stub(api_key, with_object=with_object) + server = ThreadingHTTPServer((host, port), make_handler(stub)) + server.daemon_threads = True + server.stub = stub + return server + + +def main(argv=None): + ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + ap.add_argument("--port", type=int, default=0) + ap.add_argument("--api-key", default="e2e-stand-in-key") + ap.add_argument("--with-object", action="store_true", help="add the OpenAI `object` field to responses and chunks") + a = ap.parse_args(argv) + server = serve(a.api_key, port=a.port, with_object=a.with_object) + print(f"http://127.0.0.1:{server.server_address[1]}/api/paas/v4/", flush=True) + server.serve_forever() + + +if __name__ == "__main__": + main() diff --git a/examples/providers/z-ai/tests/conftest.py b/examples/providers/z-ai/tests/conftest.py new file mode 100644 index 000000000..fc83c7243 --- /dev/null +++ b/examples/providers/z-ai/tests/conftest.py @@ -0,0 +1,201 @@ +"""Shared fixtures: the Z.ai-shaped stand-in, a recording OTLP collector, a refusing +CONNECT probe, and a runner that launches ../app.py exactly as a customer would (a +separate process).""" +import gzip +import json +import os +import subprocess +import sys +import threading +import urllib.request +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path + +import openai +import pytest +from google.protobuf.json_format import MessageToDict +from opentelemetry.proto.collector.trace.v1 import trace_service_pb2 + +HERE = Path(__file__).resolve().parent +ROOT = HERE.parent +sys.path.insert(0, str(ROOT / "standin")) +sys.path.insert(0, str(ROOT)) + +import connect_probe # noqa: E402 +import zai_stub # noqa: E402 + +# openai 3.x builds its HTTP client on `httpx2`; openai 1.x and 2.x use `httpx`. Tests that hand the +# client a mock transport need the module the installed SDK is built on. +if int(openai.__version__.split(".")[0]) >= 3: + import httpx2 as httpx +else: + import httpx + +ZAI_KEY = "test-zai-key-0001" +FI_API_KEY = "test-fi-api-key-0002" +FI_SECRET_KEY = "test-fi-secret-key-0003" +MODEL = "glm-test-model" +ZAI_URL = "https://api.z.ai/api/paas/v4/" +CODING_PLAN_URLS = ( + "https://api.z.ai/api/anthropic", + "https://api.z.ai/api/coding/paas/v4", + "https://api.z.ai/api/v1", +) + + +class Collector: + """Records every OTLP/HTTP trace export it receives, like fi-collector would see it.""" + + def __init__(self): + self.exports = [] + self.lock = threading.Lock() + collector = self + + class Handler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def log_message(self, *args): + pass + + def do_POST(self): + body = self.rfile.read(int(self.headers.get("Content-Length") or 0)) + with collector.lock: + collector.exports.append( + {"path": self.path, "headers": {k.lower(): v for k, v in self.headers.items()}, "body": body} + ) + out = trace_service_pb2.ExportTraceServiceResponse().SerializeToString() + self.send_response(200) + self.send_header("Content-Type", "application/x-protobuf") + self.send_header("Content-Length", str(len(out))) + self.end_headers() + self.wfile.write(out) + + self.server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + self.url = f"http://127.0.0.1:{self.server.server_address[1]}" + threading.Thread(target=self.server.serve_forever, daemon=True).start() + + def close(self): + self.server.shutdown() + + # decoded views ----------------------------------------------------------- + def requests(self): + out = [] + for e in self.exports: + body = e["body"] + if e["headers"].get("content-encoding") == "gzip": + body = gzip.decompress(body) + msg = trace_service_pb2.ExportTraceServiceRequest() + msg.ParseFromString(body) + out.append(MessageToDict(msg, preserving_proto_field_name=True)) + return out + + def spans(self): + found = [] + for req in self.requests(): + for rs in req.get("resource_spans", []): + resource = {a["key"]: _any(a["value"]) for a in rs.get("resource", {}).get("attributes", [])} + for ss in rs.get("scope_spans", []): + for sp in ss.get("spans", []): + sp = dict(sp) + sp["_resource"] = resource + sp["_attrs"] = {a["key"]: _any(a["value"]) for a in sp.get("attributes", [])} + found.append(sp) + return found + + def raw_texts(self): + """Every byte the collector received, as text, for a marker search.""" + return [e["body"].decode("utf-8", "replace") for e in self.exports] + [ + json.dumps(r) for r in self.requests() + ] + + +def _any(value): + for k in ("string_value", "int_value", "double_value", "bool_value"): + if k in value: + return value[k] + return json.dumps(value) + + +def kind(span): + return span["_attrs"].get("gen_ai.span.kind") + + +def llm_spans(collector): + return sorted((s for s in collector.spans() if kind(s) == "LLM"), key=lambda s: int(s["start_time_unix_nano"])) + + +@pytest.fixture(params=[False, True], ids=["documented-shape", "with-object-field"]) +def stub(request): + server = zai_stub.serve(api_key=ZAI_KEY, with_object=request.param) + threading.Thread(target=server.serve_forever, daemon=True).start() + server.root = f"http://127.0.0.1:{server.server_address[1]}" + server.url = server.root + "/api/paas/v4/" # same shape as the documented base URL, trailing slash included + yield server + server.shutdown() + + +@pytest.fixture() +def plain_stub(): + """The stand-in in its documented shape only (for tests that do not need both shapes).""" + server = zai_stub.serve(api_key=ZAI_KEY) + threading.Thread(target=server.serve_forever, daemon=True).start() + server.root = f"http://127.0.0.1:{server.server_address[1]}" + server.url = server.root + "/api/paas/v4/" + yield server + server.shutdown() + + +@pytest.fixture() +def collector(): + c = Collector() + yield c + c.close() + + +@pytest.fixture() +def probe(): + p = connect_probe.ConnectProbe().start() + yield p + p.shutdown() + + +def clean_env(): + drop = ("FI_", "ZAI_", "OPENAI", "OTEL_", "HTTP_PROXY", "HTTPS_PROXY", "ALL_PROXY", "NO_PROXY", "http_proxy", "https_proxy", "all_proxy", "no_proxy") + return {k: v for k, v in os.environ.items() if not k.startswith(drop)} + + +@pytest.fixture() +def run_app(stub, collector): + def _run(*args, env=None, base_url=None, fi_base_url=None, use_stub=True): + e = clean_env() + e.update( + { + "ZAI_API_KEY": ZAI_KEY, + "ZAI_MODEL": MODEL, + "FI_API_KEY": FI_API_KEY, + "FI_SECRET_KEY": FI_SECRET_KEY, + "FI_BASE_URL": fi_base_url or collector.url, + "FI_PROJECT_NAME": "z-ai-test", + "OTEL_EXPORTER_OTLP_TRACES_TIMEOUT": "3", + "PYTHONDONTWRITEBYTECODE": "1", + "NO_PROXY": "127.0.0.1,localhost", + } + ) + if use_stub: + e["ZAI_BASE_URL"] = base_url or stub.url + e.update(env or {}) + return subprocess.run( + [sys.executable, str(ROOT / "app.py"), *args], + cwd=ROOT, + env=e, + capture_output=True, + text=True, + timeout=120, + ) + + return _run + + +def zai_requests(stub): + with urllib.request.urlopen(stub.root + "/__requests") as r: + return json.load(r) diff --git a/examples/providers/z-ai/tests/test_host.py b/examples/providers/z-ai/tests/test_host.py new file mode 100644 index 000000000..e688326b7 --- /dev/null +++ b/examples/providers/z-ai/tests/test_host.py @@ -0,0 +1,119 @@ +"""The fixture check for the documented base URL: the real `openai` client built with the exact +string from the docs page, the real instrumentor, and no network. Nothing here calls api.z.ai. + +Asserts what the architecture asks of the TH-8339-style fixture: the base URL, the model, the +project_name and the provider label.""" +import asyncio + +import pytest +import zai_stub +from conftest import MODEL, ZAI_KEY, ZAI_URL, httpx +from fi_instrumentation import register +from fi_instrumentation.fi_types import ProjectType +from openai import AsyncOpenAI, OpenAI +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +from traceai_openai import OpenAIInstrumentor + + +@pytest.fixture() +def traced(monkeypatch): + monkeypatch.setenv("FI_API_KEY", "test-fi-api-key-0002") + monkeypatch.setenv("FI_SECRET_KEY", "test-fi-secret-key-0003") + exporter = InMemorySpanExporter() + provider = register( + project_type=ProjectType.OBSERVE, + project_name="z-ai-fixture", + batch=False, + span_exporter=exporter, + verbose=False, + ) + instrumentor = OpenAIInstrumentor() + instrumentor.instrument(tracer_provider=provider) + yield exporter + instrumentor.uninstrument() + provider.shutdown() + + +def handler(seen): + stub = zai_stub.Stub(ZAI_KEY) + + def _handle(request): + import json + + seen.append(request) + body = json.loads(request.content) + return httpx.Response(200, json=zai_stub.build_response(stub, body, 1)) + + return _handle + + +def test_documented_base_url_model_project_and_provider(traced): + seen = [] + client = OpenAI( + api_key=ZAI_KEY, + base_url="https://api.z.ai/api/paas/v4/", + http_client=httpx.Client(transport=httpx.MockTransport(handler(seen))), + ) + reply = client.chat.completions.create(model=MODEL, messages=[{"role": "user", "content": "What is the capital of France?"}]) + assert reply.choices[0].message.content == "Z.ai stand-in reply to: What is the capital of France?" + + # the request the client built from the documented string + (request,) = seen + assert str(request.url) == "https://api.z.ai/api/paas/v4/chat/completions" + assert request.method == "POST" + assert request.headers["authorization"] == f"Bearer {ZAI_KEY}" + assert str(client.base_url) == ZAI_URL + assert client.base_url.host == "api.z.ai" + + (span,) = traced.get_finished_spans() + attrs = dict(span.attributes) + assert attrs["gen_ai.request.model"] == MODEL + assert attrs["gen_ai.provider.name"] == "openai" + assert attrs["gen_ai.span.kind"] == "LLM" + assert span.resource.attributes["project_name"] == "z-ai-fixture" + # the host is not recorded anywhere on the span or its resource + assert "z.ai" not in repr(attrs) and "z.ai" not in repr(dict(span.resource.attributes)) + + +def test_the_url_without_the_trailing_slash_reaches_the_same_path(traced): + seen = [] + client = OpenAI( + api_key=ZAI_KEY, + base_url="https://api.z.ai/api/paas/v4", + http_client=httpx.Client(transport=httpx.MockTransport(handler(seen))), + ) + client.chat.completions.create(model=MODEL, messages=[{"role": "user", "content": "hi"}]) + assert str(seen[0].url) == "https://api.z.ai/api/paas/v4/chat/completions" + + +def test_the_async_client_is_traced_the_same_way(traced): + seen = [] + + async def go(): + client = AsyncOpenAI( + api_key=ZAI_KEY, + base_url="https://api.z.ai/api/paas/v4/", + http_client=httpx.AsyncClient(transport=httpx.MockTransport(handler(seen))), + ) + return await client.chat.completions.create(model=MODEL, messages=[{"role": "user", "content": "hi"}]) + + asyncio.run(go()) + assert str(seen[0].url) == "https://api.z.ai/api/paas/v4/chat/completions" + (span,) = traced.get_finished_spans() + assert dict(span.attributes)["gen_ai.request.model"] == MODEL + assert dict(span.attributes)["gen_ai.provider.name"] == "openai" + + +def test_a_coding_plan_base_url_would_be_traced_too_which_is_why_the_page_warns(traced): + """The instrumentor wraps OpenAI.request, so it cannot tell the products apart. This is the reason + the page names the three Coding Plan URLs: tracing would work, and the bill would differ.""" + seen = [] + client = OpenAI( + api_key=ZAI_KEY, + base_url="https://api.z.ai/api/coding/paas/v4", + http_client=httpx.Client(transport=httpx.MockTransport(handler(seen))), + ) + client.chat.completions.create(model=MODEL, messages=[{"role": "user", "content": "hi"}]) + assert str(seen[0].url) == "https://api.z.ai/api/coding/paas/v4/chat/completions" + (span,) = traced.get_finished_spans() + assert dict(span.attributes)["gen_ai.provider.name"] == "openai" diff --git a/examples/providers/z-ai/tests/test_hosts_dialed.py b/examples/providers/z-ai/tests/test_hosts_dialed.py new file mode 100644 index 000000000..eff16c3c0 --- /dev/null +++ b/examples/providers/z-ai/tests/test_hosts_dialed.py @@ -0,0 +1,33 @@ +"""Where the unmodified recipe client dials when it uses the documented Z.ai URL. + +No request leaves the machine: HTTPS_PROXY points at a probe that records the +`CONNECT host:port` the client asks for and answers 403. That is the strongest statement +possible about api.z.ai without a Z.ai account: the documented base URL makes the OpenAI +client dial exactly that host, on 443.""" +from conftest import kind, llm_spans + + +def test_the_documented_url_dials_exactly_api_z_ai_443(run_app, collector, probe): + result = run_app("--scenario", "chat", env={"HTTPS_PROXY": probe.url}, use_stub=False) + assert result.returncode == 0, result.stderr[-1500:] + assert "RESULT chat: APIConnectionError" in result.stdout + assert "base_url=https://api.z.ai/api/paas/v4/" in result.stdout + assert probe.seen, "the client never tried to connect" + assert {p["target"] for p in probe.seen} == {"api.z.ai:443"} + assert {p["method"] for p in probe.seen} == {"CONNECT"} + + +def test_the_connection_failure_is_an_error_span_labelled_openai(run_app, collector, probe): + assert run_app("--scenario", "chat", env={"HTTPS_PROXY": probe.url}, use_stub=False).returncode == 0 + (span,) = llm_spans(collector) + assert span["status"]["code"] == "STATUS_CODE_ERROR" + assert "APIConnectionError" in span["status"]["message"] + assert span["_attrs"]["gen_ai.provider.name"] == "openai" + assert collector.spans()[0]["_resource"]["project_name"] == "z-ai-test" + assert {kind(s) for s in collector.spans()} == {"AGENT", "LLM"} + + +def test_the_exported_bytes_do_not_name_the_zai_host(run_app, collector, probe): + assert run_app("--scenario", "chat", env={"HTTPS_PROXY": probe.url}, use_stub=False).returncode == 0 + exported = "\n".join(collector.raw_texts()) + assert "api.z.ai" not in exported diff --git a/examples/providers/z-ai/tests/test_page.py b/examples/providers/z-ai/tests/test_page.py new file mode 100644 index 000000000..9bf36ee40 --- /dev/null +++ b/examples/providers/z-ai/tests/test_page.py @@ -0,0 +1,100 @@ +"""The docs page and the code on it. + +The Python snippet on src/pages/docs/integrations/providers/z-ai.mdx is run exactly as written, +with only two substitutions (the placeholder model and the Z.ai base URL, which is swapped for the +local stand-in), and must produce the span the page describes.""" +import re +import subprocess +import sys + +import pytest +from conftest import CODING_PLAN_URLS, FI_API_KEY, FI_SECRET_KEY, MODEL, ROOT, ZAI_KEY, ZAI_URL, clean_env, kind, llm_spans + +PAGE = ROOT.parents[2] / "src" / "pages" / "docs" / "integrations" / "providers" / "z-ai.mdx" + + +@pytest.fixture(scope="module") +def page(): + return PAGE.read_text() + + +def snippets(text, lang): + return re.findall(rf"```{lang}\n(.*?)```", text, flags=re.S) + + +def test_the_page_has_the_required_sections(page): + for heading in ( + "## About", + "## Install", + "## Trace a chat completion", + "### Coding Plan is not this page", + "### Z.ai's own SDK is not traced", + "## What you get", + "## What is not captured", + "## Privacy", + "## Run it yourself", + "## Troubleshooting", + "## How this differs from the OpenAI page", + ): + assert heading in page, heading + assert page.startswith('---\ntitle: "Trace Z.ai chat completions"') + + +def test_the_page_states_version_url_and_provider_label(page): + assert "traceAI-openai 0.1.10" in page + assert page.count(f'base_url="{ZAI_URL}"') == 1 + assert "pip install traceAI-openai openai" in page + assert "`openai`, not Z.ai" in page + + +def test_the_page_names_the_three_coding_plan_urls_and_zai_sdk(page): + for url in CODING_PLAN_URLS: + assert f"`{url}`" in page, url + assert "zai-sdk" in page and "ZaiClient" in page + assert "traceai-zai" in page # said not to exist + assert "pip install zai-sdk" not in page, "the recipe must not depend on zai-sdk" + + +def test_the_page_flags_the_stand_in_and_the_unverified_parts(page): + assert "`STANDIN`" in page + assert "Verified against the real Z.ai general platform" in page + assert "Still not proven live" in page + assert "Live vendor verification (2026-10-17)" in page + + +def test_the_page_names_every_hide_setting_the_example_uses(page): + for name in ("FI_HIDE_INPUTS", "FI_HIDE_OUTPUTS", "FI_HIDE_LLM_INVOCATION_PARAMETERS"): + assert name in page + + +def test_no_credentials_in_the_page_or_the_example(): + secrets = re.compile(r"(sk-[A-Za-z0-9]{16,}|Bearer [A-Za-z0-9._-]{20,}|[0-9a-f]{32}\.[A-Za-z0-9]{8,})") + for path in [PAGE, *ROOT.rglob("*")]: + if path.is_file() and path.suffix in {".mdx", ".py", ".sh", ".md", ".txt"} and ".venv" not in path.parts: + assert not secrets.search(path.read_text()), path + + +def test_the_python_snippet_runs_as_written(page, stub, collector): + (code,) = [s for s in snippets(page, "python") if "OpenAIInstrumentor" in s] + assert code.count(f'base_url="{ZAI_URL}"') == 1 and code.count('model="your-model"') == 1 + code = code.replace(f'base_url="{ZAI_URL}"', f'base_url="{stub.url}"').replace('model="your-model"', f'model="{MODEL}"') + code += "\ntrace_provider.force_flush()\ntrace_provider.shutdown()\n" + env = clean_env() + env.update( + { + "ZAI_API_KEY": ZAI_KEY, + "FI_API_KEY": FI_API_KEY, + "FI_SECRET_KEY": FI_SECRET_KEY, + "FI_BASE_URL": collector.url, + "NO_PROXY": "127.0.0.1,localhost", + "OTEL_EXPORTER_OTLP_TRACES_TIMEOUT": "3", + } + ) + done = subprocess.run([sys.executable, "-I", "-c", code], env=env, capture_output=True, text=True, timeout=120, cwd="/") + assert done.returncode == 0, done.stderr[-1500:] + assert "Z.ai stand-in reply to: What is the capital of France?" in done.stdout + (llm,) = llm_spans(collector) + assert llm["_attrs"]["gen_ai.request.model"] == MODEL + assert llm["_attrs"]["gen_ai.provider.name"] == "openai" + assert llm["_resource"]["project_name"] == "my-chatbot" + assert {kind(s) for s in collector.spans()} == {"LLM"} diff --git a/examples/providers/z-ai/tests/test_privacy.py b/examples/providers/z-ai/tests/test_privacy.py new file mode 100644 index 000000000..501cd8d6f --- /dev/null +++ b/examples/providers/z-ai/tests/test_privacy.py @@ -0,0 +1,89 @@ +"""What leaves the process, and what the standard traceAI hide settings keep out. + +A unique marker is placed in the system prompt and the user prompt. The stand-in echoes it into +the model answer, the tool-call arguments and (through the app's tool) the tool result, so it +travels through every field a real conversation fills. The recording collector sees every byte +the process exports.""" +import json +import uuid + +from conftest import kind, llm_spans + +HIDE = { + "FI_HIDE_INPUTS": "true", + "FI_HIDE_OUTPUTS": "true", + "FI_HIDE_LLM_INVOCATION_PARAMETERS": "true", +} + + +def run_with_marker(run_app, env=None): + marker = f"ZMARK-{uuid.uuid4().hex}" + result = run_app("--prompt", f"find {marker}", "--system", f"system note {marker}", env=env) + assert result.returncode == 0, result.stderr[-1500:] + return marker, result + + +def test_content_is_captured_by_default(run_app, collector): + marker, _ = run_with_marker(run_app) + exported = "\n".join(collector.raw_texts()) + assert exported.count(marker) > 0 + spans = llm_spans(collector) + attrs = [s["_attrs"] for s in spans] + # system prompt, user prompt, answers, tool arguments and tool result all carry the marker + joined = lambda key: " ".join(str(a.get(key, "")) for a in attrs) # noqa: E731 + assert f"system note {marker}" in joined("gen_ai.input.messages.0.message.content") + assert marker in joined("gen_ai.input.messages.1.message.content") + assert marker in joined("output.value") + assert marker in joined("gen_ai.output.messages.0.message.tool_calls.0.tool_call.function.arguments") + assert marker in joined("gen_ai.input.messages.3.message.content") # the tool result, in the second tool-round call + + +def test_hide_settings_keep_the_marker_out_of_every_exported_byte(run_app, collector): + marker, _ = run_with_marker(run_app, env=HIDE) + for text in collector.raw_texts(): + assert marker not in text + assert collector.exports, "spans must still be exported" + for span in llm_spans(collector): + a = span["_attrs"] + assert a.get("input.value") in (None, "__REDACTED__") + assert a.get("output.value") in (None, "__REDACTED__") + assert "gen_ai.request.parameters" not in a + assert all(v == "__REDACTED__" for k, v in a.items() if k.startswith("gen_ai.input.messages.") and k.endswith(".content")) + + +def test_hide_settings_keep_the_model_tokens_status_and_timing(run_app, collector): + run_with_marker(run_app, env=HIDE) + ok = [s for s in llm_spans(collector) if s["status"]["code"] == "STATUS_CODE_OK"] + assert ok + for span in ok: + a = span["_attrs"] + assert int(a["gen_ai.usage.total_tokens"]) > 0 + assert a["gen_ai.provider.name"] == "openai" + assert int(span["end_time_unix_nano"]) > int(span["start_time_unix_nano"]) + assert any(s["_attrs"].get("gen_ai.request.model") == "glm-test-model" for s in ok) + assert any(s["status"]["code"] == "STATUS_CODE_ERROR" for s in llm_spans(collector)) + + +def test_the_opt_in_is_just_unsetting_the_variables(run_app, collector): + marker, _ = run_with_marker(run_app, env={k: "false" for k in HIDE}) + assert marker in "\n".join(collector.raw_texts()) + + +def test_what_the_hide_settings_do_not_remove(run_app, collector): + """Pinned so the docs page stays honest about traceAI-openai 0.1.10.""" + run_with_marker(run_app, env=HIDE) + spans = llm_spans(collector) + with_tools = [s for s in spans if s["_attrs"].get("gen_ai.tool.definitions") not in (None, "null")] + assert with_tools, "tool definitions are still exported" + assert json.loads(with_tools[0]["_attrs"]["gen_ai.tool.definitions"])[0]["function"]["name"] == "lookup_record" + assert all(s["_attrs"].get("session.id") for s in collector.spans()), "the session id you set is still exported" + (error_span,) = [s for s in spans if s["status"]["code"] == "STATUS_CODE_ERROR"] + assert "AuthenticationError" in error_span["status"]["message"], "error text is still exported" + (event,) = error_span.get("events", []) + assert event["name"] == "exception" + # the exception event travels with its message and a stack trace that names local files. If this starts + # failing, traceai-openai changed: update the Privacy section of the docs page. + attrs = {a["key"]: a["value"].get("string_value", "") for a in event["attributes"]} + assert "Traceback" in attrs["exception.stacktrace"] + assert "site-packages" in attrs["exception.stacktrace"] + assert "AuthenticationError" in attrs["exception.type"] + attrs["exception.message"] or "401" in attrs["exception.message"] diff --git a/examples/providers/z-ai/tests/test_recipe.py b/examples/providers/z-ai/tests/test_recipe.py new file mode 100644 index 000000000..593e4a53c --- /dev/null +++ b/examples/providers/z-ai/tests/test_recipe.py @@ -0,0 +1,201 @@ +"""The recipe, run exactly as a customer runs it: a separate process, the real `openai` +SDK and the real traceAI-openai instrumentor, a Z.ai-shaped stand-in, and a recording +collector in place of Future AGI.""" +import json +import uuid + +import pytest +from conftest import ( + CODING_PLAN_URLS, + FI_API_KEY, + FI_SECRET_KEY, + MODEL, + ZAI_KEY, + ZAI_URL, + kind, + llm_spans, + zai_requests, +) + +import app + + +def by_start(collector, wanted_kind): + return sorted((s for s in collector.spans() if kind(s) == wanted_kind), key=lambda s: int(s["start_time_unix_nano"])) + + +def test_the_supported_base_url_is_the_one_in_zais_openai_guide(): + assert app.ZAI_BASE_URL == "https://api.z.ai/api/paas/v4/" == ZAI_URL + assert app.resolve_base_url({}) == (ZAI_URL, None) + + +@pytest.mark.parametrize( + "url", + list(CODING_PLAN_URLS) + + [u + "/" for u in CODING_PLAN_URLS] + + ["HTTPS://API.Z.AI/API/CODING/PAAS/V4", "https://api.z.ai/api/anthropic/v1/messages", "https://api.z.ai/api/v1/"], +) +def test_the_three_coding_plan_urls_are_refused(url): + base, problem = app.resolve_base_url({"ZAI_BASE_URL": url}) + assert base is None + assert "Coding Plan" in problem + + +@pytest.mark.parametrize("url", ["https://api.z.ai/api/paas/v4/", "https://api.z.ai/api/paas/v4", "http://127.0.0.1:9/api/paas/v4/"]) +def test_the_general_platform_url_is_not_refused(url): + assert app.resolve_base_url({"ZAI_BASE_URL": url}) == (url, None) + + +def test_the_app_exits_before_any_call_when_given_a_coding_plan_url(run_app, collector, stub): + result = run_app("--scenario", "chat", env={"ZAI_BASE_URL": CODING_PLAN_URLS[1]}, use_stub=False) + assert result.returncode == 2 + assert "Coding Plan" in result.stderr + assert collector.exports == [] + assert zai_requests(stub) == [] + + +def test_chat_makes_one_llm_span_under_the_project_with_model_and_tokens(run_app, collector, stub): + result = run_app("--scenario", "chat", "--prompt", "hello there") + assert result.returncode == 0, result.stderr[-1500:] + assert "RESULT chat: Z.ai stand-in reply to: hello there" in result.stdout + spans = collector.spans() + assert {kind(s) for s in spans} == {"AGENT", "LLM"} + (llm,) = llm_spans(collector) + (agent,) = by_start(collector, "AGENT") + assert llm["parent_span_id"] == agent["span_id"] + assert llm["trace_id"] == agent["trace_id"] + a = llm["_attrs"] + assert a["gen_ai.request.model"] == MODEL + assert a["gen_ai.provider.name"] == "openai" + # the stand-in counts words: 2 in the prompt, 6 in "Z.ai stand-in reply to: hello there" + assert int(a["gen_ai.usage.input_tokens"]) == 2 + assert int(a["gen_ai.usage.output_tokens"]) == 6 + assert int(a["gen_ai.usage.total_tokens"]) == 8 + assert llm["_resource"]["project_name"] == "z-ai-test" + assert llm["status"]["code"] == "STATUS_CODE_OK" + # what reached "Z.ai": the documented path, a bearer key, the model + (req,) = zai_requests(stub) + assert req["path"] == "/api/paas/v4/chat/completions" + assert req["authorization_matches"] is True + assert req["body"]["model"] == MODEL + + +def test_the_whole_run_is_one_trace_and_one_session(run_app, collector): + session = f"s-{uuid.uuid4()}" + result = run_app("--session-id", session) + assert result.returncode == 0, result.stderr[-1500:] + spans = collector.spans() + assert len({s["trace_id"] for s in spans}) == 1 + assert {s["_attrs"].get("session.id") for s in spans} == {session} + assert f"session_id={session}" in result.stdout + + +def test_stream_reports_tokens_from_the_last_chunk_without_stream_options(run_app, collector, stub): + result = run_app("--scenario", "stream", "--prompt", "one two three") + assert result.returncode == 0, result.stderr[-1500:] + assert "RESULT stream: Z.ai stand-in reply to: one two three" in result.stdout + (req,) = zai_requests(stub) + assert req["body"]["stream"] is True + assert "stream_options" not in req["body"], "Z.ai documents usage on the last chunk; the recipe must not need stream_options" + (llm,) = llm_spans(collector) + a = llm["_attrs"] + assert int(a["gen_ai.usage.input_tokens"]) == 3 + assert int(a["gen_ai.usage.output_tokens"]) == 7 + assert int(a["gen_ai.usage.total_tokens"]) == 10 + assert [e["name"] for e in llm.get("events", [])] == ["First Token Stream Event"] + assert a["output.value"] == "Z.ai stand-in reply to: one two three" + + +def test_tool_round_trip_is_two_llm_spans_and_your_own_tool_span(run_app, collector): + result = run_app("--scenario", "tools", "--prompt", "find the record") + assert result.returncode == 0, result.stderr[-1500:] + assert "RESULT tools: The lookup returned: record for find the record" in result.stdout + first, second = llm_spans(collector) + (tool,) = by_start(collector, "TOOL") + (agent,) = by_start(collector, "AGENT") + assert {first["parent_span_id"], second["parent_span_id"], tool["parent_span_id"]} == {agent["span_id"]} + a1, a2 = first["_attrs"], second["_attrs"] + assert a1["gen_ai.output.messages.0.message.tool_calls.0.tool_call.function.name"] == "lookup_record" + assert json.loads(a1["gen_ai.output.messages.0.message.tool_calls.0.tool_call.function.arguments"]) == {"query": "find the record"} + assert json.loads(a1["gen_ai.tool.definitions"])[0]["function"]["name"] == "lookup_record" + assert a2["gen_ai.input.messages.2.message.role"] == "tool" + assert a2["gen_ai.output.messages.0.message.content"] == "The lookup returned: record for find the record" + for a in (a1, a2): + assert a["gen_ai.request.model"] == MODEL + assert int(a["gen_ai.usage.total_tokens"]) > 0 + + +def test_thinking_passes_through_to_zai_but_reasoning_text_is_not_on_the_span(run_app, collector, stub): + result = run_app("--scenario", "thinking", "--prompt", "why is the sky blue") + assert result.returncode == 0, result.stderr[-1500:] + (req,) = zai_requests(stub) + assert req["body"]["thinking"] == {"type": "enabled"} + (llm,) = llm_spans(collector) + exported = json.dumps(llm) + assert "Stand-in reasoning" not in exported, "reasoning_content is not recorded by traceAI-openai 0.1.10" + assert "reasoning_content" not in exported + assert "thinking" not in llm["_attrs"]["gen_ai.request.parameters"] + # the reasoning tokens are inside the usage the response reports, and that is what the span carries + assert int(llm["_attrs"]["gen_ai.usage.output_tokens"]) > len("Z.ai stand-in reply to: why is the sky blue".split()) + + +def test_a_failed_call_is_an_error_span_with_the_http_status(run_app, collector): + result = run_app("--scenario", "error") + assert result.returncode == 0, result.stderr[-1500:] + assert "RESULT error: HTTP 401" in result.stdout + (llm,) = llm_spans(collector) + assert llm["status"]["code"] == "STATUS_CODE_ERROR" + assert "401" in llm["status"]["message"] + assert "AuthenticationError" in llm["status"]["message"] + assert [e["name"] for e in llm.get("events", [])] == ["exception"] + assert llm["_attrs"]["gen_ai.provider.name"] == "openai" + assert llm["_resource"]["project_name"] == "z-ai-test" + + +def test_provider_label_reads_openai_for_every_call(run_app, collector): + assert run_app().returncode == 0 + labels = {s["_attrs"]["gen_ai.provider.name"] for s in llm_spans(collector)} + assert labels == {"openai"} + + +def test_keys_go_only_where_they_belong(run_app, collector, stub): + assert run_app().returncode == 0 + reqs = zai_requests(stub) + assert reqs and all(r["authorization_scheme"] == "Bearer" for r in reqs) + for r in reqs: + assert "x-api-key" not in r["header_names"] and "x-secret-key" not in r["header_names"] + assert FI_API_KEY not in json.dumps(r["body"]) and FI_SECRET_KEY not in json.dumps(r["body"]) + exported = "\n".join(collector.raw_texts()) + assert ZAI_KEY not in exported + assert "not-a-real-zai-key" not in exported + assert "Bearer" not in exported + for export in collector.exports: + assert export["headers"]["x-api-key"] == FI_API_KEY + assert export["headers"]["x-secret-key"] == FI_SECRET_KEY + + +def test_export_is_uncompressed_otlp_http_with_project_name(run_app, collector): + assert run_app("--scenario", "chat").returncode == 0 + assert collector.exports + for export in collector.exports: + assert export["path"] == "/tracer/v1/traces" + assert "content-encoding" not in export["headers"] + assert export["headers"]["content-type"] == "application/x-protobuf" + assert {s["_resource"]["project_name"] for s in collector.spans()} == {"z-ai-test"} + + +def test_an_unreachable_collector_does_not_break_the_request(run_app): + result = run_app("--scenario", "chat", "--prompt", "still works", fi_base_url="http://127.0.0.1:9") + assert result.returncode == 0 + assert "RESULT chat: Z.ai stand-in reply to: still works" in result.stdout + + +def test_a_stream_that_ends_abnormally_is_not_an_error_span(run_app, collector): + """Z.ai reports an inference that dies mid-stream through finish_reason, not an HTTP error, so the + span ends OK, shows only the text that arrived, and does not record the finish reason.""" + result = run_app("--scenario", "stream", "--prompt", "[[abnormal-stream]] tell me more") + assert result.returncode == 0, result.stderr[-1500:] + (llm,) = llm_spans(collector) + assert llm["status"]["code"] == "STATUS_CODE_OK" + assert llm["_attrs"]["output.value"] == "Z.ai stand-in" + assert "network_error" not in "\n".join(collector.raw_texts()) diff --git a/examples/providers/z-ai/tests/test_stub.py b/examples/providers/z-ai/tests/test_stub.py new file mode 100644 index 000000000..9c7784209 --- /dev/null +++ b/examples/providers/z-ai/tests/test_stub.py @@ -0,0 +1,72 @@ +"""The stand-in only claims what Z.ai documents, and answers the documented error cases.""" +import json +import urllib.error +import urllib.request + +import pytest +from conftest import MODEL, ZAI_KEY + + +def post(stub, body, key=ZAI_KEY, path="chat/completions"): + request = urllib.request.Request(stub.url + path, data=json.dumps(body).encode(), method="POST") + request.add_header("Content-Type", "application/json") + if key: + request.add_header("Authorization", f"Bearer {key}") + try: + with urllib.request.urlopen(request) as r: + return r.status, r.read().decode() + except urllib.error.HTTPError as e: + return e.code, e.read().decode() + + +MESSAGES = [{"role": "user", "content": "hi there"}] + + +def test_a_chat_response_has_the_documented_fields_and_no_object_field_by_default(plain_stub): + status, raw = post(plain_stub, {"model": MODEL, "messages": MESSAGES}) + body = json.loads(raw) + assert status == 200 + assert set(body) == {"id", "request_id", "created", "model", "choices", "usage"} + assert body["choices"][0]["finish_reason"] == "stop" + assert set(body["usage"]) == {"prompt_tokens", "completion_tokens", "prompt_tokens_details", "total_tokens"} + assert body["usage"]["total_tokens"] == body["usage"]["prompt_tokens"] + body["usage"]["completion_tokens"] + + +def test_the_stream_ends_with_finish_reason_and_usage_then_done(plain_stub): + status, raw = post(plain_stub, {"model": MODEL, "messages": MESSAGES, "stream": True}) + assert status == 200 + events = [line[len("data: "):] for line in raw.split("\n\n") if line.startswith("data: ")] + assert events[-1] == "[DONE]" + chunks = [json.loads(e) for e in events[:-1]] + assert all("usage" not in c for c in chunks[:-1]) + assert chunks[-1]["choices"][0]["finish_reason"] == "stop" + assert chunks[-1]["usage"]["total_tokens"] > 0 + assert all("object" not in c for c in chunks) + assert "".join(c["choices"][0]["delta"].get("content", "") for c in chunks) == "Z.ai stand-in reply to: hi there" + + +def test_thinking_adds_reasoning_content_only_when_enabled(plain_stub): + on = json.loads(post(plain_stub, {"model": MODEL, "messages": MESSAGES, "thinking": {"type": "enabled"}})[1]) + off = json.loads(post(plain_stub, {"model": MODEL, "messages": MESSAGES, "thinking": {"type": "disabled"}})[1]) + assert "reasoning_content" in on["choices"][0]["message"] + assert "reasoning_content" not in off["choices"][0]["message"] + + +@pytest.mark.parametrize( + "key, body, status, code", + [ + (None, {"model": MODEL, "messages": MESSAGES}, 401, "1001"), + ("wrong", {"model": MODEL, "messages": MESSAGES}, 401, "1000"), + (ZAI_KEY, {"model": "not-a-glm", "messages": MESSAGES}, 400, "1211"), + (ZAI_KEY, {"model": MODEL, "messages": MESSAGES, "tools": [{"type": "function", "function": {"name": "f"}}], "tool_choice": "required"}, 400, "1214"), + ], +) +def test_documented_errors_use_the_documented_shape(plain_stub, key, body, status, code): + got_status, raw = post(plain_stub, body, key=key) + assert got_status == status + assert json.loads(raw) == {"error": {"code": code, "message": json.loads(raw)["error"]["message"]}} + + +def test_only_the_documented_path_exists(plain_stub): + status, _ = post(plain_stub, {"model": MODEL, "messages": MESSAGES}, path="completions") + assert status == 404 diff --git a/examples/providers/z-ai/tests/test_zai_sdk.py b/examples/providers/z-ai/tests/test_zai_sdk.py new file mode 100644 index 000000000..706ad95be --- /dev/null +++ b/examples/providers/z-ai/tests/test_zai_sdk.py @@ -0,0 +1,53 @@ +"""Z.ai's own SDK is not traced by traceAI-openai. + +Optional: needs `pip install zai-sdk`. `ZaiClient` is not a subclass of `openai.OpenAI`, and the +instrumentor wraps `openai.OpenAI.request`, so a ZaiClient call produces no span while the stock +`openai.OpenAI` client, pointed at the same server, produces one. ZaiClient is only pointed at the +local stand-in here; nothing is sent to Z.ai.""" +import pytest + +zai = pytest.importorskip("zai") + +from conftest import MODEL, ZAI_KEY # noqa: E402 +from fi_instrumentation import register # noqa: E402 +from fi_instrumentation.fi_types import ProjectType # noqa: E402 +from openai import OpenAI # noqa: E402 +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter # noqa: E402 +from traceai_openai import OpenAIInstrumentor # noqa: E402 + + +@pytest.fixture() +def traced(monkeypatch): + monkeypatch.setenv("FI_API_KEY", "test-fi-api-key-0002") + monkeypatch.setenv("FI_SECRET_KEY", "test-fi-secret-key-0003") + exporter = InMemorySpanExporter() + provider = register( + project_type=ProjectType.OBSERVE, project_name="z-ai-sdk", batch=False, span_exporter=exporter, verbose=False + ) + instrumentor = OpenAIInstrumentor() + instrumentor.instrument(tracer_provider=provider) + yield exporter + instrumentor.uninstrument() + provider.shutdown() + + +def test_zaiclient_is_not_an_openai_client(): + assert not issubclass(zai.ZaiClient, OpenAI) + + +def test_zaiclient_defaults_to_the_same_general_platform_url(): + assert str(zai.ZaiClient(api_key="k").base_url).rstrip("/") == "https://api.z.ai/api/paas/v4" + + +def test_a_zaiclient_call_makes_no_span_but_the_openai_client_does(traced, plain_stub): + reply = zai.ZaiClient(api_key=ZAI_KEY, base_url=plain_stub.url).chat.completions.create( + model=MODEL, messages=[{"role": "user", "content": "hi"}] + ) + assert reply.choices[0].message.content == "Z.ai stand-in reply to: hi" + assert traced.get_finished_spans() == (), "traceAI-openai does not instrument ZaiClient" + + OpenAI(api_key=ZAI_KEY, base_url=plain_stub.url).chat.completions.create( + model=MODEL, messages=[{"role": "user", "content": "hi"}] + ) + (span,) = traced.get_finished_spans() + assert dict(span.attributes)["gen_ai.request.model"] == MODEL diff --git a/src/lib/integrations.mjs b/src/lib/integrations.mjs index 287ebff17..d76e99979 100644 --- a/src/lib/integrations.mjs +++ b/src/lib/integrations.mjs @@ -5,7 +5,7 @@ * A method describes the connection path, not a support or testing badge. * Run `npm run test:integrations` when changing this file. * - * @typedef {'providers'|'frameworks'|'gateways'|'voice'|'java'|'apps'|'import'|'export'} IntegrationCategory + * @typedef {'providers'|'frameworks'|'coding'|'gateways'|'voice'|'java'|'apps'|'import'|'export'} IntegrationCategory * @typedef {{id: string, name: string, href: string, category: IntegrationCategory, * summary: string, method: string, logo: string, aliases?: string[], sharedGuide?: boolean}} Integration * @typedef {{id: IntegrationCategory, name: string, description: string}} Category @@ -15,6 +15,7 @@ export const integrationCategories = [ { id: 'providers', name: 'Model providers', description: 'Trace model calls from your application.' }, { id: 'frameworks', name: 'Frameworks & agents', description: 'Connect the steps, tools and model calls in your application.' }, + { id: 'coding', name: 'Coding agents', description: 'Connect coding-agent sessions, turns and tools.' }, { id: 'gateways', name: 'Gateways', description: 'Trace proxied requests, using the connection method in each guide.' }, { id: 'voice', name: 'Voice & realtime', description: 'Follow conversations across your voice stack.' }, { id: 'java', name: 'Java', description: 'Set up JVM tracing with client wrappers or native OpenTelemetry.' }, @@ -25,6 +26,9 @@ export const integrationCategories = [ /** @type {ReadonlyArray} */ export const integrations = [ + { id: 'z-ai', name: 'Z.ai', href: '/docs/integrations/providers/z-ai', category: 'providers', summary: 'Trace Python chat completions on the general-platform endpoint.', method: 'OpenAI client', logo: 'generic-model' }, + { id: 'cursor', name: 'Cursor IDE', href: '/docs/integrations/coding-agents/cursor', category: 'coding', summary: 'Trace macOS IDE agent sessions through observation hooks.', method: 'Hook-to-OTLP', logo: 'generic-framework' }, + { id: 'langsmith', name: 'LangSmith', href: '/docs/integrations/langsmith', category: 'frameworks', summary: 'Forward supported LangSmith SDK traces over OTLP.', method: 'Native OTLP', logo: 'generic-framework' }, { id: 'openai', name: 'OpenAI', href: '/docs/integrations/traceai/openai', category: 'providers', summary: 'Instrument OpenAI client calls in your application.', method: 'traceAI SDK', logo: 'openai' }, { id: 'anthropic', name: 'Anthropic', href: '/docs/integrations/traceai/anthropic', category: 'providers', summary: 'Capture Claude calls, token usage and latency.', method: 'traceAI SDK', logo: 'anthropic', aliases: ['Claude'] }, { id: 'bedrock', name: 'AWS Bedrock', href: '/docs/integrations/traceai/bedrock', category: 'providers', summary: 'Trace model calls through Amazon Bedrock.', method: 'traceAI SDK', logo: 'bedrock', aliases: ['Amazon'] }, @@ -109,6 +113,11 @@ export const integrations = [ // Overview aids are links, not product-logo cards. Keep them in the same source. export const integrationAids = [ + { name: 'Langdock support status', href: '/docs/integrations/no-code/langdock', category: 'apps' }, + { name: 'Inferable support status', href: '/docs/integrations/inferable', category: 'frameworks' }, + { name: 'AI SDK C++ support status', href: '/docs/integrations/frameworks/ai-sdk-cpp', category: 'frameworks' }, + { name: 'Hook-to-OTLP emitter', href: '/docs/integrations/coding-agents/hook-to-otlp', category: 'coding' }, + { name: 'OpenAI Assistants retirement', href: '/docs/integrations/traceai/openai_assistants', category: 'providers' }, { name: 'traceAI setup', href: '/docs/integrations/traceai' }, { name: 'Java SDK setup', href: '/docs/integrations/traceai/java', category: 'java' }, ]; diff --git a/src/pages/docs/integrations/coding-agents/cursor.mdx b/src/pages/docs/integrations/coding-agents/cursor.mdx new file mode 100644 index 000000000..1852a1fdc --- /dev/null +++ b/src/pages/docs/integrations/coding-agents/cursor.mdx @@ -0,0 +1,161 @@ +--- +title: "Cursor Agent session tracing with Future AGI" +description: "Trace Cursor IDE agent sessions in Future AGI with Cursor hooks: one trace per conversation, with turns, tool calls, subagents, model, timing, and errors. Prompt and code content stay off by default." +--- + +## About + +Cursor runs [hooks](https://cursor.com/docs/hooks) at fixed points of the agent loop and passes each one a JSON payload. This page registers the Future AGI [hook-to-OTLP emitter](/docs/integrations/coding-agents/hook-to-otlp) on those hooks, so every Cursor Agent conversation shows up in Future AGI as one trace: + +``` +Cursor session one Cursor conversation (conversation_id) +└── Cursor turn one user message (generation_id), model, stop status + ├── Shell tool call: duration, status, failure type + ├── Read + ├── Subagent: explore Task subagent: duration, message and tool counts + ├── Agent thought thinking block duration + └── Agent response model, input/output/cache tokens +``` + +This is not the same as: + +- **The Cursor SDK**, which runs Cursor agents from your own code. That is a separate integration. +- **Cursor Enterprise OpenTelemetry export**, an admin feature that sends team usage metrics and logs. It does not produce a per-session trace and is not used here. +- **Model-call tracing.** Cursor calls the model on its servers. Hooks report one token total per turn, not each model request. + +| | | +| --- | --- | +| Verified scope | Cursor IDE 3.24.18 on macOS, free Hobby plan, with a POSIX login shell. This recipe supports that IDE/macOS path; Linux, WSL, native Windows, Cursor CLI, and cloud agents are outside its supported scope. | +| Cursor docs used | [Hooks](https://cursor.com/docs/hooks), read 2026-10-14 | +| Requires | Python 3.9+ on the `PATH` Cursor uses. No packages. | +| Source | [examples/coding-agents/cursor](https://github.com/future-agi/docs/tree/main/examples/coding-agents/cursor) and [hook-to-otlp](https://github.com/future-agi/docs/tree/main/examples/coding-agents/hook-to-otlp) | + +## 1. Install the hook + +Clone the examples and run the installer against your project. It copies the emitter and the Cursor adapter into `.cursor/hooks/fi-hook-to-otlp/` and writes `.cursor/hooks.json`. If the project already has a `hooks.json`, the installer leaves it alone and prints the entries to merge. + +```bash +git clone --depth 1 https://github.com/future-agi/docs.git fi-docs +fi-docs/examples/coding-agents/cursor/install.sh /path/to/your/project +``` + +To trace every project on this machine instead, install user hooks into `~/.cursor/`: + +```bash +fi-docs/examples/coding-agents/cursor/install.sh --user +``` + +The project `.cursor/hooks.json` it writes registers the emitter on the observation hooks only: + +```json +{ + "version": 1, + "hooks": { + "sessionStart": [{ "command": "(python3 -B .cursor/hooks/fi-hook-to-otlp/emit.py cursor sessionStart 2>/dev/null || true)", "timeout": 10 }], + "beforeSubmitPrompt": [{ "command": "(python3 -B .cursor/hooks/fi-hook-to-otlp/emit.py cursor beforeSubmitPrompt 2>/dev/null || true)", "timeout": 10 }], + "postToolUse": [{ "command": "(python3 -B .cursor/hooks/fi-hook-to-otlp/emit.py cursor postToolUse 2>/dev/null || true)", "timeout": 10 }], + "postToolUseFailure": [{ "command": "(python3 -B .cursor/hooks/fi-hook-to-otlp/emit.py cursor postToolUseFailure 2>/dev/null || true)", "timeout": 10 }], + "subagentStop": [{ "command": "(python3 -B .cursor/hooks/fi-hook-to-otlp/emit.py cursor subagentStop 2>/dev/null || true)", "timeout": 10 }], + "afterAgentThought": [{ "command": "(python3 -B .cursor/hooks/fi-hook-to-otlp/emit.py cursor afterAgentThought 2>/dev/null || true)", "timeout": 10 }], + "afterAgentResponse": [{ "command": "(python3 -B .cursor/hooks/fi-hook-to-otlp/emit.py cursor afterAgentResponse 2>/dev/null || true)", "timeout": 10 }], + "preCompact": [{ "command": "(python3 -B .cursor/hooks/fi-hook-to-otlp/emit.py cursor preCompact 2>/dev/null || true)", "timeout": 10 }], + "stop": [{ "command": "(python3 -B .cursor/hooks/fi-hook-to-otlp/emit.py cursor stop 2>/dev/null || true)", "timeout": 10 }], + "sessionEnd": [{ "command": "(python3 -B .cursor/hooks/fi-hook-to-otlp/emit.py cursor sessionEnd 2>/dev/null || true)", "timeout": 10 }] + } +} +``` + +Each command is guarded with `2>/dev/null || true` so a missing emitter or `python3` can never return exit code 2, which Cursor treats as "block this action". Cursor runs each command with `$SHELL -c` (falling back to `/bin/sh`), so your login shell must be a POSIX shell such as zsh or bash (fish reads the parentheses as command substitution). The parentheses are required: the IDE agent pipes the hook payload to stdin, but Cursor's headless agent host appends it to the command as a heredoc (` <<'CURSOR_HOOK_EOF'`), and without the subshell that heredoc would reach `true` instead of `python3`. `-B` keeps `__pycache__` out of your repository. Project hooks run from the project root, so the paths start with `.cursor/`. User hooks run from `~/.cursor/` and use `./hooks/fi-hook-to-otlp/emit.py` instead (the installer handles this). Project hooks only run in a [trusted workspace](https://cursor.com/docs/hooks). + + +The emitter is deliberately not registered on permission hooks (`preToolUse`, `beforeShellExecution`, `beforeMCPExecution`, `beforeReadFile`, `subagentStart`). Tool timing comes from the `duration` Cursor reports in `postToolUse`, so tracing never takes part in approving or blocking an action. On `beforeSubmitPrompt` it always answers `{"continue": true}`. If you register it on a permission hook by mistake, it prints nothing: Cursor treats that as a failed hook and lets the action through, so it never approves or blocks anything. + + +## 2. Add your Future AGI keys + +Cursor started from the Dock does not see variables exported in your shell. Put the keys in the emitter's env file instead: + +```bash +mkdir -p ~/.config/fi-hook-to-otlp +cat > ~/.config/fi-hook-to-otlp/env <<'EOF' +FI_API_KEY=your-futureagi-api-key +FI_SECRET_KEY=your-futureagi-secret-key +FI_PROJECT_NAME=cursor-sessions +EOF +chmod 600 ~/.config/fi-hook-to-otlp/env +``` + +Variables set in the environment Cursor runs in take precedence over the file. For a self-hosted Future AGI, also set `FI_BASE_URL` (traces go to `$FI_BASE_URL/tracer/v1/traces`). All settings are listed on the [hook-to-OTLP page](/docs/integrations/coding-agents/hook-to-otlp#configuration). + +## 3. Run an agent session + +Open the project in Cursor (reload the window if it was already open; Cursor also reloads `hooks.json` on save), and send one Agent message that uses a tool, for example *"List the files in this folder and read README.md"*. + +Open **Observe** in Future AGI and select the project (`cursor-sessions` above). The conversation appears as one trace named **Cursor session**. Open it to see the turn and its tool spans. Later messages in the same Cursor chat are added to the same trace as new turns. A chat that was opened and closed without a message is not sent. + +To check that hooks are firing, open Cursor's **Hooks** output channel or the Hooks tab in **Customize**. To check the emitter, set `FI_HOOK_LOG_FILE=/tmp/fi-hook.log` in the env file: each hook writes one line with the HTTP status. + +## What is captured + +| Cursor hook | Future AGI span | Fields | +| --- | --- | --- | +| `sessionStart` | **Cursor session** (root) | `session.id` = `conversation_id`, composer mode, background-agent flag | +| `beforeSubmitPrompt` | **Cursor turn** start | `generation_id`, attachment count, model and model parameters | +| `postToolUse` | tool span named after the tool (`Shell`, `Read`, `Write`, `MCP:`, ...) | `gen_ai.tool.name`, `gen_ai.tool.call.id` (`tool_use_id`), duration from `duration` | +| `postToolUseFailure` | same tool span, status `ERROR` | status message = `failure_type` (`error`, `timeout`, `permission_denied`), interrupt flag | +| `subagentStop` | **Subagent: <type>** | status, duration, message count, tool-call count, modified-file count | +| `afterAgentThought` | **Agent thought** (`LLM`) | thinking duration, model | +| `afterAgentResponse` | **Agent response** (`LLM`) | model; the turn's input, output, and cache-read/write tokens (`llm.token_count.*`) | +| `preCompact` | **Context compaction** | trigger, context tokens, window size, message counts | +| `stop` | **Cursor turn** end | stop status (`completed`, `aborted`, `error`; `error` marks the turn `ERROR`), loop count, the same token counts as `cursor.usage.*` | +| `sessionEnd` | **Cursor session** end | reason, final status, duration | + +The model on turn and response spans is Cursor's `model_id` (for example `grok-4.6`), with the composer slug in `cursor.model` when it differs. Cursor reports `default` when no specific model is selected. + +Token counts come from `input_tokens`, `output_tokens`, `cache_read_tokens`, and `cache_write_tokens`, which Cursor 3.24.18 sends on `afterAgentResponse` and `stop` although the hooks reference does not list them yet. They are the totals for the turn and are recorded once, on the **Agent response** span, so trace totals are not doubled. + +In our tests Cursor did not send `sessionStart` for chats in the IDE, so the session span starts at the first hook of the conversation. + +## What is not available + +- **Per-request tokens and cost.** Tokens are reported per turn, not per model request, and Cursor does not report cost. `preCompact` reports context-window usage, which is recorded as an attribute, not as token usage. +- **Model request spans.** There is no hook per model call, so there is no LLM span with inputs and outputs per request; *Agent response* marks when a reply finished. +- **System prompt and provider request ids.** (`cursor.tool.provider_call_id` on tool spans is the model's function-call id, not a request id.) +- **Tool calls that fire no hook.** Cursor's forum reports that the AskQuestion (human-in-the-loop) tool does not trigger hooks; such calls are missing from the trace. +- **Subagent internals.** A subagent appears as one span. Whether its own tool calls fire hooks inside the parent conversation is not documented. +- **Tab completions.** Tab hooks (`beforeTabFileRead`, `afterTabFileEdit`) are a separate surface and are not traced. +- **Session end on window close.** In Cursor 3.24.18 the `sessionEnd` hook fails when the window closes ("MainThreadShellExec not initialized" in the Hooks channel), so the session span keeps the end of the last turn and has no end reason. Turns are not affected. + +### Cursor CLI and cloud agents + +Cursor CLI (`agent`) and cloud agents are outside this recipe's supported scope. Their hook coverage differs from the IDE: Cursor staff describe missing CLI events in the [hooks forum](https://forum.cursor.com/t/cursor-cli-doesnt-send-all-events-defined-in-hooks/148316), and the [cloud-agent reference](https://cursor.com/docs/hooks#cloud-agent-support) excludes `sessionStart` and `sessionEnd`. Do not use the IDE trace shape above as a guarantee for either surface. The guarded commands accept both piped stdin and shell heredocs; those transport checks establish command behavior, not CLI or cloud-agent qualification. + +### Plan availability + +Cursor's [pricing page](https://cursor.com/pricing) lists "MCPs, skills, and hooks" under the Individual (Pro) plan. In our test on Cursor 3.24.18 the agent hooks above did run on the free Hobby plan. Plans can change, so confirm in the Hooks output channel that hooks execute on yours. + +## Privacy + +By default only structure and metadata leave your machine: conversation and generation ids, tool names and call ids, model, durations, counts, and statuses. + +- **Not exported by default:** prompt text, tool arguments, tool output (file contents, command output), thinking and response text, subagent task and summary, tool error text. +- **Never exported:** `user_email`, `transcript_path` (the emitter never opens it), and `workspace_roots`, `cwd`, and attachment paths as metadata. + +To include content, add `FI_HOOK_CAPTURE_CONTENT=1` to the env file. The prompt becomes the turn's `input.value`, the final response its `output.value`, tool arguments and output go on the tool span, and tool error text becomes the span's status message and `exception.message`. Tool input and output contain file paths (for example the Read tool's `file_path` and the Shell tool's `working_directory`); they are exported with the user name in home-directory paths replaced by ``. Each field is capped at 4096 characters and common key formats are replaced with `[REDACTED]`. See [Privacy](/docs/integrations/coding-agents/hook-to-otlp#privacy) for details. + +## Troubleshooting + +**No trace appears.** +1. Confirm the hook ran: Cursor's **Hooks** output channel lists each execution and any error. +2. Set `FI_HOOK_LOG_FILE=/tmp/fi-hook.log` in `~/.config/fi-hook-to-otlp/env` and run another message. No lines: Cursor did not run the command (check the path in `hooks.json` and that the workspace is trusted). `error=` lines: the collector is unreachable. `http=401`/`403`: wrong keys. +3. `python3` must resolve for Cursor. If your Mac has no developer tools installed, `/usr/bin/python3` prompts to install them; use an absolute interpreter path such as `/opt/homebrew/bin/python3` in `hooks.json`, and keep `-B`, the `2>/dev/null || true` guard, and the surrounding parentheses. + +**A turn is marked `ERROR` right after sending.** Cursor ended the agent loop with status `error`. On the free plan this happens when a named model is selected ("Named models unavailable"); switch the chat to Auto. + +**A tool span has `coding_agent.turn.inferred=true`.** The tool hook arrived without a `generation_id`, so the tool was attached to the turn that was open at the time. + +**A tool span sits directly under the session.** The tool hook arrived without a `generation_id` while no turn was open, so there was no turn to attach it to. These spans carry no turn marker. + +**The turn is marked incomplete.** The session ended (or the window closed) before Cursor sent `stop` for that turn. The turn is closed at session end with `coding_agent.incomplete=true`. + +**Hooks stopped firing after editing `hooks.json`.** Cursor reloads the file on save; if it does not, restart Cursor. diff --git a/src/pages/docs/integrations/coding-agents/hook-to-otlp.mdx b/src/pages/docs/integrations/coding-agents/hook-to-otlp.mdx new file mode 100644 index 000000000..520e138cf --- /dev/null +++ b/src/pages/docs/integrations/coding-agents/hook-to-otlp.mdx @@ -0,0 +1,128 @@ +--- +title: "Coding agent hooks to Future AGI (hook-to-OTLP)" +description: "A small hook command that turns coding-agent hook events (Cursor, Claude Code, and others) into OpenTelemetry traces in Future AGI: one trace per session, content off by default." +--- + +## About + +Most coding agents (Cursor, Claude Code, Gemini CLI, Kiro, and others) can run a command at fixed points of the agent loop: when a session starts, before a prompt is sent, after a tool runs, when the agent stops. Each hook receives a JSON payload on stdin (some apps append it to the command as a heredoc, which the shell feeds to stdin). + +`hook-to-otlp` is one stdlib-only Python script that you register as that hook command. It maps the payloads to OpenTelemetry spans and sends them to Future AGI over OTLP/HTTP. Each app has a small adapter that knows its event names and fields. The app-specific pages (for example [Cursor](/docs/integrations/coding-agents/cursor)) show the exact hook configuration. + +This is application session tracing: it shows what the agent did (turns, tools, subagents, timing, errors). It is not model-call instrumentation. For your own code that calls an LLM SDK, use a [traceAI integration](/docs/integrations) instead. + +Source: [examples/coding-agents/hook-to-otlp](https://github.com/future-agi/docs/tree/main/examples/coding-agents/hook-to-otlp). + +## What you get in Future AGI + +One agent session (conversation) becomes one trace: + +``` +Cursor session AGENT whole conversation, re-sent as it grows +└── Cursor turn CHAIN one user message → agent stop + ├── Shell TOOL duration, status, failure type + ├── Read TOOL + ├── Subagent: explore AGENT + └── Agent response LLM model name +``` + +| Span | Parent | `fi.span.kind` | When it is sent | +| --- | --- | --- | --- | +| Session | none | `AGENT` | together with every child span, and at session end | +| Turn | session | `CHAIN` | when the turn ends | +| Tool | turn | `TOOL` | when the tool finishes or fails | +| Subagent | turn | `AGENT` | when the subagent stops | +| Model response / thought | turn | `LLM` | when the hook fires | +| Other events (for example context compaction) | turn | `CHAIN` | when the hook fires | + +Every span has `session.id` and `gen_ai.conversation.id` set to the app's session id, so the Sessions view groups them. Tool spans have `gen_ai.tool.name` and `gen_ai.tool.call.id`. Session, turn, and model spans have `llm.model_name` when the app reports a model. Failed tools and turns are marked `ERROR`. + +How the emitter builds the tree: + +- The trace id is a hash of the app name and session id, so every hook of a session lands in the same trace without coordination. +- Span ids are derived the same way from the session, the span type, and the app's id for it (turn id, tool call id). A repeated hook overwrites its span instead of duplicating it. +- Apps that do not report turn or tool call ids still get a tree: each prompt hook opens the next turn, and each tool end is paired with the oldest unfinished start of the same tool. +- A new turn closes a previous turn that never reported its end, marking it `coding_agent.incomplete=true` (corrected if the end arrives later). +- Durations come from a matching start hook when the app has one, otherwise from the duration the app reports in the end hook. A small state file per session (in your temp directory, mode 0600) keeps start times between hook processes. With content capture on, it also holds the content of spans that have not ended yet (for example the open turn's prompt) and of turns closed as incomplete; a turn that ends normally leaves no content in it. It is deleted when the session ends, and files untouched for 24 hours are removed. +- The session span is re-sent with the same id and start time whenever a child span is sent, so it always covers the whole conversation. A conversation that was opened and closed without any activity is not exported. + +## Privacy + +By default the emitter exports structure and metadata only: ids, names, model, durations, counts, statuses. + +| Data | Default | With `FI_HOOK_CAPTURE_CONTENT=1` | +| --- | --- | --- | +| Session, turn, tool call ids, tool names, model, durations, status | exported | exported | +| Prompt text, tool arguments, tool output, model response and thinking text | **not exported** | exported as `input.value` / `output.value`, capped at 4096 characters per field, common key and token formats replaced with `[REDACTED]` | +| Tool error text | **not exported** (status message is the failure type, for example `timeout`) | exported as `exception.message` and the status message, with the user name in home-directory paths replaced by `` | +| File and workspace paths as metadata (`cwd`, workspace roots, transcript paths) | **never exported** | **never exported** | +| Paths inside prompts, tool input, and tool output (for example a Read tool's `file_path`) | **not exported** | exported as part of that content, with the user name in home-directory paths replaced by `` (`/Users//project/README.md`) | +| User email | **never exported** | **never exported** | + +The user name is the one directory after `/Users/`, `/home/`, or `C:\Users\` (and the name in `~name/`); the text after it, such as the rest of a shell command, is kept as it was. A name with spaces is replaced in full when it is this machine's own home directory. For another account's name with spaces (for example in a log copied from another machine), only the part before the first space is replaced. + +The emitter never reads files named in a payload (for example a transcript path) and never reads the app's local database or logs. It only sees the JSON the app passes to the hook. + + +Turn content on only for projects where prompts and source code may be stored in Future AGI. Redaction catches common key formats; it is not a data-loss-prevention control. + + +## Configuration + +Set these in the environment that starts the agent, or in `~/.config/fi-hook-to-otlp/env` as `KEY=VALUE` lines (useful for desktop apps that do not inherit your shell environment; `chmod 600` the file). + +| Variable | Default | Meaning | +| --- | --- | --- | +| `FI_API_KEY`, `FI_SECRET_KEY` | none | Your Future AGI keys. Without them the hook does nothing. | +| `FI_PROJECT_NAME` | the app name | Future AGI project for these traces. | +| `FI_BASE_URL` | `https://api.futureagi.com` | Traces go to `$FI_BASE_URL/tracer/v1/traces`. | +| `FI_OTLP_ENDPOINT` | none | Full traces URL, overrides `FI_BASE_URL` (self-hosted collectors). | +| `FI_HOOK_CAPTURE_CONTENT` | off | `1` exports prompts, tool input/output, and model text. | +| `FI_HOOK_MAX_CONTENT_CHARS` | `4096` | Per-field cap when content is on. | +| `FI_HOOK_TIMEOUT` | `10` (`3` on Windows) | HTTP timeout in seconds. On macOS and Linux the send runs in the background, so this never delays the agent. | +| `FI_HOOK_LOG_FILE` | none | Writes one line per hook (event, span count, HTTP status). No content, no keys. | +| `FI_HOOK_DUMP_FILE` | none | Appends every OTLP body exactly as sent, to check what leaves the machine. | +| `FI_HOOK_DISABLED` | off | `1` turns the hook into a no-op. | + +## Registering the hook + +Apps that export OpenTelemetry natively do not use this emitter and have no hook command; their pages set up the app's own telemetry export. Every app that is traced through hooks registers this same guarded command, so a missing emitter or a missing `python3` can never block the agent (many apps treat exit code 2 from a hook as "block this action"): + +```bash +(python3 -B /emit.py 2>/dev/null || true) +``` + +- `|| true` turns any failure (emitter file missing, `python3` missing, crash) into exit code 0 with empty output, which apps treat as "no decision". +- The parentheses run the command in a subshell, so the payload reaches `python3` however the app passes it. Some apps pipe it to stdin; others append it to the command as a heredoc (Cursor's headless agent host runs ` <<'CURSOR_HOOK_EOF'`). Without the parentheses that heredoc would go to `true`, and the emitter would get no payload. +- `2>/dev/null` keeps Python's own error text out of the app's logs. The emitter itself never writes to stderr. +- `-B` stops Python from writing `__pycache__` into your project. +- The event name is passed explicitly, so the emitter works even if the app leaves it out of the payload. + +This form needs a POSIX shell (`sh`, `bash`, `zsh`); fish, for example, reads the parentheses as command substitution. The Cursor recipe is qualified for the IDE on macOS, where Cursor runs each hook command with `$SHELL -c` (falling back to `/bin/sh`). Native Windows, WSL, Linux, CLI, and cloud-agent operation are outside that recipe's supported scope. Other app adapters must define their own supported platform boundary; the generic emitter's platform fallbacks below are implementation limits, not an additional qualified installation path. + +## Safety for the host app + +- The script exits 0 and prints the response the app expects (for most hooks `{}`), so it never blocks or changes the agent's behaviour. If its adapter cannot be loaded or it fails, it prints nothing. If Python cannot start the script at all (for example a half-deleted install), the guard in the hook command still turns that into exit code 0 with no output. +- On macOS and Linux the HTTP request runs in a detached background process, so the agent does not wait for the network. On Windows there is no background send: the request runs inline with a 3-second timeout, and hooks of the same session are not locked against each other, so a start time can occasionally be lost. A down collector, a wrong key, or bad JSON is logged (if `FI_HOOK_LOG_FILE` is set) and otherwise ignored. +- Register it only on hooks that observe. Do not use it as the deciding hook for permission prompts. +- Requirements: Python 3.9 or newer on the `PATH` the app uses. No packages to install. + +## Supported apps + +| App | Page | +| --- | --- | +| Cursor (IDE agent) | [Cursor](/docs/integrations/coding-agents/cursor) | + +Adding an app is one adapter file: see [the adapter guide](https://github.com/future-agi/docs/tree/main/examples/coding-agents/hook-to-otlp#writing-an-adapter). + +## Troubleshooting + +**No traces arrive.** Set `FI_HOOK_LOG_FILE=/tmp/fi-hook.log` and run one agent turn. No lines means the app did not run the hook (check the app's hook configuration and logs). `error=` lines mean the collector was unreachable; `http=401` or `http=403` means the keys are wrong. + +**Keys are set in my shell but nothing is sent.** Desktop apps started from the Dock or Start menu do not see shell variables. Put the variables in `~/.config/fi-hook-to-otlp/env`, or start the app from a terminal. + +**A span has `coding_agent.turn.inferred=true`.** The app did not report a turn id for that event, so the span was attached to the turn that was open at the time. + +**A tool span sits directly under the session.** The app reported no turn id for it and no turn was open at the time (for example, the app did not run its prompt hook). Such spans carry no turn marker; the session span is their parent. + +**I want to check exactly what is sent.** Set `FI_HOOK_DUMP_FILE=/tmp/fi-hook-dump.jsonl` and search it for a phrase from your prompt. With content capture off it must not appear. diff --git a/src/pages/docs/integrations/frameworks/ai-sdk-cpp.mdx b/src/pages/docs/integrations/frameworks/ai-sdk-cpp.mdx new file mode 100644 index 000000000..13a86d406 --- /dev/null +++ b/src/pages/docs/integrations/frameworks/ai-sdk-cpp.mdx @@ -0,0 +1,97 @@ +--- +title: "AI SDK C++ is not a supported tracing path" +description: "ClickHouse AI SDK C++ has no OTLP or OpenTelemetry export, and its Langfuse client cannot be pointed at Future AGI. No setup steps, and the evidence why." +--- + +## About + +[AI SDK C++](https://github.com/ClickHouse/ai-sdk-cpp) is ClickHouse's C++20 library for calling OpenAI and Anthropic models, with tools and multi-step runs. You cannot trace it into Future AGI today, so this page has no setup steps. It records what was checked, so you do not have to repeat it. + + +Not supported. No AI SDK C++ trace has been sent to Future AGI, and nothing on this page makes one appear. Every finding below comes from reading the source at one pinned commit and from running the real library against a local stand-in for a Langfuse server. + + +The only tracing code in the SDK is `ai::langfuse`, a client for Langfuse's own ingestion API. It does not speak OpenTelemetry, and it cannot send the headers or the project name that Future AGI requires. If ClickHouse adds an OTLP export, this page will be replaced by a setup guide. + +| | | +| --- | --- | +| Checked | 2026-10-16 | +| Repository | [ClickHouse/ai-sdk-cpp](https://github.com/ClickHouse/ai-sdk-cpp), Apache-2.0 | +| Commit | `9d8cd18200a33b64460c9101ed0dd6d4988bea61` (2026-08-19), still the head of `main` on 2026-10-16 | +| Release | None. No GitHub release and no tag. `CMakeLists.txt` says version 0.1.0 | +| OTLP or OpenTelemetry in the tree | 0 files | +| Built and run with | CMake 4.4.4, Apple clang 17, OpenSSL 3.6, macOS arm64. Model calls went to a local OpenAI-compatible endpoint | +| Source | [examples/frameworks/ai-sdk-cpp](https://github.com/future-agi/docs/tree/main/examples/frameworks/ai-sdk-cpp): the commands that produced these results, with tests | + +## What was checked + +All 194 tracked files at the pinned commit were searched, ignoring case: + +| Search | Files that match | +| --- | --- | +| `otlp` | 0 | +| `opentelemetry` | 0 | +| `exporter` | 0 | +| `traceparent` | 0 | +| `gen_ai` | 0 | +| `/v1/traces` | 0 | +| `langfuse` (control) | 7 | +| `x-api-key` (control) | 2 | + +The two control rows show that the search finds terms that are there. The tracing code that does exist is `include/ai/langfuse.h` (231 lines) and `src/langfuse/tracer.cpp`. Upstream has no open pull request for an OpenTelemetry export: the only open pull request adds Bedrock support (pull request list read on 2026-10-16). + +## Why the Langfuse client cannot be pointed at Future AGI + +The module was built from the pinned commit and run once with a tool call (`ai::langfuse::generate_text`, then `Trace::end()`), with the Langfuse host set to a loopback capture stub. This is what Future AGI needs, next to what arrived: + +| | Future AGI needs | `ai::langfuse` sends | +| --- | --- | --- | +| Format | OTLP traces | One JSON body, `{"batch": [...]}`, of Langfuse events: `trace-create`, `span-create`, `span-update`, `generation-create` | +| URL | The Future AGI trace endpoint | `/api/public/ingestion`. The path is a constant in the source. The host may carry a path prefix, but the suffix is always appended | +| Auth | `X-Api-Key` and `X-Secret-Key` headers | `Authorization: Basic`, built from a Langfuse public and secret key pair, plus `User-Agent` and two `X-Langfuse-Sdk-*` headers. There is no setting for other headers | +| Project | A `project_name` resource attribute | None. The batch is not OTLP, so it has no resource attributes | + +Future AGI receives traces over OTLP ([Observability model](/docs/observe/concepts/observability-model)), which this client does not speak. Pointing its `host` at Future AGI sends Langfuse-shaped JSON to a path that is not a Future AGI trace endpoint. + +Wrapping the client, or translating its batch into OTLP, would mean shipping a C++ component to Future AGI customers. The [traceAI repository](https://github.com/future-agi/traceAI) has Python, TypeScript, Java and C# packages and no C++ one (top-level folders on its `dev` branch, read 2026-10-16), and Future AGI has not built one for this SDK. + +## What `ai::langfuse` does at runtime + +Observed on the pinned build. Any future OTLP export for this SDK has to match these rules, because they decide whether the last trace of a process is lost. + +| Case | Observed | +| --- | --- | +| One run with a tool call | Events are held in memory and sent in one request at `end()`: a trace, a tool span under the generation, and the generation with model and token usage | +| `end()` called twice | One request in total. Both calls return true | +| Process exits without `end()` | Nothing is sent. The trace is lost | +| `generate_text` fails (unknown model) | One request. The generation has level `ERROR`, and the provider's raw error text is in `statusMessage` | +| Langfuse host unreachable, default `kStrict` | The first `end()` returns false and logs an error. The batch is dropped, not retried. A second `end()` returns true and sends nothing | +| Langfuse host unreachable, `kBestEffort` | `end()` returns true and logs an error | +| All of the above | No exception left the SDK | + +## Privacy + +Nothing from AI SDK C++ reaches Future AGI, so there is no capture setting to choose. + +For reference, `ai::langfuse` has no switch to leave content out. In the run above the batch carried the system and user prompt, the tool arguments and result, and the model output. In the failed run it carried the provider's raw error text. Any bridge to another backend would have to add its own redaction first. + +## Scope boundary + +This is a source-backed unsupported-status notice, not an installation guide. It does not prescribe a Langfuse-import bridge or a C++ OpenTelemetry component for AI SDK C++. + +## Troubleshooting + +**I set the Langfuse `host` to Future AGI and no traces appear.** This is expected. `ai::langfuse` sends Langfuse ingestion batches to `/api/public/ingestion` and never calls a Future AGI OTLP trace endpoint. + +**My last trace is missing.** `Trace::end()` is the only call that sends anything. A process that exits without it loses the trace. With the default `kStrict` policy, a failed flush is reported once and the batch is not retried. + +**Does a newer commit change this?** The findings are about the commit above. The source folder has a script that repeats the search on any commit. + +## When this changes + +This page is replaced when upstream ships an OTLP export and it is tested against Future AGI. Until then there is nothing to configure. + +## Next steps + +- [Observability model](/docs/observe/concepts/observability-model): how Future AGI receives and stores traces. +- [Integrations overview](/docs/integrations): the frameworks and providers that are supported. diff --git a/src/pages/docs/integrations/inferable.mdx b/src/pages/docs/integrations/inferable.mdx new file mode 100644 index 000000000..79fa4919b --- /dev/null +++ b/src/pages/docs/integrations/inferable.mdx @@ -0,0 +1,46 @@ +--- +title: "Inferable Tracing Is Not Supported Yet" +description: "Future AGI does not support tracing Inferable yet. What we checked in the Inferable source, why a Langfuse base URL is not a supported path, and what would change that." +--- + +Inferable ([inferablehq/inferable](https://github.com/inferablehq/inferable)) is a durable-execution runtime for AI workflows. **Future AGI does not support tracing it yet.** There is no setup guide on this page on purpose. + +Evidence date: 2026-10-16. Nothing on this page was run against a live Inferable cluster. The checks below are source and documentation reads only. + +## What we checked + +| Check | Result | +| --- | --- | +| Trace export in the Inferable control plane | The only export we found is the Langfuse integration (`control-plane/src/modules/integrations/langfuse.ts`). It builds a client from the `langfuse` npm package (`^3.31.0`) and calls its `trace()` and `generation()` methods. It does not use an OpenTelemetry SDK or an OTLP exporter for runs. | +| Settings on the Inferable side | Secret key, public key, base URL and a "send message payloads" switch. The base URL is user-settable ([pinned integration source](https://github.com/inferablehq/inferable/blob/c67acb81/control-plane/src/modules/integrations/langfuse.ts)). | +| Export hook in the Node or Go SDK | None found. | +| Source checked | `main` at `c67acb81` (committed 2025-07-05), MIT licence. The repository has had no later commit on `main`. | + +## Why this is not supported + +Future AGI ingests OpenTelemetry traces over OTLP. The Langfuse client that Inferable uses is not an OTLP exporter, so the request it sends is a Langfuse client request, not OTLP. We did not record that request, and a recipe that says "paste your Future AGI keys into the Langfuse fields" would be a guess. This page does not make that claim. + +We did not add a second ingest protocol, a new collector route or an adapter for it. The repository has had no commit on `main` since July 2025 and we have no customer request for it, so we do not think the effort is justified yet. + +## What you should not do + +- Do not paste Future AGI keys into the Inferable Langfuse fields and expect traces. +- Do not turn on "send message payloads" for an experiment. It sends prompts and tool arguments. + +## What would change this + +We will add a supported page if one of these becomes true: + +- Inferable (or a maintained fork) exports OTLP traces, or exports a body Future AGI already accepts. +- A recorded request from a self-hosted Inferable run shows a body Future AGI already accepts. The record would be attached to the page as evidence. + +If a recording shows only Langfuse's own batch body, this page stays "not supported". + +## Related + +If your application calls model providers directly, use the model instrumentors under [traceAI](/docs/integrations/traceai). This page is about Inferable's own runs and tool calls, which those instrumentors do not see. + +## Troubleshooting + +- **No traces appear in Future AGI from Inferable.** Expected. This integration is not supported yet. +- **A guide tells you to set a Langfuse base URL to Future AGI.** It is not verified. Do not follow it for production data. diff --git a/src/pages/docs/integrations/no-code/langdock.mdx b/src/pages/docs/integrations/no-code/langdock.mdx new file mode 100644 index 000000000..fdd83b5e1 --- /dev/null +++ b/src/pages/docs/integrations/no-code/langdock.mdx @@ -0,0 +1,46 @@ +--- +title: "Langdock with Future AGI: Not Supported Yet" +description: "Langdock has no documented Future AGI tracing path. A source-backed unsupported-status notice, with the protocol and configuration gaps that block direct support." +--- + + +**Status: not supported.** Langdock has no documented way to send agent traces to Future AGI. This page records what we checked so you do not have to, and what would change that. Nothing on this page was run against a Langdock workspace. + + +## What Langdock offers today + +Langdock is a closed, hosted AI platform for building no-code agents. Its only documented tracing path is Langfuse (checked 2026-10-16): + +- A **workspace admin** enables agent logging in the workspace settings (**Allow agent logs**). +- An **agent editor** then enables Langfuse tracing on each agent, sets the **Tracing cloud URL** (default `https://cloud.langfuse.com`), and adds Langfuse API keys. +- Langdock says a self-hosted Langfuse instance can be used by pointing the Tracing cloud URL at it. + +Sources: [Langdock agent configuration](https://docs.langdock.com/en/using-langdock/agents/configuration) and [Langfuse: Langdock integration](https://langfuse.com/integrations/no-code/langdock). + +## Why Future AGI cannot receive these traces + +- **No generic export.** Langdock documents no OTLP or other vendor-neutral export setting. The setting is specific to Langfuse and takes Langfuse public and secret keys. +- **Wire protocol unknown.** Langdock does not say whether it calls the Langfuse ingestion REST API or the Langfuse OpenTelemetry endpoint (`/api/public/otel`). We have not been able to observe it without a Langdock workspace. +- **Whether a non-Langfuse host is accepted is unverified.** The docs mention self-hosted Langfuse only. +- **Future AGI does not serve a Langfuse-compatible endpoint.** The Future AGI collector accepts OTLP/HTTP traces at `/v1/traces` and `/tracer/v1/traces` with `X-Api-Key` and `X-Secret-Key` headers. It has no Langfuse ingestion or `/api/public/otel` route, and it needs a `project_name` resource attribute that Langdock has no way to set. + +Because of this, pointing the Tracing cloud URL at Future AGI will not work. Do not try it with your production keys. + +## Supported scope + +There is no supported Langdock-to-Future-AGI setup in this guide. Neither changing the Langfuse host nor importing Langdock-origin traces is prescribed as a working integration. This notice supplies no installation commands or workaround recipe. + +## What would unlock direct support + +1. Langdock documenting a generic OTLP export, or accepting an arbitrary tracing endpoint with a documented protocol. +2. A Langfuse-ingestion-compatible endpoint in Future AGI. This is separate platform work and has not been scheduled. +3. Access to a Langdock workspace to verify the protocol and build a tested recipe. + +## Evidence boundary + +These findings are based on the linked vendor documentation, not a live Langdock integration. No Future AGI trace was produced from a Langdock workspace. + +## Troubleshooting + +- **You do not see "Allow agent logs" in Langdock.** Your workspace admin must enable agent logging first. This is a Langdock setting and is unrelated to Future AGI. +- **You pointed the Tracing cloud URL at Future AGI and see no traces.** Expected. Reset it to your Langfuse host. diff --git a/src/pages/docs/integrations/providers/z-ai.mdx b/src/pages/docs/integrations/providers/z-ai.mdx new file mode 100644 index 000000000..09b9a0588 --- /dev/null +++ b/src/pages/docs/integrations/providers/z-ai.mdx @@ -0,0 +1,222 @@ +--- +title: "Trace Z.ai chat completions" +description: "Trace Z.ai general platform calls made with the OpenAI Python client in Future AGI using traceAI-openai. One base URL, no new package, Coding Plan and zai-sdk not covered. Verified live against api.z.ai on 2026-10-17." +--- + +## About + +Z.ai's general platform serves an OpenAI-compatible chat API. You keep using `openai.OpenAI`, set `base_url` to the Z.ai general platform URL, and the existing [traceAI-openai](/docs/integrations/traceai/openai) instrumentor traces the call. There is no Z.ai package to install and nothing to configure on the Z.ai side. + +This page is not: + +- **A Z.ai export.** It traces calls your own code makes. It does not read Z.ai usage logs or billing, and it does not import history. +- **A new traceAI package.** There is no `traceai-zai`. The existing instrumentor handles the call, and [What is not captured](#what-is-not-captured) lists the gaps found. +- **Coding Plan setup.** The GLM Coding Plan is a different Z.ai product with different endpoints. See [Coding Plan is not this page](#coding-plan-is-not-this-page). +- **A recipe for Z.ai's own `zai-sdk`.** Its `ZaiClient` is not traced by traceAI-openai. See [Z.ai's own SDK is not traced](#zais-own-sdk-is-not-traced). + +The provider name on the span reads `openai`, not Z.ai. That is a display limit of the instrumentor, not something this page can change. + +
+ +| | | +| --- | --- | +| Tested with | traceAI-openai 0.1.10 (latest on PyPI, checked 2026-10-15), fi-instrumentation-otel 1.1.0, openai 3.31.0 (latest, Python 3.13) and 1.109.1 (Python 3.10). The traceAI-openai metadata requires Python below 3.14. | +| Z.ai docs read | [API introduction](https://docs.z.ai/api-reference/introduction), [OpenAI Python SDK guide](https://docs.z.ai/guides/develop/openai/python), [Chat completion reference](https://docs.z.ai/api-reference/llm/chat-completion), [Streaming](https://docs.z.ai/guides/capabilities/streaming), [Errors](https://docs.z.ai/api-reference/api-code), [Coding Plan quick start](https://docs.z.ai/devpack/quick-start), [Coding Plan tool integration](https://docs.z.ai/devpack/tool/others) and [Pricing](https://docs.z.ai/guides/overview/pricing) (search excerpt only), all read 2026-10-15 | +| Source | [examples/providers/z-ai](https://github.com/future-agi/docs/tree/main/examples/providers/z-ai) | + +
+ + +Verified against the real Z.ai general platform (`https://api.z.ai/api/paas/v4/`) on 2026-10-17 with the free `glm-4.7-flash` model: chat, streaming, a tool round trip, `thinking`, and a wrong-key failure all produced the spans described below. See [Live vendor verification](#live-vendor-verification-2026-10-17). Still not proven live: `finish_reason: network_error` on an abnormal stream, `request_id` handling, cost for GLM model ids, paid models, and every Z.ai API other than chat completions. Those rows say so, and the local stand-in (`STANDIN`) in the example is the only thing that exercises them. + + +## Install + +Install the OpenAI instrumentor and the OpenAI SDK. You do not install `zai-sdk`. + +```bash +pip install traceAI-openai openai +``` + +## Trace a chat completion + +Set the three keys. The Z.ai key goes to the OpenAI client. The Future AGI keys go to `register()`. Neither key belongs on a span. + +```bash +export ZAI_API_KEY="YOUR_ZAI_API_KEY" +export FI_API_KEY="YOUR_API_KEY" +export FI_SECRET_KEY="YOUR_SECRET_KEY" +``` + +Then register the tracer, install the instrumentor, and build the client. Replace the model with a current model code from Z.ai's model list. + +```python +import os + +from fi_instrumentation import register +from fi_instrumentation.fi_types import ProjectType +from openai import OpenAI +from traceai_openai import OpenAIInstrumentor + +trace_provider = register( + project_type=ProjectType.OBSERVE, + project_name="my-chatbot", +) +OpenAIInstrumentor().instrument(tracer_provider=trace_provider) + +client = OpenAI( + api_key=os.environ["ZAI_API_KEY"], + base_url="https://api.z.ai/api/paas/v4/", +) + +response = client.chat.completions.create( + model="your-model", + messages=[{"role": "user", "content": "What is the capital of France?"}], +) +print(response.choices[0].message.content) +``` + +
+ +| Piece | Value | +| --- | --- | +| Client | `openai.OpenAI` from the `openai` package. Not `zai.ZaiClient` | +| Base URL | `https://api.z.ai/api/paas/v4/`, with the trailing slash, as Z.ai's OpenAI Python guide prints it | +| Auth | `Authorization: Bearer `, which the OpenAI client sends for `api_key` | +| Model | A current model code from Z.ai. Z.ai's own examples use `glm-5.3` (read 2026-10-15); this page does not pin one | +| Project | `project_name` in `register()`. The instrumentor does not set it | +| Self-hosted Future AGI | Also export `FI_BASE_URL` with your collector URL | + +
+ +Keep the trailing slash. It is the string in Z.ai's OpenAI Python guide. Z.ai's API introduction prints the same host without the slash as the general endpoint, and the OpenAI client sends both to the same path, `/api/paas/v4/chat/completions`. + +Build the client after `register()` and `instrument()`, as above. Z.ai still receives your prompt: tracing sends a copy to Future AGI and does not change what the API sees. + +### Coding Plan is not this page + + +If your base URL is one of the three below, this page does not apply. Z.ai's GLM Coding Plan has dedicated endpoints for coding tools: `https://api.z.ai/api/anthropic` (Anthropic Messages), `https://api.z.ai/api/coding/paas/v4` (OpenAI Chat Completions) and `https://api.z.ai/api/v1` (OpenAI Responses). Z.ai says that incorrect endpoint configuration will result in being unable to use Coding Plan subscription quota, and that the Coding Plan is limited to its officially supported tools and products. Z.ai's Team Plan key is also not interchangeable with its other API keys. This page does not document the Coding Plan, and the example app refuses all three URLs. + + +The URLs and warnings are from Z.ai's [Coding Plan quick start](https://docs.z.ai/devpack/quick-start) and [tool integration](https://docs.z.ai/devpack/tool/others) pages, read 2026-10-15. Treat Z.ai's pages as the authority if they differ. A call to a Coding Plan URL through the OpenAI client would still be traced, because the instrumentor wraps the client and does not look at the URL. That is why this page names them: the trace would look the same and the bill would not. + +### Z.ai's own SDK is not traced + +Z.ai ships `zai-sdk` with a `ZaiClient`. It is not a subclass of `openai.OpenAI`, and traceAI-openai produced no spans for it while the stock `openai.OpenAI` client, pointed at the same server, produced one. This was checked with zai-sdk 0.2.3 (the version Z.ai's docs install) against the local stand-in, not against Z.ai. If you use `ZaiClient`, this page gives you no traces. Switch to `openai.OpenAI` with the URL above, which Z.ai's own OpenAI Python guide supports. + +`zai-sdk` also ships `ZhipuAiClient`, whose default URL in that package is `https://open.bigmodel.cn/api/paas/v4`. That is Zhipu's mainland platform. It is not the endpoint in Z.ai's OpenAI Python guide, and this page does not cover it. + +## What you get + +Each call below produced the spans shown. "Live" means the real Z.ai API on 2026-10-17 with `glm-4.7-flash`; "stand-in" means the Z.ai-shaped local server in the example, which is not Z.ai. + +
+ +| Call | What lands in Observe | Notes | +| --- | --- | --- | +| `chat.completions.create` | One `LLM` span with the model, input and output messages, and input, output and total tokens | On a non-streaming call the model on the span is the `model` field of the response body, not the one you sent | +| `stream=True` | One `LLM` span with a first-token event, the joined output text and token counts | Z.ai's streaming guide shows `usage` on the last chunk with no `stream_options`, so the recipe does not send one. Live: the span carried the counts from the last chunk (no `stream_options` sent). On 0.1.10 the live streaming span has no model | +| Tool calls | Two `LLM` spans, one per round trip. Both carry the tool definitions | Running the tool is your code. It becomes a span only if you create one. Z.ai documents `tool_choice` as `auto` only | +| A failed call | An `LLM` span with status `ERROR`, the exception type and the HTTP status in the message, and an `exception` event | Live: a wrong key gave HTTP 401, body `{'error': {'code': '401', 'message': 'token expired or incorrect'}}`, and an `ERROR` span with that message. A live 429 (`code` 1305, "The service may be temporarily overloaded") also became an `ERROR` span | +| `extra_body={"thinking": {"type": "enabled"}}` | The call is traced like any other | The field reaches Z.ai. It is not a span attribute | + +
+ +Spans group under any parent span or session you create. The example app wraps its calls in one `AGENT` span and a session id so the whole run is one trace. + +## What is not captured + +- **`thinking` and reasoning text.** The `thinking` field travels in `extra_body`, and the instrumentor does not record it. Reasoning text in `reasoning_content`, whether in a response or a stream, is not on the span either. The span reports the response's `usage.completion_tokens` as output tokens; live, `glm-4.7-flash` includes reasoning tokens in `usage.completion_tokens` (a 20-token reply reported `completion_tokens` 20 and `reasoning_tokens` 20), so a short answer can show a large output token count. +- **Finish reason.** The span does not record `finish_reason`. Z.ai documents that a stream which ends abnormally reports the cause in `finish_reason` (for example `network_error`) instead of an HTTP error. Against the stand-in (not seen live), such a stream produced a span with status `OK` that shows only the text that arrived. Check `finish_reason` in your own code if truncated streams matter to you. +- **Cached tokens.** Z.ai reports `prompt_tokens_details.cached_tokens`. The span carries input, output and total tokens only. +- **The host.** The exported span carries no Z.ai hostname. Put the endpoint in your `project_name` or in your own span attribute if you need to tell Z.ai calls from other OpenAI-client calls. +- **Z.ai request ids and headers.** The `request_id` in Z.ai's response body, a `request_id` you send, and an `Accept-Language` header are not span attributes and do not become a Future AGI session id. +- **The model on streaming and failed calls, on 0.1.10.** traceAI-openai 0.1.10 leaves `gen_ai.request.model` off streaming spans and failed-call spans. [traceAI PR 249](https://github.com/future-agi/traceAI/pull/249) (merged to dev; unavailable in the 0.1.10 release used here) records it. The tests in the example pass on both. +- **Provider name.** `gen_ai.provider.name` is `openai` for `api.z.ai`. PR 249 adds a host-to-provider table that does not contain `api.z.ai`, so the value stays `openai` there too. +- **Cost.** The span carries no price. Whether the platform prices real GLM model ids was not checked, and `glm-4.7-flash` is free anyway. +- **Other Z.ai APIs.** Web search, retrieval and vision tools, image/video generation, audio, embeddings, and the Responses API are outside this recipe’s supported scope. The supported path is Python OpenAI-client chat completions on the general-platform endpoint. +- **Free-tier rate limits.** With the free model, two privacy runs started back to back each saw a few HTTP 429 (code 1305) responses. The OpenAI client retried, and calls that still failed were traced as `ERROR` spans. +- **Other clients.** This recipe supports the Python OpenAI client only; TypeScript, Java, and `zai-sdk` are outside its supported scope. + +## Privacy + +traceAI-openai follows the standard traceAI `TraceConfig` defaults: prompts, completions, tool arguments and results, and request parameters are captured unless you turn them off. Set these before the process starts, or pass the same settings to `TraceConfig`. + +```bash +export FI_HIDE_INPUTS=true +export FI_HIDE_OUTPUTS=true +export FI_HIDE_LLM_INVOCATION_PARAMETERS=true +``` + +With those three set, the model, token counts, status and timing still arrive and the content does not. Input and output fields hold `__REDACTED__`, the request parameters attribute is absent, and a unique marker placed in the system prompt, the user prompt, the model's answer, the tool-call arguments and the tool result appeared in no stored column. With the variables unset, the same marker appeared in the prompts, answers, tool arguments and results. Content is therefore on by default. To turn it back on after hiding it, unset the variables or set them to `false`. + +What the hide settings do not remove on traceAI-openai 0.1.10: + +- **Tool definitions.** The name, description and JSON schema of tools you pass are still on the span. +- **Session id.** The id you set, if any, stays. +- **Error text.** The status message, such as `AuthenticationError: Error code: 401 - ...`, includes the error body Z.ai returns. The exported `exception` event also carries the exception message and a stack trace with local file paths. The local platform kept only the event's name and time and did not store the stack trace or message, but they are on the wire. + +If your errors can contain sensitive text, do not rely on the hide settings for them. + +## Run it yourself + +The example in [`examples/providers/z-ai`](https://github.com/future-agi/docs/tree/main/examples/providers/z-ai) contains the recipe as a runnable app, a Z.ai-shaped stand-in, and tests. Its `run_e2e.sh` installs the pinned versions in a virtual environment, runs the tests, and exports to your collector. + +```bash +cd examples/providers/z-ai +export FI_API_KEY="YOUR_API_KEY" FI_SECRET_KEY="YOUR_SECRET_KEY" +export FI_BASE_URL="http://localhost:4318" # your collector, if self-hosted +./run_e2e.sh # local stand-in, no Z.ai account needed + +ZAI_API_KEY="YOUR_ZAI_API_KEY" ZAI_MODEL="your-model" ./run_e2e.sh # real Z.ai, may cost money +``` + +With `ZAI_API_KEY` set and no `ZAI_BASE_URL`, the script calls `https://api.z.ai/api/paas/v4/`. It never creates an account. Z.ai's pricing page lists some models as free of charge; check it for the model you pick. + +## Troubleshooting + +
+ +| Symptom | Cause and fix | +| --- | --- | +| The span says `openai`, not Z.ai | Known display limit. Look for the project and model, not the provider name | +| HTTP 401 from Z.ai | A wrong or missing key. Z.ai documents business code 1000 for a failed authentication and 1001 for a missing one, in an `error` object with `code` and `message`. Check that `ZAI_API_KEY` is a general platform key, not a Coding Plan key | +| HTTP 429 from Z.ai | Z.ai documents several 429 business codes for rate limits and usage limits, including some that name the Coding Plan. Read the `code` and `message` in the exception | +| No spans appear | Install the instrumentor with `tracer_provider=trace_provider` and use the `openai` package. Check `FI_API_KEY`, `FI_SECRET_KEY` and, if self-hosted, `FI_BASE_URL`. See [No traces appearing](/docs/observe/troubleshooting/no-traces-appearing) | +| No spans when using Z.ai's `ZaiClient` | traceAI-openai does not instrument `zai.ZaiClient`. Switch to `openai.OpenAI` | +| Calls are traced but billed differently than expected | Check the base URL. The three Coding Plan URLs above are a different product, which this page does not cover | +| `ai-evaluation is not installed, please install it to trace protect` printed at startup | Warning logged by `traceai-openai` on import. It concerns tracing Protect guardrails and did not affect chat tracing in these runs | +| `pip install` fails on Python 3.14 | traceAI-openai 0.1.10 requires Python below 3.14. Use 3.10 to 3.13 | + +
+ +## How this differs from the OpenAI page + +The [OpenAI integration](/docs/integrations/traceai/openai) assumes `api.openai.com`. This page assumes a different vendor's endpoint behind the same client: the one supported base URL, the Coding Plan URLs to avoid, the SDK that is not covered, and what the spans do and do not show for a vendor that is not OpenAI. + +## Next steps + +- [OpenAI integration](/docs/integrations/traceai/openai) for everything traceAI-openai captures on `api.openai.com`. +- [No traces appearing](/docs/observe/troubleshooting/no-traces-appearing) if the export does not arrive. + +## Live vendor verification (2026-10-17) + +Run exactly as this page describes, with `ZAI_API_KEY` and `ZAI_MODEL=glm-4.7-flash` (free) and no `ZAI_BASE_URL`, so the client called `https://api.z.ai/api/paas/v4/`. Traces went to a local Future AGI (project `th8103-rest-z-ai-live`). Versions: Python 3.13.15, traceAI-openai 0.1.10, fi-instrumentation-otel 1.1.0, openai 3.31.0, opentelemetry-sdk 1.45.1. + +- Default run, trace `57201cc8-31fd-48af-8b10-3340ba8246db`, `fi_verify` PASS: one `AGENT` span with four `LLM` spans and one `ERROR` span. +- Content capture off (`FI_HIDE_*`), trace `42a48102-d387-455c-8c9c-78a90e9a3594`: includes the `TOOL` span; the unique marker was found in 0 stored `spans` columns. +- Default, trace `22c8eb0f-80a5-453d-bd92-0d0f6eda7216`: the marker was found in `attrs_string`, `input`, `output` and `attributes_extra`. + +```text +trace 57201cc8-31fd-48af-8b10-3340ba8246db (6 spans) +- [agent] z-ai-demo + - [llm] ChatCompletion model=glm-4.7-flash tok=14/367 + - [llm] ChatCompletion tok=14/1079 <- stream: usage present, model missing on 0.1.10 + - [llm] ChatCompletion model=glm-4.7-flash tok=165/139 + - [llm] ChatCompletion model=glm-4.7-flash tok=14/1332 + - [llm] ChatCompletion status=ERROR:AuthenticationError: Error code: 401 - token expired or incorrect +PASS +``` + +What the live run showed that the stand-in could not: the model on the span is `glm-4.7-flash`, the `provider` is `openai`, reasoning tokens are inside `completion_tokens`, `reasoning_content` is not on the span, and the stream span lacks the model on 0.1.10 (traceAI PR 249 addresses this). The exception event still carries a stack trace and local paths with the hide flags on; that is the existing traceai-openai behaviour described above. diff --git a/src/pages/docs/integrations/traceai/openai_assistants.mdx b/src/pages/docs/integrations/traceai/openai_assistants.mdx index b88898cd3..da7728a72 100644 --- a/src/pages/docs/integrations/traceai/openai_assistants.mdx +++ b/src/pages/docs/integrations/traceai/openai_assistants.mdx @@ -7,7 +7,7 @@ description: "The OpenAI Assistants API was sunset on 26 August 2026 and is no l The OpenAI Assistants API was sunset on 26 August 2026 and is no longer available. There is no supported Future AGI setup for it. Do not look here for setup steps. This page has none. -Alpha. This page is a read of OpenAI's migration guide, not a live call. No request to OpenAI was made to write it. +This page is a read of OpenAI's migration guide, not a live call. No request to OpenAI was made to write it. OpenAI's instruction on the migration guide (retrieved 2026-10-15): "The Assistants API was officially sunset on August 26, 2026, and is no longer available. Use the Responses API for new integrations."