From 5fe3b10580f001fabbea402b942ade2457d3565d Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 13 Oct 2026 19:33:54 +0000 Subject: [PATCH 01/10] feat(telemetry): add export reporting and document the default-off build Replay the reporting and docs slice onto current main. Telemetry stays off. The local report aggregates exported hook observations; gateway telemetry remains canonical. Co-authored-by: Teal Larson --- AGENTS.md | 11 + ARCHITECTURE.md | 12 +- README.md | 11 + docs/install/claude-code.md | 13 + docs/install/copilot.md | 13 + docs/install/vscode.md | 3 + docs/support-matrix.md | 21 +- docs/telemetry.md | 387 +++++++++++++++ scripts/generate-manifests.mjs | 12 + scripts/telemetry-docs.mjs | 59 +++ scripts/telemetry-report.mjs | 449 ++++++++++++++++++ test/fixtures/telemetry-report/events.jsonl | 21 + .../telemetry-report/expected-report.json | 111 +++++ test/generate-manifests.test.mjs | 5 + test/helpers.mjs | 4 +- test/telemetry-docs.test.mjs | 29 ++ test/telemetry-report.test.mjs | 369 ++++++++++++++ 17 files changed, 1522 insertions(+), 8 deletions(-) create mode 100644 docs/telemetry.md create mode 100644 scripts/telemetry-docs.mjs create mode 100644 scripts/telemetry-report.mjs create mode 100644 test/fixtures/telemetry-report/events.jsonl create mode 100644 test/fixtures/telemetry-report/expected-report.json create mode 100644 test/telemetry-docs.test.mjs create mode 100644 test/telemetry-report.test.mjs diff --git a/AGENTS.md b/AGENTS.md index bc09b6b..257cd90 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -14,6 +14,17 @@ for a marked rules block. - Hooks: add a hook or a client in `hooks/hook-hosts.mjs`, never in a `hooks.json`. A new client also needs its expected output in `test/hooks.test.mjs`. +- Telemetry: `hooks/telemetry-contract.mjs` defines every event, property, + and allowed value; change them there only. Client-specific hook input mapping + lives only in `hooks/telemetry-adapters/.mjs`. Wire a client through + `HOSTS.telemetry` in `hooks/hook-hosts.mjs` (generated manifests follow + that source). `telemetry.mjs` prints nothing; only `telemetry-send.mjs` may + touch the network (a test enforces both). `TELEMETRY_ENABLED` in + `hooks/telemetry-config.mjs` stays `false` without separate approval; while + false, `npm run generate` writes no telemetry hooks. Reporting and maintained + docs for exports live in `scripts/telemetry-report.mjs` and + `docs/telemetry.md`. `test/telemetry-report.test.mjs` cross-checks adapters + when `hooks/telemetry-adapters/.mjs` exists on the branch. - Codex hooks are blocked upstream ([docs/install/codex.md](docs/install/codex.md)). Don't remove the root `$schema` to force them; that breaks Agent Plugins conformance. diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 207d7a3..eadc0c7 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -85,6 +85,12 @@ success or narrating tool internals. The Arcade MCP server is the canonical place to record request, authentication, tool-discovery, tool-call, and completion outcomes. This package does not ask a -model to self-report tokens, turns, or success, and it ships no telemetry hook. -If a host-specific hook later adds supplemental signals, it must be explicit, -opt-in, and documented as non-portable. +model to self-report tokens, turns, or success. + +Optional client hooks in `hooks/telemetry-adapters/` can send supplemental, +scoped usage events when `TELEMETRY_ENABLED` is `true` and the user has not +opted out. **This build keeps `TELEMETRY_ENABLED` false**, so no telemetry +hooks are generated and nothing is sent. See [docs/telemetry.md](docs/telemetry.md) +for the contract, opt-outs, local state, and `scripts/telemetry-report.mjs` +for aggregating exports. Gateway telemetry remains authoritative; plugin events +do not establish task success. diff --git a/README.md b/README.md index e72108b..f444dd3 100644 --- a/README.md +++ b/README.md @@ -69,6 +69,17 @@ anything is sent, created, or deleted. - Privacy: tasks run through Arcade's hosted gateway and the apps you connect — [privacy policy](https://www.arcade.dev/privacy-policy). +## Plugin usage events (off in this build) + +Claude Code and Copilot CLI include telemetry adapters, but **`TELEMETRY_ENABLED` +is `false`**: this package sends no usage events and generates no telemetry +hooks in client manifests. [docs/telemetry.md](docs/telemetry.md) describes +the event contract, what would be stored locally when enabled, opt-outs, and how +to aggregate exports with `scripts/telemetry-report.mjs`. The Arcade MCP +gateway remains the canonical source for request and tool-call telemetry. +Adapter behavior is covered by in-process tests when those files are present; +see [docs/telemetry.md](docs/telemetry.md) for what CI does and does not exercise. + ## Develop Agents editing this repo should read [AGENTS.md](AGENTS.md). diff --git a/docs/install/claude-code.md b/docs/install/claude-code.md index b8b4bfa..e0678b5 100644 --- a/docs/install/claude-code.md +++ b/docs/install/claude-code.md @@ -37,6 +37,19 @@ If your host has other Arcade MCP connectors too, Claude may pick the wrong one connected and prefer disabling other Arcade connectors while testing this plugin. +## Telemetry + +Telemetry is **off** in this build (`TELEMETRY_ENABLED` is `false`): no usage +events are sent and generated manifests include no telemetry hooks. When +telemetry is enabled, the adapter in `hooks/telemetry-adapters/claude-code.mjs` +classifies prompts, records `PreToolUse` on the plugin and claude.ai Arcade +prefixes, and stores scope under `CLAUDE_PLUGIN_DATA`. See +[telemetry.md](../telemetry.md) for opt-outs (`ARCADE_PLUGIN_TELEMETRY=0`, +`DISABLE_TELEMETRY`, `CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC`) and limits. + +**Repo coverage:** in-process hook tests and `claude plugin validate --strict` +(**2.1.258**). Not live sessions, IDE extensions, desktop Code tab, or Cowork. + ## First steps - "What's on my calendar tomorrow?" diff --git a/docs/install/copilot.md b/docs/install/copilot.md index 542ae24..6ad5682 100644 --- a/docs/install/copilot.md +++ b/docs/install/copilot.md @@ -26,6 +26,19 @@ from config files, so there's no per-prompt reminder here. VS Code reads the same file but can't run Agent Plugins hook commands yet, so it relies on the skills. +## Telemetry + +Telemetry is **off** in this build: generated manifests include no telemetry +hooks and nothing is sent. When enabled, `hooks/telemetry-adapters/copilot-cli.mjs` +records MCP tools as `-`, operator stops with `subagent_session`, +and session-scoped prompt state under `COPILOT_PLUGIN_DATA` (no `turn`, no +`PreToolUse`, no built-in CLI/web events). See [telemetry.md](../telemetry.md). + +**Repo coverage:** in-process fixtures and `npm run verify:copilot` (**1.0.88**). +Not live sessions, Windows PowerShell hook commands, or VS Code agent sessions. +Shared `com.github.copilot/hooks/hooks.json` uses `runOnlyIfScriptExists` so VS +Code exits quietly without a plugin path. + ## First steps - "What's on my calendar tomorrow?" diff --git a/docs/install/vscode.md b/docs/install/vscode.md index 35e824b..b57c9ed 100644 --- a/docs/install/vscode.md +++ b/docs/install/vscode.md @@ -17,6 +17,9 @@ VS Code loads root `plugin.json` as an Agent Plugin: 2 skills, the gateway, and `arcade-operator` from `com.github.copilot/agents/`. It does not read the `.cursor-plugin/` adapter. No lifecycle hooks: VS Code doesn't yet run Agent Plugins hook commands with the plugin's path or pass their output to the model. +Copilot hook commands check that the script exists at the plugin path and exit +when the path is missing; this package has no validated VS Code telemetry flow, +and nothing is sent from VS Code. If you already installed the plugin via Copilot CLI, VS Code may auto-discover it from `~/.copilot/installed-plugins/`. Install in one place. diff --git a/docs/support-matrix.md b/docs/support-matrix.md index c64034c..7e26775 100644 --- a/docs/support-matrix.md +++ b/docs/support-matrix.md @@ -27,6 +27,19 @@ session-start text doesn't reach the main conversation. The files are `clients/cursor/hooks/hooks.json` (Cursor CLI), and `com.github.copilot/hooks/hooks.json` (Copilot CLI). +Telemetry adapters are separate from routing. **`TELEMETRY_ENABLED` is false** +in this build, so manifests include no telemetry hooks and nothing is sent. +When enabled, wiring comes from `hooks/hook-hosts.mjs`: + +| Client adapter | Telemetry events (when enabled) | +| --- | --- | +| Claude Code | `UserPromptSubmit`, `PreToolUse` on `^(?:mcp__plugin_arcade_arcade__\|mcp__claude_ai_arcade__)`, MCP `PostToolUse` / `PostToolUseFailure`, built-in `WebFetch` / `WebSearch` / listed `Bash` CLIs, Arcade operator `SubagentStop` | +| Copilot CLI | `UserPromptSubmit`, MCP `PostToolUse` / `PostToolUseFailure` (`-`), Arcade operator `SubagentStop` with `subagent_session` (no `PreToolUse`, no `turn`, no built-in tools) | + +Session start clears local prompt scope through the routing hook; it sends no +telemetry event. No other adapter in this package sends telemetry. See +[what's sent and its limits](telemetry.md). + ¹ The Cursor IDE (3.21.18) lists the commands on the plugin page but not in the `/` menu. Other plugins' commands don't appear there either. ² The IDE (3.21.18) and Cloud Agents don't run plugin hooks, so the @@ -37,10 +50,12 @@ load the always-apply rule. ⁴ Cowork runs the prompt and subagent hooks but doesn't add the session-start text, so its main conversation gets the short reminder and the skill, not the full rules. -⁵ Copilot CLI drops the output of prompt hooks from config files, so it gets -session and subagent hooks only. +⁵ Copilot CLI drops the output of **routing** prompt hooks from config files, +so the main conversation gets session and subagent hooks only. Telemetry still +hooks `UserPromptSubmit` when enabled; `reminder_sent` is always `false` there. ⁶ VS Code reads `com.github.copilot/hooks/hooks.json` but doesn't expand -`${PLUGIN_ROOT}` for Agent Plugins hooks or pass their output to the model yet. +`${PLUGIN_ROOT}` for Agent Plugins hooks or pass their output to the model yet; +`runOnlyIfScriptExists` makes those commands no-ops without a plugin path. ⁷ Blocked upstream; see [codex.md](install/codex.md). Skills are `try-arcade` and `scale-arcade`. The subagent is diff --git a/docs/telemetry.md b/docs/telemetry.md new file mode 100644 index 0000000..95240b6 --- /dev/null +++ b/docs/telemetry.md @@ -0,0 +1,387 @@ +# Plugin telemetry + +**Telemetry is off in this build.** The plugin sends no usage events, and +`npm run generate` writes no telemetry hooks into any client manifest. +`TELEMETRY_ENABLED` in `hooks/telemetry-config.mjs` stays `false` until +collection is separately approved after ingestion, opt-outs, privacy, and +distribution requirements are verified. + +The sections below describe what **would** be sent when telemetry is turned on, +how to read exported events, and how that relates to gateway telemetry and +offline routing evaluation. Nothing here implies that this package currently +transmits data. + +When enabled, Claude Code and Copilot CLI hooks would inspect prompts locally +in each session to recognize app-related work. Relevance state belongs to that +session; it does not carry between sessions. Prompts classified as unrelated +would send no event. Direct Arcade tool calls would remain observable even +without a classified prompt. Alternative MCP, CLI, and web tools would send +events only during app-related work. + +Events would carry hashed session IDs, with no ID lasting across sessions. They +would exclude prompt text, commands, app data, names, email addresses, and Arcade +account IDs. These observations do not establish task success or whether +Arcade was needed. Install and signed-in-user counts require gateway data. + +## Prompt scope + +A prompt matching the local app classifier or mentioning Arcade would open a +30-minute observation period for its session. A short explicit confirmation such +as “yes, send it” would continue that period without extending its expiry. An +unrelated substantive prompt would close it. Background task notifications would +leave the current period unchanged. Expired, absent, or invalid state would +produce no alternative-tool telemetry. + +`could_use_arcade` and `service_hints` describe keywords in the current prompt, +not the preceding task. A confirmation reply can therefore send a scoped prompt +event with `could_use_arcade: false` and no service hints. Keyword matching can +misclassify prompts; the labeled evaluation measures that limitation separately. + +Session starts send no usage event. Only the Arcade operator's stop reports +send subagent events. The routing reminder goes on every prompt except short +acknowledgements and background task results, whether or not telemetry is on. + +## Opt-outs when telemetry is on + +While `TELEMETRY_ENABLED` is `false`, these switches are inert but documented +for a future enabled build. They are implemented in `hooks/telemetry-run.mjs` +and each client's adapter in `hooks/telemetry-adapters/`. + +Set `ARCADE_PLUGIN_TELEMETRY=0` in your environment, or in Claude Code's +`settings.json`: + +```json +{ "env": { "ARCADE_PLUGIN_TELEMETRY": "0" } } +``` + +`false`, `off`, and `no` also work. Telemetry is also off when `DO_NOT_TRACK` +is set to any non-empty value other than `0`, `false`, `off`, or `no`. In +Claude Code, `DISABLE_TELEMETRY` and `CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC` +opt out when set to **any** non-empty value (including `0` and `false`), matching +Claude's own telemetry switches. + +With telemetry opted out, the client still invokes its configured Node hooks. +The telemetry hook exits before classifying the prompt or storing state, and the +routing hook still adds its reminder. An environment variable cannot remove +hooks from the manifest. + +In Copilot CLI, set `ARCADE_PLUGIN_TELEMETRY=0` in your shell before starting +`copilot`. `COPILOT_OFFLINE` uses the same `0` / `false` / `off` / `no` semantics +as `ARCADE_PLUGIN_TELEMETRY`; any other non-empty value turns telemetry off +(along with other Copilot network activity). + +For testing an enabled build, `ARCADE_PLUGIN_TELEMETRY_HOST` sends events to a +different host. + +## Where it runs + +The Claude Code and Copilot CLI adapters are the only ones wired for telemetry +in `hooks/hook-hosts.mjs` (`hooks/telemetry-adapters/claude-code.mjs` and +`hooks/telemetry-adapters/copilot-cli.mjs`). Claude maps MCP tools under +`mcp__plugin____` (this plugin's gateway) and +`mcp__claude_ai_arcade__` (the claude.ai Arcade connection). Attempt events +(`PreToolUse`) are emitted only for those two prefixes. Copilot maps MCP tools +as `-` (for example `arcade-Arcade_SelectTools` when the MCP +server name matches `mcp.json`). Copilot sends no `turn`, no `PreToolUse` +attempt events, and no built-in CLI or web tool events. + +Claude ties alternative-tool observations to the hashed `turn` for the +current `prompt_id` (`requiresTurn: true`). Copilot keeps prompt scope per +`session` without a prompt ID (`requiresTurn: false`). + +Local state uses `CLAUDE_PLUGIN_DATA` (Claude Code) and `COPILOT_PLUGIN_DATA` +(Copilot CLI) for `arcade-used` and `prompt-scope/` files. + +Copilot's manifest is shared with VS Code. Hook commands use +`runOnlyIfScriptExists`: each command checks that its script exists at +`${PLUGIN_ROOT}` and exits quietly when the path is missing, so VS Code does +not run telemetry today. Cursor is not wired up; its hook input can include the +user's email. claude.ai, ChatGPT, Codex, and OpenCode don't run telemetry hooks +from this package. + +### What CI exercises (when adapters are present) + +| Adapter | Tested in this repo | Not covered here | +| --- | --- | --- | +| Claude Code | In-process hook captures, contract tests, `scripts/telemetry-report.mjs` cross-check, `claude plugin validate --strict` on **2.1.258** | Live user sessions, IDE extensions, desktop Code tab, Cowork delivery and opt-out | +| Copilot CLI | In-process captures, fixture JSON, report cross-check, `npm run verify:copilot` on **1.0.88** | Live sessions, Windows PowerShell hook commands, VS Code agent sessions | + +Those checks do not prove end-user transmission while `TELEMETRY_ENABLED` is +`false`; they validate the adapters and reporting math for an enabled build. + +## What is stored on your machine + +When telemetry is enabled, each client's plugin data folder contains its own +`arcade-used` flag, readable only by you. It holds `true` after an Arcade call +succeeds in that client plugin installation and supplies `arcade_used_before`. + +Prompt relevance state lives in `prompt-scope/.json` in +the same folder. It contains a relevance boolean, expiry timestamp, and optional +hashed prompt ID that ties tool calls to their prompt. It contains no prompt text, +commands, tool arguments, or service content. Expired state cannot authorize +observation. The next prompt-state write removes expired or malformed entries; +at most 256 session state files are retained. Session start clears that +session's state, except after Claude Code compacts the conversation. Claude +alternative-tool observations require a matching hashed prompt ID; an absent +prompt ID cannot authorize those observations. + +- Claude Code: `~/.claude/plugins/data//` +- Copilot CLI: `~/.copilot/plugin-data/<…>/` + +If the client supplies no plugin data folder, the plugin sends nothing. + +## What is sent + + +Every event has these properties: + +| Property | Value | +| --- | --- | +| `distinct_id` | the same value as `session` | +| `session` | `sha256(session_id)`, first 16 hex characters, where `session_id` is the client's random ID for the session | +| `turn` | `sha256(session_id + ":" + prompt_id)`, first 16 hex characters; Claude Code only, because Copilot CLI has no prompt ID | +| `arcade_used_before` | whether an Arcade tool call had succeeded for this client plugin installation before this event (from the `arcade-used` file) | +| `host` | `claude-code` \| `copilot-cli` | +| `telemetry_version` | `2`, the scoped event contract; earlier events have no version | +| `plugin_version` | from `VERSION` | +| `os` | `darwin` \| `linux` \| `win32` \| `other` | +| `$process_person_profile` | `false` | +| `$geoip_disable` | `true` | +| `$ip` | `0.0.0.0`, so PostHog stores this instead of your real IP address | + +Events and their extra properties: + +| Event | When | Extra properties | +| --- | --- | --- | +| `Plugin prompt submitted` | UserPromptSubmit, only for locally classified app work and short confirmations of that work; background task results are excluded. In Copilot CLI a relevant subagent prompt also sends it | `could_use_arcade`: boolean, a local keyword guess (see below). `service_hints`: service categories the prompt mentions. `reminder_sent`: boolean, whether the routing reminder was added (always `false` in Copilot CLI). | +| `Plugin tool attempted` | PreToolUse, in Claude Code, before an MCP call through this plugin's Arcade gateway or the claude.ai Arcade connection; an attempt does not show whether the tool finished | `server`: `arcade` (this plugin's gateway) \| `other_arcade` (the claude.ai Arcade connection). `tool`: only for `arcade` and `other_arcade`: an exact gateway tool name, `app_tool` for a recognized service category, or `other`; app tool names are never sent. `service`: the service category, when the tool, the app tool passed to `Arcade_UseTool`, or the server name matches one. | +| `Plugin tool called` | PostToolUse, on Arcade tools, or alternative MCP tools while the current turn concerns app work | `server`: `arcade` (this plugin's gateway) \| `other_arcade` (another connection exposing Arcade's gateway tools) \| `other`. `tool`: only for `arcade` and `other_arcade`: an exact gateway tool name, `app_tool` for a recognized service category, or `other`; app tool names are never sent. `service`: the service category, when the tool, the app tool passed to `Arcade_UseTool`, or the server name matches one. `auth_needed`: only for `System_ManageAuthorization`: whether its answer says a service still needs sign-in. | +| `Plugin tool failed` | PostToolUseFailure, on Arcade tools, or alternative MCP tools while the current turn concerns app work | `server`: `arcade` (this plugin's gateway) \| `other_arcade` (another connection exposing Arcade's gateway tools) \| `other`. `tool`: only for `arcade` and `other_arcade`: an exact gateway tool name, `app_tool` for a recognized service category, or `other`; app tool names are never sent. `service`: the service category, when the tool, the app tool passed to `Arcade_UseTool`, or the server name matches one. `failure_kind`: picked on your machine from the error message; the message is not sent: `auth_required` \| `session_expired` \| `unreachable` \| `timeout` \| `http_error` \| `interrupted` \| `tool_error`. | +| `Plugin built-in tool called` | PostToolUse, Claude Code only, on `WebFetch` and `WebSearch`, and on `Bash` commands that run `gh` \| `glab` \| `curl` \| `wget` \| `http` \| `osascript`; only while the current turn concerns app work | `tool`: `Bash` \| `WebFetch` \| `WebSearch`. `cli`: only for `Bash`: the program the command runs, `gh` \| `glab` \| `curl` \| `wget` \| `http` \| `osascript`. `service`: the program's service category, only for `gh` \| `glab`: `code_hosting`. | +| `Plugin built-in tool failed` | PostToolUseFailure, Claude Code only, on `WebFetch` and `WebSearch`, and on `Bash` commands that run `gh` \| `glab` \| `curl` \| `wget` \| `http` \| `osascript`; only while the current turn concerns app work | `tool`: `Bash` \| `WebFetch` \| `WebSearch`. `cli`: only for `Bash`: the program the command runs, `gh` \| `glab` \| `curl` \| `wget` \| `http` \| `osascript`. `service`: the program's service category, only for `gh` \| `glab`: `code_hosting`. | +| `Plugin subagent stopped` | SubagentStop, only for arcade-operator | `agent`: `arcade-operator`. `status`: only for `arcade-operator`: the status line of its final report, `completed` \| `needs_auth` \| `needs_confirmation` \| `needs_clarification` \| `failed` \| `unknown`. `subagent_session`: `sha256(agent_id)`, first 16 hex characters. In Copilot CLI the subagent's own events carry this as `session`. | + +Service categories: `email`, `calendar`, `chat`, `issues`, `docs`, `meetings`, `crm`, `code_hosting`, `analytics`, `storage`. + + +`could_use_arcade` is a local keyword guess (in +`hooks/telemetry-classify.mjs`) at whether the prompt is a task Arcade could +do: email, calendar, chat, and the other categories above. + +`failure_kind`, `auth_needed`, and `cli` are values from fixed lists, picked +on your machine. The error text, the tool output, and the command they are +picked from are never sent. During an app-related observation period, a Bash +command sends an event only when one of its +commands starts with a listed program; commands that call a program by its +full path, such as `/opt/homebrew/bin/gh`, are not counted. A command that runs +two listed programs, such as `gh … && curl …`, sends one event for each, so +count CLI use by turn, not by event. Claude Code starts these hooks only for +commands that match (the hook `if` field, Claude Code 2.1.246 and later). + +## Never sent + +- prompt text, or any text you or the model wrote +- tool inputs, tool outputs, or error messages +- commands, their arguments, URLs, and search queries +- file paths, the working directory, or transcript paths +- names of MCP servers other than Arcade's, or their tool names +- names of subagents other than Arcade's +- your email, username, hostname, repository, or Arcade account + +The plugin drops any property not listed on this page before sending. + +PostHog would see the IP address the request comes from, like any web request, +but would not store it: every event sets `$ip` to `0.0.0.0`, and location +lookup is off. + +## Gateway telemetry + +The Arcade MCP gateway (`https://api.arcade.dev/mcp/arcade`) remains the +canonical source for request, authentication, discovery, tool-call, and +completion telemetry. Plugin events are supplemental observations from client +hooks. They do not replace gateway records and cannot prove task success on +their own. + +## Reading the numbers + +Export PostHog rows in the shape `{ event, distinct_id, properties, timestamp? }` +and aggregate them locally: + +```bash +node scripts/telemetry-report.mjs path/to/export.jsonl +``` + +The script prints JSON with counts, denominators, and a `limits` list. Before +validating, it ignores export envelope fields outside +`{ event, distinct_id, properties }` (such as `timestamp` and `uuid`) and +PostHog-added properties whose names start with `$` but are not on the contract +for that event. Every other property must match the contract; leaked hook +fields such as `prompt` or `cwd` make a row invalid. It counts invalid rows and +legacy rows (no `telemetry_version`) separately, and excludes both from grouped +counts. Groups are split by `host`, `plugin_version`, and +`telemetry_version`. Claude Code and Copilot CLI are never combined into one +denominator. If the file is not valid JSON or JSON Lines, the script exits +nonzero with a short message naming the failing line, prints no report, and +does not echo the file's contents. + +These events measure what plugin hooks observed, not whether Arcade was needed +or whether the user's task succeeded. `could_use_arcade` and `service_hints` +come from a local keyword classifier. A flagged prompt is a candidate for +review, not a confirmed opportunity. Prompt events are selected by relevance, +so they cannot measure the share of all prompts needing Arcade; unrelated +prompts are deliberately absent. Use labeled task evaluations to score routing, and compare the +plugin with a baseline or variant before attributing a change to it. +Neither host emits a task ID. + +Keep the observation stages separate: + +| Stage | Evidence | What it establishes | +| --- | --- | --- | +| Tool attempt (Claude Code) | `Plugin tool attempted` on PreToolUse | The model invoked a tool. An attempt alone has no observed outcome. | +| Gateway discovery or selection | `Plugin tool called` or `Plugin tool failed` for `Arcade_ListApps` or `Arcade_SelectTools` | The discovery or selection call completed or failed; a successful selection is not an app action. | +| Authorization check | `System_ManageAuthorization`, reported separately | `auth_needed: true` means its answer said sign-in was needed. A check alone says nothing about app use; `false` does not prove every app is connected. | +| App action | `Arcade_UseTool` or `app_tool`, split by `Plugin tool called` and `Plugin tool failed` | The hook observed a tool completion or failure. Tools with an unrecognized service category are `other` and cannot be assigned to this stage. Neither outcome proves the user's task succeeded. | +| No Arcade call observed | A prompt with no Arcade tool event in the observable group | The hooks saw no call. This is not a routing miss without an independently labeled need and complete tool visibility. | + +Count each unit once at each stage, and show the denominator, host, date range, +plugin version, telemetry version, and observation coverage beside every rate. +`telemetry_version: 2` identifies scoped events; a missing value identifies the +legacy contract. Keep those populations separate, even at the same plugin +version. Keep the number of +observed prompt units and sessions visible even when a chart has no app actions. +Do not extrapolate rates from a test sample or telemetry-enabled sessions to all users. +Do not mix Claude turns with Copilot session counts in one rate. + +Report field meanings (from `scripts/telemetry-report.mjs`): + +- `observed_relevant_turns` / `observed_relevant_sessions`: distinct Claude + `turn` values or Copilot `session` values with a `Plugin prompt submitted` + event, after Copilot operator-linked child sessions are excluded from the + session denominator. +- `tool_only_turns`: Claude turns with tool events but no prompt event in that + turn (reported outside the turn denominator). +- `attempt_observed_outcome_unknown`: Claude turns with a `Plugin tool + attempted` on a `server` category (`arcade` or `other_arcade`) but no `Plugin + tool called` or `Plugin tool failed` on that same category in the same turn. + An outcome on one category never settles an attempt on the other. + `other_arcade` pools every other Arcade connection, and one outcome settles + every attempt on its category in the turn, so the check cannot pair a result + with a particular connection or call. +- `no_call_observed`: relevant turns or sessions with no Arcade MCP tool event + (`Plugin tool attempted`, `Plugin tool called`, or `Plugin tool failed` on + `server: arcade` or `other_arcade`). +- `parent_attribution_unknown_sessions`: Copilot sessions with a prompt event + that are not operator-linked children and have no operator stop on that + session to confirm parent/child linkage. +- Stage counts use the denominators above; they are not paired attempt/result + metrics and do not imply recall, precision, or task success. + +### Claude Code turns + +The denominator is distinct `turn` values with a `Plugin prompt submitted` +event. This is a count of observed relevant turns, not all prompts or tasks. +Exclude tool-only turns: Claude Code filters background task results +from prompt events. Group tool events with the same `turn`, including an +arcade-operator's events, and report sessions and turns separately. A short +follow-up such as “yes, send it” is a separate observed turn while scope is +active, even when its keyword flag is false. Turn counts do not describe whole +tasks. Report tool-only turns separately from this denominator. + +`server: other_arcade` means the hook observed another connection's Arcade +gateway tool. Results recognize exact gateway tool names on arbitrary MCP +server aliases; attempts recognize only the two configured lowercase Arcade +prefixes. UUID or capitalized aliases can therefore have results without an +attempt event. Direct app tools on an unidentified connection may be classified +as `other`. Gateway names are a recognition heuristic, not verified server +provenance. + +Events contain no tool-call ID. Multiple calls in the same turn cannot be +paired individually, even when their tool categories match. Count observed +events or turn-level stages; do not present a per-attempt completion rate. + +A tool invocation is not always followed by a PostToolUse or +PostToolUseFailure event. A `Plugin tool attempted` event can show the +invocation, but only `Plugin tool called` or `Plugin tool failed` records its +outcome. Count an attempt without either outcome as **attempt +observed, outcome unknown**, not app action success. It does not set +`arcade_used_before`. Label flagged turns with no Arcade tool event **no call +observed**, not **missed**. A direct app tool on another gateway may appear +as `server: other`, without an identifiable Arcade call. + +A built-in CLI or web event is a tool observation, not evidence of fallback. +Only call it a possible fallback after linking it to a labeled Arcade-eligible +task and establishing that it served the same request. +`arcade_used_before: false` says no Arcade call has previously succeeded for +that client plugin installation; it does not describe another client on the +same machine or prove that the gateway or a particular app was unconnected. + +### Copilot CLI sessions + +Copilot CLI supplies no `prompt_id` or `turn`. Count distinct sessions with a +scoped `Plugin prompt submitted` event as **observed relevant sessions**. Report +multiple-prompt sessions separately. This is not a count of all sessions, user +tasks, or turns. Session starts are not transmitted and cannot define roots. + +An Arcade operator's stop report carries `subagent_session`, equal to that +operator's own event `session`. Exclude known linked operator prompts from the +parent-session denominator and include their tool events with the parent. +Other subagents do not send stop reports, so their prompt sessions cannot be +reliably distinguished from roots. Report that parent attribution is unknown +rather than labeling every unlinked session as a root. + +Detached event delivery can change arrival order. A missing operator stop or a +rolling-window boundary can also remove a parent link. Do not reconstruct +per-prompt outcomes from timestamps alone or link resumed tasks across sessions. + +Copilot names MCP tools `-` with no plugin prefix, so an MCP +server named `arcade` counts as `server: arcade` even if this plugin did not +install it. Copilot sends no built-in CLI or web events; its `arcade-operator` +can report `status: unknown`. Do not infer a fallback or task result from +either absence. + +### Failure and outcome limits + +`failure_kind` is a local classifier of error text, not a server error code. +An app sign-in failure can be `tool_error` when its wording does not match the +classifier; bad input, rate limits, and upstream errors can also get that +value. + +`auth_needed: true` on an authorization check reports a service still needing +sign-in; operator `status: needs_auth` is the model's report, not a verified +server status. `auth_required`, `session_expired`, `timeout`, and `unreachable` +match known client or MCP SDK error wording; other wording can classify +differently. Copilot does not report interrupted calls, so its `interrupted` +category is empty. + +Operator status describes its report, not the parent task's result. The hooks +do not send final answers or satisfaction signals. Task success, true routing +misses, and improvement from this plugin require labeled evaluations outside +this telemetry. + +## Routing evaluation + +Evaluate classification and client routing on controlled, labeled cases separately +from production usage events. Include implicit app requests, contextual follow-ups, +and local coding tasks that name an app. Fixture accuracy does not establish +real-user routing quality. + +The existing [Tool Recommendation Cost Eval](https://github.com/ArcadeAI/tool-recommendation-cost-eval) +provides an offline cross-client harness and published results. Record the plugin +SHA and client configuration when evaluating the plugin; Tool Recommendation +on/off results do not establish the plugin's effect on routing or cost. Plugin +telemetry and that harness are complementary; neither replaces the other. + +## Before turning it on + +Separate approval is required before setting `TELEMETRY_ENABLED` to `true` and +shipping telemetry hooks. Confirm, outside this repository: + +- user-facing disclosure and privacy policy coverage for plugin usage events +- analytics retention and processor handling for the PostHog project +- verified ingestion per client and `telemetry_version` for the build under review +- opt-out switches verified on each supported client surface +- applicable store or distribution decisions for builds that transmit data + +These are release acceptance requirements; the current code and CI do not +enforce them. A repository install can consume a branch before a GitHub release +is tagged, so a later release PR is not a transmission gate. diff --git a/scripts/generate-manifests.mjs b/scripts/generate-manifests.mjs index bb1a89a..d047de2 100644 --- a/scripts/generate-manifests.mjs +++ b/scripts/generate-manifests.mjs @@ -11,6 +11,7 @@ import { SKILL_RULES, } from "../hooks/routing-guidance.mjs"; import { HOOK_TIMEOUT_SEC, HOOKS, HOSTS } from "../hooks/hook-hosts.mjs"; +import { fillTelemetryTables, TELEMETRY_DOC } from "./telemetry-docs.mjs"; const ROOT = resolve(dirname(fileURLToPath(import.meta.url)), ".."); @@ -26,6 +27,9 @@ export const FILES_WITH_GENERATED_RULES = { "skills/try-arcade/SKILL.md": SKILL_RULES, }; +/** Hand-written files whose event tables are generated from hooks/telemetry-contract.mjs. */ +export const FILES_WITH_GENERATED_TABLES = [TELEMETRY_DOC]; + /** Generated copy → the file it is copied from. */ export const COPIED_FILES = { // Each skill folder has to work on its own. @@ -63,6 +67,7 @@ export const FILE_SOURCES = { "com.github.copilot/agents/arcade-operator.agent.md": [OPERATOR, "hooks/routing-guidance.mjs"], [OPERATOR]: ["hooks/routing-guidance.mjs"], "skills/try-arcade/SKILL.md": ["hooks/routing-guidance.mjs"], + [TELEMETRY_DOC]: ["hooks/telemetry-contract.mjs"], }; /** Formats an array of source paths as a human-readable list. */ @@ -89,6 +94,9 @@ const outOfDateError = (path) => { if (path in FILES_WITH_GENERATED_RULES) { return `${path}: the rules block is out of date. Run npm run generate. If you edited the block by hand, make the change in hooks/routing-guidance.mjs instead.`; } + if (FILES_WITH_GENERATED_TABLES.includes(path)) { + return `${path}: the telemetry tables are out of date. Run npm run generate. If you edited the tables by hand, make the change in hooks/telemetry-contract.mjs instead.`; + } const sources = FILE_SOURCES[path]; const joined = joinSources(sources); if (path in COPIED_FILES) { @@ -268,6 +276,10 @@ const buildFiles = (root) => { files.set(path, fillRulesBlock(readText(root, path), rules, path)); } + for (const path of FILES_WITH_GENERATED_TABLES) { + files.set(path, fillTelemetryTables(readText(root, path))); + } + // Every output must have a source entry so error messages can name it. for (const path of files.keys()) { requireSources(path, FILE_SOURCES); diff --git a/scripts/telemetry-docs.mjs b/scripts/telemetry-docs.mjs new file mode 100644 index 0000000..dd88f28 --- /dev/null +++ b/scripts/telemetry-docs.mjs @@ -0,0 +1,59 @@ +// @ts-check +/** Writes the event tables in docs/telemetry.md from hooks/telemetry-contract.mjs. */ + +import { COMMON_PROPERTIES, EVENTS, SERVICE_CATEGORIES } from "../hooks/telemetry-contract.mjs"; + +export const TELEMETRY_DOC = "docs/telemetry.md"; +export const TELEMETRY_BLOCK_BEGIN = + ""; +export const TELEMETRY_BLOCK_END = ""; + +const code = (/** @type {string} */ text) => `\`${text}\``; + +export const buildTelemetryTables = () => { + const common = [ + "| Property | Value |", + "| --- | --- |", + "| `distinct_id` | the same value as `session` |", + ...Object.entries(COMMON_PROPERTIES).map(([key, { doc }]) => `| ${code(key)} | ${doc} |`), + ]; + const events = [ + "| Event | When | Extra properties |", + "| --- | --- | --- |", + ...Object.entries(EVENTS).map(([name, spec]) => { + const when = spec.when ? `${spec.hook}, ${spec.when}` : spec.hook; + const extras = Object.entries(spec.properties) + .map(([key, { doc }]) => `${code(key)}: ${doc}.`) + .join(" "); + return `| ${code(name)} | ${when} | ${extras} |`; + }), + ]; + return [ + "Every event has these properties:", + "", + ...common, + "", + "Events and their extra properties:", + "", + ...events, + "", + `Service categories: ${SERVICE_CATEGORIES.map(code).join(", ")}.`, + ].join("\n"); +}; + +/** + * Replaces the generated block in docs/telemetry.md. + * @param {string} text + */ +export const fillTelemetryTables = (text) => { + const begin = text.indexOf(TELEMETRY_BLOCK_BEGIN); + const end = text.indexOf(TELEMETRY_BLOCK_END); + if (begin === -1 || end === -1 || end < begin) { + throw new Error(`${TELEMETRY_DOC} is missing the generated telemetry block markers`); + } + return ( + text.slice(0, begin + TELEMETRY_BLOCK_BEGIN.length) + + `\n${buildTelemetryTables()}\n` + + text.slice(end) + ); +}; diff --git a/scripts/telemetry-report.mjs b/scripts/telemetry-report.mjs new file mode 100644 index 0000000..93a676a --- /dev/null +++ b/scripts/telemetry-report.mjs @@ -0,0 +1,449 @@ +// @ts-check +/** Aggregate exported PostHog plugin telemetry rows into count-only reports. */ + +import { readFileSync } from "node:fs"; +import { resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import Ajv2020Module from "ajv/dist/2020.js"; +import { allowedProperties, EVENTS, eventSchema } from "../hooks/telemetry-contract.mjs"; + +const Ajv2020 = /** @type {new (options?: object) => import("ajv").default} */ ( + /** @type {any} */ (Ajv2020Module).default ?? Ajv2020Module +); + +const KNOWN_EVENTS = new Set(Object.keys(EVENTS)); + +const DISCOVERY_TOOLS = new Set(["Arcade_ListApps", "Arcade_SelectTools"]); +const MCP_EVENTS = new Set(["Plugin tool attempted", "Plugin tool called", "Plugin tool failed"]); +const OUTCOME_EVENTS = new Set(["Plugin tool called", "Plugin tool failed"]); +const APP_ACTION_TOOLS = new Set(["Arcade_UseTool", "app_tool"]); + +const validateRow = new Ajv2020({ allErrors: true }).compile(eventSchema()); + +/** @param {unknown} row */ +const isValidRow = (row) => validateRow(row) === true; + +/** + * Drops export envelope fields and PostHog `$` noise; other properties must match the contract. + * @param {unknown} raw + * @returns {{ event: string, distinct_id: string, properties: Record } | null} + */ +export const normalizeExportRow = (raw) => { + if (!raw || typeof raw !== "object") return null; + const record = /** @type {Record} */ (raw); + const { event, distinct_id, properties } = record; + if (typeof event !== "string" || typeof distinct_id !== "string") return null; + if (!properties || typeof properties !== "object" || Array.isArray(properties)) return null; + if (!KNOWN_EVENTS.has(event)) return null; + const props = /** @type {Record} */ (properties); + const allowed = new Set(allowedProperties(event)); + /** @type {Record} */ + const trimmed = { ...props }; + for (const key of Object.keys(trimmed)) { + if (key.startsWith("$") && !allowed.has(key)) delete trimmed[key]; + } + return { event, distinct_id, properties: trimmed }; +}; + +/** @param {unknown} row */ +const isLegacyRow = (row) => { + if (!row || typeof row !== "object") return false; + const record = /** @type {{ event?: string, properties?: Record }} */ (row); + return ( + typeof record.event === "string" && + KNOWN_EVENTS.has(record.event) && + record.properties && + typeof record.properties === "object" && + record.properties.telemetry_version === undefined + ); +}; + +/** + * @param {Record} properties + * @returns {{ host: string, plugin_version: string, telemetry_version: number | "legacy" }} + */ +const groupKey = (properties) => ({ + host: String(properties.host), + plugin_version: String(properties.plugin_version), + telemetry_version: + properties.telemetry_version === undefined ? "legacy" : Number(properties.telemetry_version), +}); + +/** @param {string} server */ +const isArcadeServer = (server) => server === "arcade" || server === "other_arcade"; + +/** + * @param {{ event: string, properties: Record }} row + */ +const isArcadeMcpRow = (row) => + MCP_EVENTS.has(row.event) && isArcadeServer(String(row.properties.server ?? "")); + +/** + * @param {{ event: string, properties: Record }} row + */ +const toolName = (row) => (typeof row.properties.tool === "string" ? row.properties.tool : ""); + +/** + * @param {{ event: string, properties: Record }} row + */ +const isDiscoveryRow = (row) => + OUTCOME_EVENTS.has(row.event) && DISCOVERY_TOOLS.has(toolName(row)); + +/** + * @param {{ event: string, properties: Record }} row + */ +const isAuthRow = (row) => + OUTCOME_EVENTS.has(row.event) && toolName(row) === "System_ManageAuthorization"; + +/** + * @param {{ event: string, properties: Record }} row + */ +const isAppActionCalledRow = (row) => + row.event === "Plugin tool called" && APP_ACTION_TOOLS.has(toolName(row)); + +/** + * @param {{ event: string, properties: Record }} row + */ +const isAppActionFailedRow = (row) => + row.event === "Plugin tool failed" && APP_ACTION_TOOLS.has(toolName(row)); + +/** + * @param {Set} turnIds + * @param {Map }[]>} byTurn + */ +const countTurnStages = (turnIds, byTurn) => { + let gateway = 0; + let authNeededTrue = 0; + let authNeededFalse = 0; + let appCalled = 0; + let appFailed = 0; + let attemptOutcomeUnknown = 0; + let noCallObserved = 0; + + for (const turn of turnIds) { + const rows = byTurn.get(turn) ?? []; + const arcadeRows = rows.filter(isArcadeMcpRow); + if (gatewayRow(rows)) gateway += 1; + if (authRow(rows, true)) authNeededTrue += 1; + if (authRow(rows, false)) authNeededFalse += 1; + if (rows.some(isAppActionCalledRow)) appCalled += 1; + if (rows.some(isAppActionFailedRow)) appFailed += 1; + if (attemptWithoutOutcome(arcadeRows)) attemptOutcomeUnknown += 1; + if (arcadeRows.length === 0) noCallObserved += 1; + } + + return { + gateway_discovery_or_selection: { count: gateway, denominator: "observed_relevant_turns" }, + authorization_check_auth_needed_true: { count: authNeededTrue, denominator: "observed_relevant_turns" }, + authorization_check_auth_needed_false: { count: authNeededFalse, denominator: "observed_relevant_turns" }, + app_action_called: { count: appCalled, denominator: "observed_relevant_turns" }, + app_action_failed: { count: appFailed, denominator: "observed_relevant_turns" }, + attempt_observed_outcome_unknown: { count: attemptOutcomeUnknown, denominator: "observed_relevant_turns" }, + no_call_observed: { count: noCallObserved, denominator: "observed_relevant_turns" }, + }; +}; + +/** @param {{ event: string, properties: Record }[]} rows */ +const gatewayRow = (rows) => rows.some(isDiscoveryRow); + +/** + * @param {{ event: string, properties: Record }[]} rows + * @param {boolean} needed + */ +const authRow = (rows, needed) => + rows.some( + (row) => isAuthRow(row) && row.properties.auth_needed === needed, + ); + +/** + * An outcome only settles attempts on its own `server` category, so an `other_arcade` + * result cannot hide an `arcade` attempt. Within a category the check is turn-level. + * @param {{ event: string, properties: Record }[]} arcadeRows + */ +const attemptWithoutOutcome = (arcadeRows) => + ["arcade", "other_arcade"].some((server) => { + const onServer = arcadeRows.filter((row) => row.properties.server === server); + const hasAttempt = onServer.some((row) => row.event === "Plugin tool attempted"); + const hasOutcome = onServer.some((row) => OUTCOME_EVENTS.has(row.event)); + return hasAttempt && !hasOutcome; + }); + +/** + * @param {{ event: string, properties: Record }[]} rows + * @param {string} eventName + */ +const builtinCounts = (rows, eventName) => { + /** @type {Record>} */ + const out = {}; + for (const row of rows) { + if (row.event !== eventName) continue; + const tool = String(row.properties.tool ?? "unknown"); + if (!out[tool]) out[tool] = {}; + const cli = row.properties.cli; + const key = typeof cli === "string" ? cli : "_"; + out[tool][key] = (out[tool][key] ?? 0) + 1; + } + return out; +}; + +/** + * @param {{ event: string, properties: Record }[]} rows + */ +const failureKindCounts = (rows) => { + /** @type {Record} */ + const out = {}; + for (const row of rows) { + if (row.event !== "Plugin tool failed") continue; + const kind = String(row.properties.failure_kind ?? "unknown"); + out[kind] = (out[kind] ?? 0) + 1; + } + return out; +}; + +/** + * @param {{ event: string, properties: Record }[]} rows + */ +const operatorStatusCounts = (rows) => { + /** @type {Record} */ + const out = {}; + for (const row of rows) { + if (row.event !== "Plugin subagent stopped") continue; + const status = String(row.properties.status ?? "unknown"); + out[status] = (out[status] ?? 0) + 1; + } + return out; +}; + +/** + * @param {{ event: string, properties: Record }[]} rows + */ +const buildClaudeGroup = (rows) => { + const prompts = rows.filter((row) => row.event === "Plugin prompt submitted"); + const relevantTurns = new Set( + prompts.map((row) => String(row.properties.turn ?? "")).filter(Boolean), + ); + + /** @type {Map }[]>} */ + const byTurn = new Map(); + for (const row of rows) { + const turn = row.properties.turn; + if (typeof turn !== "string" || !turn) continue; + const list = byTurn.get(turn) ?? []; + list.push(row); + byTurn.set(turn, list); + } + + const toolOnlyTurns = [...byTurn.keys()].filter( + (turn) => !relevantTurns.has(turn) && (byTurn.get(turn) ?? []).some((row) => row.event !== "Plugin prompt submitted"), + ).length; + + return { + unit: "turn", + denominators: { observed_relevant_turns: relevantTurns.size }, + tool_only_turns: toolOnlyTurns, + stages: countTurnStages(relevantTurns, byTurn), + builtin_tools_called: builtinCounts(rows, "Plugin built-in tool called"), + builtin_tools_failed: builtinCounts(rows, "Plugin built-in tool failed"), + failure_kinds: failureKindCounts(rows), + operator_stop_status: operatorStatusCounts(rows), + }; +}; + +/** + * @param {{ event: string, properties: Record }[]} rows + */ +const buildCopilotGroup = (rows) => { + const prompts = rows.filter((row) => row.event === "Plugin prompt submitted"); + const promptSessions = new Set(prompts.map((row) => String(row.properties.session))); + + /** @type {Map} */ + const childToParent = new Map(); + const linkedChildren = new Set(); + for (const row of rows) { + if (row.event !== "Plugin subagent stopped") continue; + const parent = String(row.properties.session); + const child = row.properties.subagent_session; + if (typeof child === "string" && child) { + childToParent.set(child, parent); + linkedChildren.add(child); + } + } + + const denominatorSessions = new Set( + [...promptSessions].filter((session) => !linkedChildren.has(session)), + ); + + const multiPromptSessions = [...denominatorSessions].filter((session) => { + const count = prompts.filter((row) => String(row.properties.session) === session).length; + return count > 1; + }).length; + + const parentSessionsWithStop = new Set( + rows + .filter((row) => row.event === "Plugin subagent stopped" && row.properties.subagent_session) + .map((row) => String(row.properties.session)), + ); + const parentAttributionUnknown = [...denominatorSessions].filter( + (session) => !parentSessionsWithStop.has(session), + ).length; + + /** @type {Map }[]>} */ + const bySession = new Map(); + /** @param {{ event: string, properties: Record }} row */ + const attributeSession = (row) => { + const session = String(row.properties.session); + if (linkedChildren.has(session)) { + const parent = childToParent.get(session); + return parent ?? session; + } + return session; + }; + + for (const row of rows) { + const session = attributeSession(row); + const list = bySession.get(session) ?? []; + list.push(row); + bySession.set(session, list); + } + + let gateway = 0; + let authNeededTrue = 0; + let authNeededFalse = 0; + let appCalled = 0; + let appFailed = 0; + let noCallObserved = 0; + + for (const session of denominatorSessions) { + const sessionRows = bySession.get(session) ?? []; + const arcadeRows = sessionRows.filter(isArcadeMcpRow); + if (gatewayRow(sessionRows)) gateway += 1; + if (authRow(sessionRows, true)) authNeededTrue += 1; + if (authRow(sessionRows, false)) authNeededFalse += 1; + if (sessionRows.some(isAppActionCalledRow)) appCalled += 1; + if (sessionRows.some(isAppActionFailedRow)) appFailed += 1; + if (arcadeRows.length === 0) noCallObserved += 1; + } + + return { + unit: "session", + denominators: { observed_relevant_sessions: denominatorSessions.size }, + operator_linked_sessions_excluded: linkedChildren.size, + multi_prompt_sessions: multiPromptSessions, + parent_attribution_unknown_sessions: parentAttributionUnknown, + stages: { + gateway_discovery_or_selection: { count: gateway, denominator: "observed_relevant_sessions" }, + authorization_check_auth_needed_true: { count: authNeededTrue, denominator: "observed_relevant_sessions" }, + authorization_check_auth_needed_false: { count: authNeededFalse, denominator: "observed_relevant_sessions" }, + app_action_called: { count: appCalled, denominator: "observed_relevant_sessions" }, + app_action_failed: { count: appFailed, denominator: "observed_relevant_sessions" }, + no_call_observed: { count: noCallObserved, denominator: "observed_relevant_sessions" }, + }, + failure_kinds: failureKindCounts(rows), + operator_stop_status: operatorStatusCounts(rows), + }; +}; + +const REPORT_LIMITS = [ + "Counts describe plugin hook observations only, not task outcomes or whether Arcade was needed.", + "Prompt events are limited to locally classified app-related work; unrelated prompts are absent.", + "Claude Code turn counts and Copilot CLI session counts must not be combined into one rate.", + "Copilot CLI does not emit tool-attempt events; attempt stages apply only to Claude Code.", + "Multiple tool calls in one turn or session cannot be paired without a tool-call ID.", + "Attempt outcomes are checked per server category; other_arcade pools every other Arcade connection, so an outcome there cannot be tied to the connection or call that was attempted.", + "The Arcade MCP gateway remains the canonical source for request, auth, discovery, and tool-call telemetry.", +]; + +/** + * @param {unknown[]} events + */ +export const buildReport = (events) => { + let invalid = 0; + let legacy = 0; + /** @type {Map }[]>} */ + const groups = new Map(); + + for (const raw of events) { + const row = normalizeExportRow(raw); + if (row === null) { + invalid += 1; + continue; + } + if (isLegacyRow(row)) { + legacy += 1; + continue; + } + if (!isValidRow(row)) { + invalid += 1; + continue; + } + const key = JSON.stringify(groupKey(row.properties)); + const list = groups.get(key) ?? []; + list.push(row); + groups.set(key, list); + } + + /** @type {object[]} */ + const reportGroups = []; + for (const [key, rows] of [...groups.entries()].sort(([a], [b]) => a.localeCompare(b))) { + const meta = groupKey(rows[0].properties); + const body = + meta.host === "claude-code" + ? buildClaudeGroup(rows) + : meta.host === "copilot-cli" + ? buildCopilotGroup(rows) + : { unit: "unknown", denominators: {}, stages: {} }; + reportGroups.push({ ...meta, ...body }); + } + + return { + excluded: { invalid, legacy }, + groups: reportGroups, + limits: REPORT_LIMITS, + }; +}; + +/** Thrown for unparseable input; the message never contains the input text. */ +export class ExportParseError extends Error {} + +/** @param {string} text @param {string} where */ +const parseJson = (text, where) => { + try { + return JSON.parse(text); + } catch { + throw new ExportParseError(`${where} is not valid JSON`); + } +}; + +/** + * @param {string} text + * @returns {unknown[]} + */ +export const parseExportedEvents = (text) => { + const trimmed = text.trim(); + if (!trimmed) return []; + if (trimmed.startsWith("[")) return parseJson(trimmed, "the array"); + return trimmed + .split("\n") + .map((line, index) => ({ line: line.trim(), number: index + 1 })) + .filter(({ line }) => line) + .map(({ line, number }) => parseJson(line, `line ${number}`)); +}; + +if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + const file = process.argv[2]; + if (!file) { + console.error("usage: node scripts/telemetry-report.mjs "); + process.exit(1); + } + /** @type {unknown[]} */ + let events; + try { + events = parseExportedEvents(readFileSync(file, "utf8")); + } catch (error) { + // Error text from JSON.parse or the runtime can quote the input, so print only our own message. + const reason = error instanceof ExportParseError ? error.message : "the file could not be read"; + console.error(`telemetry-report: ${reason}; no report was produced`); + process.exit(1); + } + console.log(JSON.stringify(buildReport(events), null, 2)); +} diff --git a/test/fixtures/telemetry-report/events.jsonl b/test/fixtures/telemetry-report/events.jsonl new file mode 100644 index 0000000..b4b983e --- /dev/null +++ b/test/fixtures/telemetry-report/events.jsonl @@ -0,0 +1,21 @@ +{"event":"Plugin prompt submitted","distinct_id":"aaaaaaaaaaaaaaaa","properties":{"session":"aaaaaaaaaaaaaaaa","turn":"1111111111111111","host":"claude-code","plugin_version":"0.2.0","telemetry_version":2,"os":"linux","arcade_used_before":false,"$process_person_profile":false,"$geoip_disable":true,"$ip":"0.0.0.0","could_use_arcade":true,"service_hints":["email"],"reminder_sent":true}} +{"event":"Plugin tool called","distinct_id":"aaaaaaaaaaaaaaaa","properties":{"session":"aaaaaaaaaaaaaaaa","turn":"1111111111111111","host":"claude-code","plugin_version":"0.2.0","telemetry_version":2,"os":"linux","arcade_used_before":false,"$process_person_profile":false,"$geoip_disable":true,"$ip":"0.0.0.0","server":"arcade","tool":"Arcade_SelectTools"}} +{"event":"Plugin tool called","distinct_id":"aaaaaaaaaaaaaaaa","properties":{"session":"aaaaaaaaaaaaaaaa","turn":"1111111111111111","host":"claude-code","plugin_version":"0.2.0","telemetry_version":2,"os":"linux","arcade_used_before":false,"$process_person_profile":false,"$geoip_disable":true,"$ip":"0.0.0.0","server":"arcade","tool":"System_ManageAuthorization","auth_needed":false}} +{"event":"Plugin tool called","distinct_id":"aaaaaaaaaaaaaaaa","properties":{"session":"aaaaaaaaaaaaaaaa","turn":"1111111111111111","host":"claude-code","plugin_version":"0.2.0","telemetry_version":2,"os":"linux","arcade_used_before":true,"$process_person_profile":false,"$geoip_disable":true,"$ip":"0.0.0.0","server":"arcade","tool":"app_tool","service":"email"}} +{"event":"Plugin built-in tool called","distinct_id":"aaaaaaaaaaaaaaaa","properties":{"session":"aaaaaaaaaaaaaaaa","turn":"1111111111111111","host":"claude-code","plugin_version":"0.2.0","telemetry_version":2,"os":"linux","arcade_used_before":true,"$process_person_profile":false,"$geoip_disable":true,"$ip":"0.0.0.0","tool":"Bash","cli":"gh","service":"code_hosting"}} +{"event":"Plugin prompt submitted","distinct_id":"aaaaaaaaaaaaaaaa","properties":{"session":"aaaaaaaaaaaaaaaa","turn":"2222222222222222","host":"claude-code","plugin_version":"0.2.0","telemetry_version":2,"os":"linux","arcade_used_before":true,"$process_person_profile":false,"$geoip_disable":true,"$ip":"0.0.0.0","could_use_arcade":true,"service_hints":[],"reminder_sent":true}} +{"event":"Plugin tool attempted","distinct_id":"aaaaaaaaaaaaaaaa","properties":{"session":"aaaaaaaaaaaaaaaa","turn":"2222222222222222","host":"claude-code","plugin_version":"0.2.0","telemetry_version":2,"os":"linux","arcade_used_before":true,"$process_person_profile":false,"$geoip_disable":true,"$ip":"0.0.0.0","server":"arcade","tool":"Arcade_UseTool"}} +{"event":"Plugin tool failed","distinct_id":"aaaaaaaaaaaaaaaa","properties":{"session":"aaaaaaaaaaaaaaaa","turn":"1111111111111111","host":"claude-code","plugin_version":"0.2.0","telemetry_version":2,"os":"linux","arcade_used_before":true,"$process_person_profile":false,"$geoip_disable":true,"$ip":"0.0.0.0","server":"arcade","tool":"app_tool","service":"chat","failure_kind":"timeout"}} +{"event":"Plugin prompt submitted","distinct_id":"aaaaaaaaaaaaaaaa","properties":{"session":"aaaaaaaaaaaaaaaa","turn":"3333333333333333","host":"claude-code","plugin_version":"0.2.0","telemetry_version":2,"os":"linux","arcade_used_before":true,"$process_person_profile":false,"$geoip_disable":true,"$ip":"0.0.0.0","could_use_arcade":false,"service_hints":[],"reminder_sent":true}} +{"event":"Plugin tool called","distinct_id":"aaaaaaaaaaaaaaaa","properties":{"session":"aaaaaaaaaaaaaaaa","turn":"4444444444444444","host":"claude-code","plugin_version":"0.2.0","telemetry_version":2,"os":"linux","arcade_used_before":true,"$process_person_profile":false,"$geoip_disable":true,"$ip":"0.0.0.0","server":"arcade","tool":"Arcade_ListApps"}} +{"event":"Plugin subagent stopped","distinct_id":"aaaaaaaaaaaaaaaa","properties":{"session":"aaaaaaaaaaaaaaaa","host":"claude-code","plugin_version":"0.2.0","telemetry_version":2,"os":"linux","arcade_used_before":true,"$process_person_profile":false,"$geoip_disable":true,"$ip":"0.0.0.0","agent":"arcade-operator","status":"completed"}} +{"event":"Plugin prompt submitted","distinct_id":"bbbbbbbbbbbbbbbb","properties":{"session":"bbbbbbbbbbbbbbbb","host":"copilot-cli","plugin_version":"0.2.0","telemetry_version":2,"os":"linux","arcade_used_before":false,"$process_person_profile":false,"$geoip_disable":true,"$ip":"0.0.0.0","could_use_arcade":true,"service_hints":["issues"],"reminder_sent":false}} +{"event":"Plugin tool called","distinct_id":"bbbbbbbbbbbbbbbb","properties":{"session":"bbbbbbbbbbbbbbbb","host":"copilot-cli","plugin_version":"0.2.0","telemetry_version":2,"os":"linux","arcade_used_before":false,"$process_person_profile":false,"$geoip_disable":true,"$ip":"0.0.0.0","server":"arcade","tool":"Arcade_UseTool","service":"issues"}} +{"event":"Plugin subagent stopped","distinct_id":"bbbbbbbbbbbbbbbb","properties":{"session":"bbbbbbbbbbbbbbbb","host":"copilot-cli","plugin_version":"0.2.0","telemetry_version":2,"os":"linux","arcade_used_before":false,"$process_person_profile":false,"$geoip_disable":true,"$ip":"0.0.0.0","agent":"arcade-operator","status":"needs_auth","subagent_session":"cccccccccccccccc"}} +{"event":"Plugin prompt submitted","distinct_id":"cccccccccccccccc","properties":{"session":"cccccccccccccccc","host":"copilot-cli","plugin_version":"0.2.0","telemetry_version":2,"os":"linux","arcade_used_before":false,"$process_person_profile":false,"$geoip_disable":true,"$ip":"0.0.0.0","could_use_arcade":true,"service_hints":[],"reminder_sent":false}} +{"event":"Plugin tool called","distinct_id":"cccccccccccccccc","properties":{"session":"cccccccccccccccc","host":"copilot-cli","plugin_version":"0.2.0","telemetry_version":2,"os":"linux","arcade_used_before":false,"$process_person_profile":false,"$geoip_disable":true,"$ip":"0.0.0.0","server":"arcade","tool":"Arcade_SelectTools"}} +{"event":"Plugin prompt submitted","distinct_id":"dddddddddddddddd","properties":{"session":"dddddddddddddddd","host":"copilot-cli","plugin_version":"0.2.0","telemetry_version":2,"os":"linux","arcade_used_before":false,"$process_person_profile":false,"$geoip_disable":true,"$ip":"0.0.0.0","could_use_arcade":true,"service_hints":["docs"],"reminder_sent":false}} +{"event":"Plugin prompt submitted","distinct_id":"dddddddddddddddd","properties":{"session":"dddddddddddddddd","host":"copilot-cli","plugin_version":"0.2.0","telemetry_version":2,"os":"linux","arcade_used_before":false,"$process_person_profile":false,"$geoip_disable":true,"$ip":"0.0.0.0","could_use_arcade":false,"service_hints":[],"reminder_sent":false}} +{"event":"Plugin prompt submitted","distinct_id":"eeeeeeeeeeeeeeee","properties":{"session":"eeeeeeeeeeeeeeee","host":"copilot-cli","plugin_version":"0.2.0","telemetry_version":2,"os":"linux","arcade_used_before":false,"$process_person_profile":false,"$geoip_disable":true,"$ip":"0.0.0.0","could_use_arcade":true,"service_hints":["calendar"],"reminder_sent":false}} +{"event":"Plugin prompt submitted","distinct_id":"aaaaaaaaaaaaaaaa","properties":{"session":"aaaaaaaaaaaaaaaa","host":"claude-code","plugin_version":"0.2.0","os":"linux","arcade_used_before":false,"$process_person_profile":false,"$geoip_disable":true,"$ip":"0.0.0.0","could_use_arcade":true,"service_hints":[],"reminder_sent":true}} +{"event":"not-a-contract-event","distinct_id":"ffffffffffffffff","properties":{"session":"ffffffffffffffff","host":"claude-code","plugin_version":"0.2.0","telemetry_version":2,"os":"linux","arcade_used_before":false,"$process_person_profile":false,"$geoip_disable":true,"$ip":"0.0.0.0"}} diff --git a/test/fixtures/telemetry-report/expected-report.json b/test/fixtures/telemetry-report/expected-report.json new file mode 100644 index 0000000..05662f6 --- /dev/null +++ b/test/fixtures/telemetry-report/expected-report.json @@ -0,0 +1,111 @@ +{ + "excluded": { + "invalid": 1, + "legacy": 1 + }, + "groups": [ + { + "host": "claude-code", + "plugin_version": "0.2.0", + "telemetry_version": 2, + "unit": "turn", + "denominators": { + "observed_relevant_turns": 3 + }, + "tool_only_turns": 1, + "stages": { + "gateway_discovery_or_selection": { + "count": 1, + "denominator": "observed_relevant_turns" + }, + "authorization_check_auth_needed_true": { + "count": 0, + "denominator": "observed_relevant_turns" + }, + "authorization_check_auth_needed_false": { + "count": 1, + "denominator": "observed_relevant_turns" + }, + "app_action_called": { + "count": 1, + "denominator": "observed_relevant_turns" + }, + "app_action_failed": { + "count": 1, + "denominator": "observed_relevant_turns" + }, + "attempt_observed_outcome_unknown": { + "count": 1, + "denominator": "observed_relevant_turns" + }, + "no_call_observed": { + "count": 1, + "denominator": "observed_relevant_turns" + } + }, + "builtin_tools_called": { + "Bash": { + "gh": 1 + } + }, + "builtin_tools_failed": {}, + "failure_kinds": { + "timeout": 1 + }, + "operator_stop_status": { + "completed": 1 + } + }, + { + "host": "copilot-cli", + "plugin_version": "0.2.0", + "telemetry_version": 2, + "unit": "session", + "denominators": { + "observed_relevant_sessions": 3 + }, + "operator_linked_sessions_excluded": 1, + "multi_prompt_sessions": 1, + "parent_attribution_unknown_sessions": 2, + "stages": { + "gateway_discovery_or_selection": { + "count": 1, + "denominator": "observed_relevant_sessions" + }, + "authorization_check_auth_needed_true": { + "count": 0, + "denominator": "observed_relevant_sessions" + }, + "authorization_check_auth_needed_false": { + "count": 0, + "denominator": "observed_relevant_sessions" + }, + "app_action_called": { + "count": 1, + "denominator": "observed_relevant_sessions" + }, + "app_action_failed": { + "count": 0, + "denominator": "observed_relevant_sessions" + }, + "no_call_observed": { + "count": 2, + "denominator": "observed_relevant_sessions" + } + }, + "failure_kinds": {}, + "operator_stop_status": { + "needs_auth": 1 + } + } + ], + "limits": [ + "Counts describe plugin hook observations only, not task outcomes or whether Arcade was needed.", + "Prompt events are limited to locally classified app-related work; unrelated prompts are absent.", + "Claude Code turn counts and Copilot CLI session counts must not be combined into one rate.", + "Copilot CLI does not emit tool-attempt events; attempt stages apply only to Claude Code.", + "Multiple tool calls in one turn or session cannot be paired without a tool-call ID.", + "Attempt outcomes are checked per server category; other_arcade pools every other Arcade connection, so an outcome there cannot be tied to the connection or call that was attempted.", + "The Arcade MCP gateway remains the canonical source for request, auth, discovery, and tool-call telemetry." + ] +} diff --git a/test/generate-manifests.test.mjs b/test/generate-manifests.test.mjs index dc237f1..85140f6 100644 --- a/test/generate-manifests.test.mjs +++ b/test/generate-manifests.test.mjs @@ -5,6 +5,7 @@ import { test } from "node:test"; import { FILE_SOURCES, FILES_WITH_GENERATED_RULES, + FILES_WITH_GENERATED_TABLES, generateManifests, requireSources, } from "../scripts/generate-manifests.mjs"; @@ -24,6 +25,7 @@ const EXPECTED_SOURCES = { "com.github.copilot/agents/arcade-operator.agent.md": ["agents/arcade-operator.agent.md", "hooks/routing-guidance.mjs"], "agents/arcade-operator.agent.md": ["hooks/routing-guidance.mjs"], "skills/try-arcade/SKILL.md": ["hooks/routing-guidance.mjs"], + "docs/telemetry.md": ["hooks/telemetry-contract.mjs"], }; test("FILE_SOURCES lists the expected sources for every generated path", () => { @@ -48,6 +50,9 @@ test("check mode fails with a source-naming error when a generated file is hand- if (path in FILES_WITH_GENERATED_RULES) { // Edit inside the block so the rules-block path is specifically tested. writeFileSync(fullPath, readFileSync(fullPath, "utf8").replace("Gateway: Arcade is connected", "Gateway: Any server is connected")); + } else if (FILES_WITH_GENERATED_TABLES.includes(path)) { + // Only the tables are generated, so the edit has to be inside them. + writeFileSync(fullPath, readFileSync(fullPath, "utf8").replace("Service categories:", "Service kinds:")); } else { writeFileSync(fullPath, `${readFileSync(fullPath, "utf8")} `); } diff --git a/test/helpers.mjs b/test/helpers.mjs index 707de63..72dafeb 100644 --- a/test/helpers.mjs +++ b/test/helpers.mjs @@ -3,7 +3,7 @@ import { cpSync, mkdtempSync, readFileSync } from "node:fs"; import { tmpdir } from "node:os"; import path from "node:path"; import { fileURLToPath } from "node:url"; -import { COPIED_FILES, FILES_WITH_GENERATED_RULES } from "../scripts/generate-manifests.mjs"; +import { COPIED_FILES, FILES_WITH_GENERATED_RULES, FILES_WITH_GENERATED_TABLES } from "../scripts/generate-manifests.mjs"; export const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); @@ -52,7 +52,7 @@ export const skipOnWindows = { skip: process.platform === "win32" && "the comman /** A temp copy of the generator's source files, for tests that write. */ export const makeFixture = () => { const root = mkdtempSync(path.join(tmpdir(), "arcade-plugin-")); - const sources = ["VERSION", "plugin.json", "mcp.json", ...Object.keys(FILES_WITH_GENERATED_RULES), ...Object.values(COPIED_FILES).flat()]; + const sources = ["VERSION", "plugin.json", "mcp.json", ...Object.keys(FILES_WITH_GENERATED_RULES), ...FILES_WITH_GENERATED_TABLES, ...Object.values(COPIED_FILES).flat()]; for (const file of sources) { cpSync(path.join(ROOT, file), path.join(root, file)); } diff --git a/test/telemetry-docs.test.mjs b/test/telemetry-docs.test.mjs new file mode 100644 index 0000000..e955f15 --- /dev/null +++ b/test/telemetry-docs.test.mjs @@ -0,0 +1,29 @@ +// @ts-check + +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import path from "node:path"; +import { test } from "node:test"; +import { + buildTelemetryTables, + fillTelemetryTables, + TELEMETRY_BLOCK_BEGIN, + TELEMETRY_BLOCK_END, +} from "../scripts/telemetry-docs.mjs"; +import { ROOT } from "./helpers.mjs"; + +test("buildTelemetryTables lists every contract event and common property", () => { + const tables = buildTelemetryTables(); + assert.match(tables, /Plugin prompt submitted/); + assert.match(tables, /telemetry_version/); + assert.match(tables, /Service categories:/); +}); + +test("fillTelemetryTables replaces only the generated block", () => { + const docPath = path.join(ROOT, "docs/telemetry.md"); + const source = readFileSync(docPath, "utf8"); + const filled = fillTelemetryTables(source); + assert.ok(filled.includes(TELEMETRY_BLOCK_BEGIN)); + assert.ok(filled.includes(TELEMETRY_BLOCK_END)); + assert.ok(filled.includes(buildTelemetryTables().split("\n")[0])); +}); diff --git a/test/telemetry-report.test.mjs b/test/telemetry-report.test.mjs new file mode 100644 index 0000000..2e9ee44 --- /dev/null +++ b/test/telemetry-report.test.mjs @@ -0,0 +1,369 @@ +// @ts-check + +import assert from "node:assert/strict"; +import { existsSync, readFileSync, writeFileSync } from "node:fs"; +import { spawnSync } from "node:child_process"; +import path from "node:path"; +import { test } from "node:test"; +import { TELEMETRY_HOSTS } from "../hooks/telemetry-contract.mjs"; +import { loadTelemetryAdapter } from "../hooks/telemetry-adapter.mjs"; +import { buildReport, normalizeExportRow, parseExportedEvents } from "../scripts/telemetry-report.mjs"; +import { assertMatchesContract, captureTelemetry, hookInput, tempDataDir } from "./telemetry-helpers.mjs"; +import { ROOT } from "./helpers.mjs"; + +const FIXTURE_DIR = path.join(ROOT, "test/fixtures/telemetry-report"); +const FORBIDDEN_KEY = /recall|precision|success.?rate|task.?success|routing.?miss|matched.?attempt/i; + +const loadFixture = () => { + const events = parseExportedEvents(readFileSync(path.join(FIXTURE_DIR, "events.jsonl"), "utf8")); + for (const event of events) { + if (event.event?.startsWith("Plugin") && event.properties?.telemetry_version === 2) { + assertMatchesContract(event); + } + } + const expected = JSON.parse(readFileSync(path.join(FIXTURE_DIR, "expected-report.json"), "utf8")); + return { events, expected }; +}; + +/** @param {unknown} value @param {string[]} keys */ +const collectKeys = (value, keys = []) => { + if (value && typeof value === "object") { + if (Array.isArray(value)) { + for (const item of value) collectKeys(item, keys); + } else { + for (const [key, child] of Object.entries(value)) { + keys.push(key); + collectKeys(child, keys); + } + } + } + return keys; +}; + +test("buildReport matches the telemetry-report fixture", () => { + const { events, expected } = loadFixture(); + assert.deepEqual(buildReport(events), expected); +}); + +test("buildReport strips PostHog export fields before validating", () => { + const { events, expected } = loadFixture(); + const noisy = events.map((row) => { + if (!row || typeof row !== "object" || !row.properties) return row; + return { + ...row, + timestamp: "2026-01-01T00:00:00.000Z", + uuid: "550e8400-e29b-41d4-a716-446655440000", + properties: { + ...row.properties, + $lib: "posthog-node", + $lib_version: "5.0.0", + }, + }; + }); + for (const row of noisy) { + if (row?.event?.startsWith("Plugin")) { + const trimmed = normalizeExportRow(row); + assert.ok(trimmed); + assert.equal(trimmed.properties.$lib, undefined); + } + } + assert.deepEqual(buildReport(noisy), expected); +}); + +test("buildReport rejects leaked hook fields in export properties", () => { + const { events } = loadFixture(); + const sample = events.find( + (row) => row.event === "Plugin prompt submitted" && row.properties?.telemetry_version === 2, + ); + assert.ok(sample); + const withPrompt = { ...sample, properties: { ...sample.properties, prompt: "secret text" } }; + const withCwd = { ...sample, properties: { ...sample.properties, cwd: "/Users/secret" } }; + assert.equal(buildReport([withPrompt]).excluded.invalid, 1); + assert.equal(buildReport([withCwd]).excluded.invalid, 1); +}); + +test("CLI prints the same JSON as buildReport", () => { + const { events, expected } = loadFixture(); + const file = path.join(FIXTURE_DIR, "events.jsonl"); + const result = spawnSync(process.execPath, [path.join(ROOT, "scripts/telemetry-report.mjs"), file], { + encoding: "utf8", + }); + assert.equal(result.status, 0, result.stderr); + assert.deepEqual(JSON.parse(result.stdout), expected); + assert.deepEqual(buildReport(events).excluded, { invalid: 1, legacy: 1 }); +}); + +const claudeRow = (event, extra) => ({ + event, + distinct_id: "aaaaaaaaaaaaaaaa", + properties: { + session: "aaaaaaaaaaaaaaaa", + turn: "1111111111111111", + host: "claude-code", + plugin_version: "0.2.0", + telemetry_version: 2, + os: "linux", + arcade_used_before: false, + $process_person_profile: false, + $geoip_disable: true, + $ip: "0.0.0.0", + ...extra, + }, +}); + +const unknownOutcomeCount = (rows) => { + for (const row of rows) assertMatchesContract(row); + const report = buildReport(rows); + assert.deepEqual(report.excluded, { invalid: 0, legacy: 0 }); + return report.groups[0].stages.attempt_observed_outcome_unknown.count; +}; + +test("an outcome on one Arcade server category does not settle an attempt on the other", () => { + const prompt = claudeRow("Plugin prompt submitted", { could_use_arcade: true, service_hints: ["email"], reminder_sent: false }); + const attempt = (server) => claudeRow("Plugin tool attempted", { server, tool: "Arcade_UseTool" }); + const called = (server) => claudeRow("Plugin tool called", { server, tool: "Arcade_SelectTools" }); + const failed = (server) => claudeRow("Plugin tool failed", { server, tool: "Arcade_UseTool", failure_kind: "timeout" }); + + assert.equal(unknownOutcomeCount([prompt, attempt("arcade"), called("other_arcade")]), 1); + assert.equal(unknownOutcomeCount([prompt, attempt("other_arcade"), failed("arcade")]), 1); + assert.equal(unknownOutcomeCount([prompt, attempt("arcade"), called("arcade")]), 0); + assert.equal(unknownOutcomeCount([prompt, attempt("other_arcade"), failed("other_arcade")]), 0); + assert.equal(unknownOutcomeCount([prompt, attempt("arcade"), attempt("other_arcade"), called("arcade")]), 1); + // Turn-level only: one outcome on a category settles every attempt on that category. + assert.equal(unknownOutcomeCount([prompt, attempt("arcade"), attempt("arcade"), called("arcade")]), 0); +}); + +test("report limits state that other_arcade pools connections and calls are not paired", () => { + const limits = buildReport([]).limits.join("\n"); + assert.match(limits, /other_arcade.*pools/i); + assert.match(limits, /cannot be paired/i); +}); + +test("CLI rejects malformed JSON without echoing the input", (t) => { + const marker = "SYNTHETIC-PRIVATE-MARKER-7f3a"; + const dir = tempDataDir(); + const valid = readFileSync(path.join(FIXTURE_DIR, "events.jsonl"), "utf8").split("\n")[0]; + const inputs = { + "lines.jsonl": `${valid}\n{"event":"Plugin prompt submitted","properties":{"prompt":"${marker}"\n`, + "array.json": `[{"event":"${marker}",}]`, + "bare.jsonl": `${marker} not json`, + }; + for (const [name, contents] of Object.entries(inputs)) { + const file = path.join(dir, name); + writeFileSync(file, contents); + const result = spawnSync(process.execPath, [path.join(ROOT, "scripts/telemetry-report.mjs"), file], { + encoding: "utf8", + }); + t.diagnostic(`${name}: ${result.stderr.trim()}`); + assert.notEqual(result.status, 0, name); + assert.equal(result.stdout, "", name); + assert.match(result.stderr, /^telemetry-report: /, name); + for (const stream of [result.stdout, result.stderr]) { + assert.equal(stream.includes(marker), false, `${name}: ${stream}`); + assert.equal(stream.includes("Plugin prompt submitted"), false, `${name}: ${stream}`); + } + } +}); + +test("report keys avoid forbidden metric names and hosts stay separate", () => { + const { events } = loadFixture(); + const report = buildReport(events); + for (const key of collectKeys(report)) { + assert.doesNotMatch(key, FORBIDDEN_KEY, key); + } + const hosts = report.groups.map((group) => group.host); + assert.deepEqual(new Set(hosts), new Set(hosts)); + for (const group of report.groups) { + if (group.host === "copilot-cli") { + assert.equal(group.stages.attempt_observed_outcome_unknown, undefined); + } + if (group.host === "claude-code") { + assert.ok(group.stages.attempt_observed_outcome_unknown); + } + } +}); + +const authStatusResponse = (statuses) => [ + { + type: "text", + text: JSON.stringify({ + message: statuses.includes("authorization_required") ? "Not yet authorized." : "All authorized.", + providers: statuses.map((status, index) => ({ provider: `provider${index}`, status })), + }), + }, +]; + +const claudeArcadePrefix = () => { + const plugin = JSON.parse(readFileSync(path.join(ROOT, "plugin.json"), "utf8")); + const [server] = Object.keys(JSON.parse(readFileSync(path.join(ROOT, "mcp.json"), "utf8")).mcpServers); + return `mcp__plugin_${plugin.name}_${server}__`; +}; + +const COPILOT_SESSION = "dd3beb80-4471-4513-99a4-a57d3d7c08df"; +const COPILOT_SUBAGENT = "57946be7-1a73-40ca-a042-abb0f68d9445"; +const COPILOT_OPERATOR = "arcade:arcade-operator"; + +/** @param {import("../hooks/telemetry-adapter.mjs").TelemetryAdapter} adapter */ +const runClaudeCrossCheck = async (adapter) => { + const dataDir = tempDataDir(); + const prefix = claudeArcadePrefix(); + const session = "cross-check-claude-session"; + /** @type {object[]} */ + const events = []; + const cap = async (fields, argv = []) => { + const { sent } = await captureTelemetry({ + adapter, + dataDir, + input: hookInput({ session_id: session, ...fields }), + argv, + }); + events.push(...sent); + }; + const p1 = "turn-one-11111111"; + const p2 = "turn-two-22222222"; + const p3 = "turn-three-33333333"; + const p4 = "turn-four-44444444"; + await cap({ hook_event_name: "UserPromptSubmit", prompt: "What is on my calendar tomorrow?", prompt_id: p1 }); + await cap({ hook_event_name: "PostToolUse", tool_name: `${prefix}Arcade_SelectTools`, prompt_id: p1 }); + await cap({ + hook_event_name: "PostToolUse", + tool_name: `${prefix}System_ManageAuthorization`, + tool_response: authStatusResponse(["authorized"]), + prompt_id: p1, + }); + await cap({ + hook_event_name: "PostToolUse", + tool_name: `${prefix}Arcade_UseTool`, + tool_input: { tool_name: "Gmail.ListEmails" }, + prompt_id: p1, + }); + await cap({ + hook_event_name: "PostToolUseFailure", + tool_name: `${prefix}Slack_SendMessage`, + error: "rate limited", + prompt_id: p1, + }); + await cap({ hook_event_name: "UserPromptSubmit", prompt: "Fix the parser in src/main.ts", prompt_id: p2 }); + await cap({ + hook_event_name: "PostToolUse", + tool_name: "mcp__granola__Granola_ListMeetings", + prompt_id: p2, + }); + await cap({ hook_event_name: "UserPromptSubmit", prompt: "What is on my calendar tomorrow?", prompt_id: p3 }); + await cap({ + hook_event_name: "PreToolUse", + tool_name: `${prefix}Arcade_UseTool`, + tool_input: { tool_name: "Gmail.ListEmails" }, + prompt_id: p3, + }); + await cap({ hook_event_name: "UserPromptSubmit", prompt: "Summarize unread email from this week", prompt_id: p4 }); + return events; +}; + +/** @param {import("../hooks/telemetry-adapter.mjs").TelemetryAdapter} adapter */ +const runCopilotCrossCheck = async (adapter) => { + const dataDir = tempDataDir(); + /** @type {object[]} */ + const events = []; + const cap = async (fields) => { + const { sent } = await captureTelemetry({ adapter, dataDir, input: fields }); + events.push(...sent); + }; + const parent = (fields) => ({ + session_id: COPILOT_SESSION, + timestamp: "2026-09-24T21:27:02.743Z", + cwd: "/Users/someone/private-repo", + ...fields, + }); + const child = (fields) => parent({ session_id: COPILOT_SUBAGENT, ...fields }); + await cap(parent({ hook_event_name: "UserPromptSubmit", prompt: "What is on my calendar tomorrow?" })); + await cap(parent({ hook_event_name: "PostToolUse", tool_name: "arcade-Arcade_SelectTools", tool_input: {}, tool_result: { result_type: "success", text_result_for_llm: "[]" } })); + await cap(parent({ + hook_event_name: "PostToolUse", + tool_name: "arcade-System_ManageAuthorization", + tool_input: {}, + tool_result: { result_type: "success", text_result_for_llm: '{"providers":[{"status":"authorized"}]}' }, + })); + await cap(parent({ + hook_event_name: "PostToolUse", + tool_name: "arcade-Arcade_UseTool", + tool_input: { tool_name: "Gmail.ListEmails" }, + tool_result: { result_type: "success", text_result_for_llm: "ok" }, + })); + await cap(parent({ + hook_event_name: "PostToolUseFailure", + tool_name: "arcade-Slack_SendMessage", + tool_input: {}, + error: "MCP server 'arcade': Something went wrong in the upstream service", + })); + await cap(parent({ hook_event_name: "UserPromptSubmit", prompt: "Fix the parser in src/main.ts" })); + await cap(parent({ + hook_event_name: "PostToolUse", + tool_name: "granola-Granola_ListMeetings", + tool_input: {}, + tool_result: { result_type: "success", text_result_for_llm: "[]" }, + })); + await cap(parent({ + hook_event_name: "SubagentStop", + agent_type: COPILOT_OPERATOR, + agent_id: COPILOT_SUBAGENT, + last_assistant_message: "status: needs_auth\nsummary: sign in", + })); + await cap(child({ hook_event_name: "UserPromptSubmit", prompt: "What is on my calendar tomorrow?" })); + return events; +}; + +/** @param {object} group */ +const claudeCrossCheckExpectation = (group) => { + assert.equal(group.unit, "turn"); + assert.deepEqual(group.denominators, { observed_relevant_turns: 3 }); + assert.equal(group.tool_only_turns, 0); + assert.deepEqual(group.stages, { + gateway_discovery_or_selection: { count: 1, denominator: "observed_relevant_turns" }, + authorization_check_auth_needed_true: { count: 0, denominator: "observed_relevant_turns" }, + authorization_check_auth_needed_false: { count: 1, denominator: "observed_relevant_turns" }, + app_action_called: { count: 1, denominator: "observed_relevant_turns" }, + app_action_failed: { count: 1, denominator: "observed_relevant_turns" }, + attempt_observed_outcome_unknown: { count: 1, denominator: "observed_relevant_turns" }, + no_call_observed: { count: 1, denominator: "observed_relevant_turns" }, + }); +}; + +/** @param {object} group */ +const copilotCrossCheckExpectation = (group) => { + assert.equal(group.unit, "session"); + assert.deepEqual(group.denominators, { observed_relevant_sessions: 1 }); + assert.equal(group.operator_linked_sessions_excluded, 1); + assert.equal(group.multi_prompt_sessions, 0); + assert.equal(group.parent_attribution_unknown_sessions, 0); + assert.deepEqual(group.stages, { + gateway_discovery_or_selection: { count: 1, denominator: "observed_relevant_sessions" }, + authorization_check_auth_needed_true: { count: 0, denominator: "observed_relevant_sessions" }, + authorization_check_auth_needed_false: { count: 1, denominator: "observed_relevant_sessions" }, + app_action_called: { count: 1, denominator: "observed_relevant_sessions" }, + app_action_failed: { count: 1, denominator: "observed_relevant_sessions" }, + no_call_observed: { count: 0, denominator: "observed_relevant_sessions" }, + }); + assert.equal(group.stages.attempt_observed_outcome_unknown, undefined); + assert.deepEqual(group.operator_stop_status, { needs_auth: 1 }); +}; + +for (const host of TELEMETRY_HOSTS) { + test(`adapter cross-check buildReport counts for ${host}`, async (t) => { + const adapterPath = path.join(ROOT, "hooks/telemetry-adapters", `${host}.mjs`); + if (!existsSync(adapterPath)) { + t.skip(`${host} adapter not in this branch`); + return; + } + const adapter = await loadTelemetryAdapter(host); + const events = host === "claude-code" ? await runClaudeCrossCheck(adapter) : await runCopilotCrossCheck(adapter); + for (const event of events) assertMatchesContract(event); + const report = buildReport(events); + assert.deepEqual(report.excluded, { invalid: 0, legacy: 0 }); + assert.equal(report.groups.length, 1); + const group = report.groups[0]; + assert.equal(group.host, host); + if (host === "claude-code") claudeCrossCheckExpectation(group); + else copilotCrossCheckExpectation(group); + }); +} From 9b0158682587af10ba475921777d60171260d47e Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 13 Oct 2026 19:39:29 +0000 Subject: [PATCH 02/10] fix(telemetry-report): address the October 13 readiness review Assert the distinct host groups, require both adapters, share the six stage counters, and describe the Claude-only reminder and Copilot opt-out switches accurately. Telemetry stays off. Co-authored-by: Teal Larson --- AGENTS.md | 4 +- docs/telemetry.md | 24 ++++--- scripts/telemetry-report.mjs | 116 ++++++++++++++++----------------- test/telemetry-report.test.mjs | 76 ++++++++++++--------- 4 files changed, 118 insertions(+), 102 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 257cd90..406b289 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -23,8 +23,8 @@ for a marked rules block. `hooks/telemetry-config.mjs` stays `false` without separate approval; while false, `npm run generate` writes no telemetry hooks. Reporting and maintained docs for exports live in `scripts/telemetry-report.mjs` and - `docs/telemetry.md`. `test/telemetry-report.test.mjs` cross-checks adapters - when `hooks/telemetry-adapters/.mjs` exists on the branch. + `docs/telemetry.md`. `test/telemetry-report.test.mjs` cross-checks every + adapter named by the contract and fails if that adapter file is missing. - Codex hooks are blocked upstream ([docs/install/codex.md](docs/install/codex.md)). Don't remove the root `$schema` to force them; that breaks Agent Plugins conformance. diff --git a/docs/telemetry.md b/docs/telemetry.md index 95240b6..8782a91 100644 --- a/docs/telemetry.md +++ b/docs/telemetry.md @@ -38,8 +38,10 @@ event with `could_use_arcade: false` and no service hints. Keyword matching can misclassify prompts; the labeled evaluation measures that limitation separately. Session starts send no usage event. Only the Arcade operator's stop reports -send subagent events. The routing reminder goes on every prompt except short -acknowledgements and background task results, whether or not telemetry is on. +send subagent events. Claude Code adds its routing reminder on every prompt +except short acknowledgements and background task results, whether or not +telemetry is on. Copilot CLI has no routing reminder hook, so `reminder_sent` +stays false there. ## Opt-outs when telemetry is on @@ -61,14 +63,16 @@ opt out when set to **any** non-empty value (including `0` and `false`), matchin Claude's own telemetry switches. With telemetry opted out, the client still invokes its configured Node hooks. -The telemetry hook exits before classifying the prompt or storing state, and the -routing hook still adds its reminder. An environment variable cannot remove -hooks from the manifest. +The telemetry hook exits before classifying the prompt or storing state. Claude +Code's routing hook still adds its reminder. An environment variable cannot +remove hooks from the manifest. -In Copilot CLI, set `ARCADE_PLUGIN_TELEMETRY=0` in your shell before starting -`copilot`. `COPILOT_OFFLINE` uses the same `0` / `false` / `off` / `no` semantics -as `ARCADE_PLUGIN_TELEMETRY`; any other non-empty value turns telemetry off -(along with other Copilot network activity). +In Copilot CLI, set `ARCADE_PLUGIN_TELEMETRY` to `0`, `false`, `off`, or `no` +in your shell before starting `copilot`. Any other value, including an unset +variable, leaves that switch inactive. `COPILOT_OFFLINE` turns telemetry off +when it is set to any non-empty value other than `0`, `false`, `off`, or `no`. +An unset or empty `COPILOT_OFFLINE`, and those four values, leave telemetry on +through this switch. For testing an enabled build, `ARCADE_PLUGIN_TELEMETRY_HOST` sends events to a different host. @@ -99,7 +103,7 @@ not run telemetry today. Cursor is not wired up; its hook input can include the user's email. claude.ai, ChatGPT, Codex, and OpenCode don't run telemetry hooks from this package. -### What CI exercises (when adapters are present) +### What CI exercises | Adapter | Tested in this repo | Not covered here | | --- | --- | --- | diff --git a/scripts/telemetry-report.mjs b/scripts/telemetry-report.mjs index 93a676a..3338967 100644 --- a/scripts/telemetry-report.mjs +++ b/scripts/telemetry-report.mjs @@ -107,66 +107,87 @@ const isAppActionCalledRow = (row) => const isAppActionFailedRow = (row) => row.event === "Plugin tool failed" && APP_ACTION_TOOLS.has(toolName(row)); +/** @param {{ event: string, properties: Record }[]} rows */ +const gatewayRow = (rows) => rows.some(isDiscoveryRow); + /** - * @param {Set} turnIds - * @param {Map }[]>} byTurn + * @param {{ event: string, properties: Record }[]} rows + * @param {boolean} needed */ -const countTurnStages = (turnIds, byTurn) => { +const authRow = (rows, needed) => + rows.some( + (row) => isAuthRow(row) && row.properties.auth_needed === needed, + ); + +/** + * An outcome only settles attempts on its own `server` category, so an `other_arcade` + * result cannot hide an `arcade` attempt. Within a category the check is turn-level. + * @param {{ event: string, properties: Record }[]} arcadeRows + */ +const attemptWithoutOutcome = (arcadeRows) => + ["arcade", "other_arcade"].some((server) => { + const onServer = arcadeRows.filter((row) => row.properties.server === server); + const hasAttempt = onServer.some((row) => row.event === "Plugin tool attempted"); + const hasOutcome = onServer.some((row) => OUTCOME_EVENTS.has(row.event)); + return hasAttempt && !hasOutcome; + }); + +/** + * The six stages both hosts share. Each entry is one Claude turn or one Copilot session. + * @param {Iterable<{ event: string, properties: Record }[]>} units + */ +const countSharedStages = (units) => { let gateway = 0; let authNeededTrue = 0; let authNeededFalse = 0; let appCalled = 0; let appFailed = 0; - let attemptOutcomeUnknown = 0; let noCallObserved = 0; - for (const turn of turnIds) { - const rows = byTurn.get(turn) ?? []; + for (const rows of units) { const arcadeRows = rows.filter(isArcadeMcpRow); if (gatewayRow(rows)) gateway += 1; if (authRow(rows, true)) authNeededTrue += 1; if (authRow(rows, false)) authNeededFalse += 1; if (rows.some(isAppActionCalledRow)) appCalled += 1; if (rows.some(isAppActionFailedRow)) appFailed += 1; - if (attemptWithoutOutcome(arcadeRows)) attemptOutcomeUnknown += 1; if (arcadeRows.length === 0) noCallObserved += 1; } - return { - gateway_discovery_or_selection: { count: gateway, denominator: "observed_relevant_turns" }, - authorization_check_auth_needed_true: { count: authNeededTrue, denominator: "observed_relevant_turns" }, - authorization_check_auth_needed_false: { count: authNeededFalse, denominator: "observed_relevant_turns" }, - app_action_called: { count: appCalled, denominator: "observed_relevant_turns" }, - app_action_failed: { count: appFailed, denominator: "observed_relevant_turns" }, - attempt_observed_outcome_unknown: { count: attemptOutcomeUnknown, denominator: "observed_relevant_turns" }, - no_call_observed: { count: noCallObserved, denominator: "observed_relevant_turns" }, - }; + return { gateway, authNeededTrue, authNeededFalse, appCalled, appFailed, noCallObserved }; }; -/** @param {{ event: string, properties: Record }[]} rows */ -const gatewayRow = (rows) => rows.some(isDiscoveryRow); - /** - * @param {{ event: string, properties: Record }[]} rows - * @param {boolean} needed + * @param {ReturnType} counts + * @param {string} denominator + * @param {Record} [beforeNoCall] + * Claude inserts attempt_observed_outcome_unknown here. Copilot omits it. */ -const authRow = (rows, needed) => - rows.some( - (row) => isAuthRow(row) && row.properties.auth_needed === needed, - ); +const sharedStageReport = (counts, denominator, beforeNoCall = {}) => ({ + gateway_discovery_or_selection: { count: counts.gateway, denominator }, + authorization_check_auth_needed_true: { count: counts.authNeededTrue, denominator }, + authorization_check_auth_needed_false: { count: counts.authNeededFalse, denominator }, + app_action_called: { count: counts.appCalled, denominator }, + app_action_failed: { count: counts.appFailed, denominator }, + ...beforeNoCall, + no_call_observed: { count: counts.noCallObserved, denominator }, +}); /** - * An outcome only settles attempts on its own `server` category, so an `other_arcade` - * result cannot hide an `arcade` attempt. Within a category the check is turn-level. - * @param {{ event: string, properties: Record }[]} arcadeRows + * @param {Set} turnIds + * @param {Map }[]>} byTurn */ -const attemptWithoutOutcome = (arcadeRows) => - ["arcade", "other_arcade"].some((server) => { - const onServer = arcadeRows.filter((row) => row.properties.server === server); - const hasAttempt = onServer.some((row) => row.event === "Plugin tool attempted"); - const hasOutcome = onServer.some((row) => OUTCOME_EVENTS.has(row.event)); - return hasAttempt && !hasOutcome; +const countTurnStages = (turnIds, byTurn) => { + const denominator = "observed_relevant_turns"; + const units = [...turnIds].map((turn) => byTurn.get(turn) ?? []); + let attemptOutcomeUnknown = 0; + for (const rows of units) { + if (attemptWithoutOutcome(rows.filter(isArcadeMcpRow))) attemptOutcomeUnknown += 1; + } + return sharedStageReport(countSharedStages(units), denominator, { + attempt_observed_outcome_unknown: { count: attemptOutcomeUnknown, denominator }, }); +}; /** * @param {{ event: string, properties: Record }[]} rows @@ -306,23 +327,7 @@ const buildCopilotGroup = (rows) => { bySession.set(session, list); } - let gateway = 0; - let authNeededTrue = 0; - let authNeededFalse = 0; - let appCalled = 0; - let appFailed = 0; - let noCallObserved = 0; - - for (const session of denominatorSessions) { - const sessionRows = bySession.get(session) ?? []; - const arcadeRows = sessionRows.filter(isArcadeMcpRow); - if (gatewayRow(sessionRows)) gateway += 1; - if (authRow(sessionRows, true)) authNeededTrue += 1; - if (authRow(sessionRows, false)) authNeededFalse += 1; - if (sessionRows.some(isAppActionCalledRow)) appCalled += 1; - if (sessionRows.some(isAppActionFailedRow)) appFailed += 1; - if (arcadeRows.length === 0) noCallObserved += 1; - } + const sessionUnits = [...denominatorSessions].map((session) => bySession.get(session) ?? []); return { unit: "session", @@ -330,14 +335,7 @@ const buildCopilotGroup = (rows) => { operator_linked_sessions_excluded: linkedChildren.size, multi_prompt_sessions: multiPromptSessions, parent_attribution_unknown_sessions: parentAttributionUnknown, - stages: { - gateway_discovery_or_selection: { count: gateway, denominator: "observed_relevant_sessions" }, - authorization_check_auth_needed_true: { count: authNeededTrue, denominator: "observed_relevant_sessions" }, - authorization_check_auth_needed_false: { count: authNeededFalse, denominator: "observed_relevant_sessions" }, - app_action_called: { count: appCalled, denominator: "observed_relevant_sessions" }, - app_action_failed: { count: appFailed, denominator: "observed_relevant_sessions" }, - no_call_observed: { count: noCallObserved, denominator: "observed_relevant_sessions" }, - }, + stages: sharedStageReport(countSharedStages(sessionUnits), "observed_relevant_sessions"), failure_kinds: failureKindCounts(rows), operator_stop_status: operatorStatusCounts(rows), }; diff --git a/test/telemetry-report.test.mjs b/test/telemetry-report.test.mjs index 2e9ee44..3ebf2ba 100644 --- a/test/telemetry-report.test.mjs +++ b/test/telemetry-report.test.mjs @@ -1,12 +1,13 @@ // @ts-check import assert from "node:assert/strict"; -import { existsSync, readFileSync, writeFileSync } from "node:fs"; +import { readFileSync, writeFileSync } from "node:fs"; import { spawnSync } from "node:child_process"; import path from "node:path"; import { test } from "node:test"; import { TELEMETRY_HOSTS } from "../hooks/telemetry-contract.mjs"; import { loadTelemetryAdapter } from "../hooks/telemetry-adapter.mjs"; +import { ARCADE_TOOL_PREFIX } from "../hooks/telemetry-adapters/claude-code.mjs"; import { buildReport, normalizeExportRow, parseExportedEvents } from "../scripts/telemetry-report.mjs"; import { assertMatchesContract, captureTelemetry, hookInput, tempDataDir } from "./telemetry-helpers.mjs"; import { ROOT } from "./helpers.mjs"; @@ -129,6 +130,10 @@ test("an outcome on one Arcade server category does not settle an attempt on the assert.equal(unknownOutcomeCount([prompt, attempt("arcade"), called("arcade")]), 0); assert.equal(unknownOutcomeCount([prompt, attempt("other_arcade"), failed("other_arcade")]), 0); assert.equal(unknownOutcomeCount([prompt, attempt("arcade"), attempt("other_arcade"), called("arcade")]), 1); + // Order within the turn does not matter: an earlier outcome still settles its category. + assert.equal(unknownOutcomeCount([prompt, called("arcade"), attempt("arcade")]), 0); + assert.equal(unknownOutcomeCount([prompt, failed("other_arcade"), attempt("other_arcade")]), 0); + assert.equal(unknownOutcomeCount([prompt, called("other_arcade"), attempt("arcade")]), 1); // Turn-level only: one outcome on a category settles every attempt on that category. assert.equal(unknownOutcomeCount([prompt, attempt("arcade"), attempt("arcade"), called("arcade")]), 0); }); @@ -165,22 +170,42 @@ test("CLI rejects malformed JSON without echoing the input", (t) => { } }); +test("empty input produces a report with no groups", () => { + assert.deepEqual(parseExportedEvents(""), []); + assert.deepEqual(parseExportedEvents(" \n\n"), []); + const report = buildReport([]); + assert.deepEqual(report.excluded, { invalid: 0, legacy: 0 }); + assert.deepEqual(report.groups, []); + assert.ok(report.limits.length > 0); + + const file = path.join(tempDataDir(), "empty.jsonl"); + writeFileSync(file, ""); + const result = spawnSync(process.execPath, [path.join(ROOT, "scripts/telemetry-report.mjs"), file], { + encoding: "utf8", + }); + assert.equal(result.status, 0, result.stderr); + assert.deepEqual(JSON.parse(result.stdout), report); +}); + test("report keys avoid forbidden metric names and hosts stay separate", () => { - const { events } = loadFixture(); + const { events, expected } = loadFixture(); const report = buildReport(events); for (const key of collectKeys(report)) { assert.doesNotMatch(key, FORBIDDEN_KEY, key); } - const hosts = report.groups.map((group) => group.host); - assert.deepEqual(new Set(hosts), new Set(hosts)); - for (const group of report.groups) { - if (group.host === "copilot-cli") { - assert.equal(group.stages.attempt_observed_outcome_unknown, undefined); - } - if (group.host === "claude-code") { - assert.ok(group.stages.attempt_observed_outcome_unknown); - } - } + assert.deepEqual( + report.groups.map((group) => group.host), + ["claude-code", "copilot-cli"], + ); + const byHost = Object.fromEntries(report.groups.map((group) => [group.host, group])); + const expectedByHost = Object.fromEntries(expected.groups.map((group) => [group.host, group])); + assert.equal(byHost["claude-code"].unit, "turn"); + assert.equal(byHost["copilot-cli"].unit, "session"); + assert.deepEqual(byHost["claude-code"].stages, expectedByHost["claude-code"].stages); + assert.deepEqual(byHost["copilot-cli"].stages, expectedByHost["copilot-cli"].stages); + assert.equal(byHost["copilot-cli"].stages.attempt_observed_outcome_unknown, undefined); + assert.ok(byHost["claude-code"].stages.attempt_observed_outcome_unknown); + assert.notDeepEqual(byHost["claude-code"].stages, byHost["copilot-cli"].stages); }); const authStatusResponse = (statuses) => [ @@ -193,12 +218,6 @@ const authStatusResponse = (statuses) => [ }, ]; -const claudeArcadePrefix = () => { - const plugin = JSON.parse(readFileSync(path.join(ROOT, "plugin.json"), "utf8")); - const [server] = Object.keys(JSON.parse(readFileSync(path.join(ROOT, "mcp.json"), "utf8")).mcpServers); - return `mcp__plugin_${plugin.name}_${server}__`; -}; - const COPILOT_SESSION = "dd3beb80-4471-4513-99a4-a57d3d7c08df"; const COPILOT_SUBAGENT = "57946be7-1a73-40ca-a042-abb0f68d9445"; const COPILOT_OPERATOR = "arcade:arcade-operator"; @@ -206,7 +225,6 @@ const COPILOT_OPERATOR = "arcade:arcade-operator"; /** @param {import("../hooks/telemetry-adapter.mjs").TelemetryAdapter} adapter */ const runClaudeCrossCheck = async (adapter) => { const dataDir = tempDataDir(); - const prefix = claudeArcadePrefix(); const session = "cross-check-claude-session"; /** @type {object[]} */ const events = []; @@ -224,22 +242,22 @@ const runClaudeCrossCheck = async (adapter) => { const p3 = "turn-three-33333333"; const p4 = "turn-four-44444444"; await cap({ hook_event_name: "UserPromptSubmit", prompt: "What is on my calendar tomorrow?", prompt_id: p1 }); - await cap({ hook_event_name: "PostToolUse", tool_name: `${prefix}Arcade_SelectTools`, prompt_id: p1 }); + await cap({ hook_event_name: "PostToolUse", tool_name: `${ARCADE_TOOL_PREFIX}Arcade_SelectTools`, prompt_id: p1 }); await cap({ hook_event_name: "PostToolUse", - tool_name: `${prefix}System_ManageAuthorization`, + tool_name: `${ARCADE_TOOL_PREFIX}System_ManageAuthorization`, tool_response: authStatusResponse(["authorized"]), prompt_id: p1, }); await cap({ hook_event_name: "PostToolUse", - tool_name: `${prefix}Arcade_UseTool`, + tool_name: `${ARCADE_TOOL_PREFIX}Arcade_UseTool`, tool_input: { tool_name: "Gmail.ListEmails" }, prompt_id: p1, }); await cap({ hook_event_name: "PostToolUseFailure", - tool_name: `${prefix}Slack_SendMessage`, + tool_name: `${ARCADE_TOOL_PREFIX}Slack_SendMessage`, error: "rate limited", prompt_id: p1, }); @@ -252,7 +270,7 @@ const runClaudeCrossCheck = async (adapter) => { await cap({ hook_event_name: "UserPromptSubmit", prompt: "What is on my calendar tomorrow?", prompt_id: p3 }); await cap({ hook_event_name: "PreToolUse", - tool_name: `${prefix}Arcade_UseTool`, + tool_name: `${ARCADE_TOOL_PREFIX}Arcade_UseTool`, tool_input: { tool_name: "Gmail.ListEmails" }, prompt_id: p3, }); @@ -349,12 +367,7 @@ const copilotCrossCheckExpectation = (group) => { }; for (const host of TELEMETRY_HOSTS) { - test(`adapter cross-check buildReport counts for ${host}`, async (t) => { - const adapterPath = path.join(ROOT, "hooks/telemetry-adapters", `${host}.mjs`); - if (!existsSync(adapterPath)) { - t.skip(`${host} adapter not in this branch`); - return; - } + test(`adapter cross-check buildReport counts for ${host}`, async () => { const adapter = await loadTelemetryAdapter(host); const events = host === "claude-code" ? await runClaudeCrossCheck(adapter) : await runCopilotCrossCheck(adapter); for (const event of events) assertMatchesContract(event); @@ -364,6 +377,7 @@ for (const host of TELEMETRY_HOSTS) { const group = report.groups[0]; assert.equal(group.host, host); if (host === "claude-code") claudeCrossCheckExpectation(group); - else copilotCrossCheckExpectation(group); + else if (host === "copilot-cli") copilotCrossCheckExpectation(group); + else assert.fail(`no cross-check for ${host}`); }); } From 758d0cd92efc44fc6e658d9a31655fdf2d71760e Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 13 Oct 2026 19:55:27 +0000 Subject: [PATCH 03/10] fix(telemetry-report): count only excluded prompt sessions operator_linked_sessions_excluded now counts Copilot prompt sessions removed from the denominator. A stop whose child never sent a prompt does not increase it. Co-authored-by: Teal Larson --- docs/telemetry.md | 3 +++ scripts/telemetry-report.mjs | 5 +++- test/telemetry-report.test.mjs | 43 ++++++++++++++++++++++++++++++++++ 3 files changed, 50 insertions(+), 1 deletion(-) diff --git a/docs/telemetry.md b/docs/telemetry.md index 8782a91..5a4af1e 100644 --- a/docs/telemetry.md +++ b/docs/telemetry.md @@ -262,6 +262,9 @@ Report field meanings (from `scripts/telemetry-report.mjs`): `turn` values or Copilot `session` values with a `Plugin prompt submitted` event, after Copilot operator-linked child sessions are excluded from the session denominator. +- `operator_linked_sessions_excluded`: Copilot prompt sessions left out of + that session denominator because an operator stop named them as + `subagent_session`. A stop whose child sent no prompt does not increase it. - `tool_only_turns`: Claude turns with tool events but no prompt event in that turn (reported outside the turn denominator). - `attempt_observed_outcome_unknown`: Claude turns with a `Plugin tool diff --git a/scripts/telemetry-report.mjs b/scripts/telemetry-report.mjs index 3338967..ce40628 100644 --- a/scripts/telemetry-report.mjs +++ b/scripts/telemetry-report.mjs @@ -293,6 +293,9 @@ const buildCopilotGroup = (rows) => { const denominatorSessions = new Set( [...promptSessions].filter((session) => !linkedChildren.has(session)), ); + // Only prompt sessions actually removed from the denominator. A stop whose + // child never sent a prompt does not count. + const operatorLinkedSessionsExcluded = promptSessions.size - denominatorSessions.size; const multiPromptSessions = [...denominatorSessions].filter((session) => { const count = prompts.filter((row) => String(row.properties.session) === session).length; @@ -332,7 +335,7 @@ const buildCopilotGroup = (rows) => { return { unit: "session", denominators: { observed_relevant_sessions: denominatorSessions.size }, - operator_linked_sessions_excluded: linkedChildren.size, + operator_linked_sessions_excluded: operatorLinkedSessionsExcluded, multi_prompt_sessions: multiPromptSessions, parent_attribution_unknown_sessions: parentAttributionUnknown, stages: sharedStageReport(countSharedStages(sessionUnits), "observed_relevant_sessions"), diff --git a/test/telemetry-report.test.mjs b/test/telemetry-report.test.mjs index 3ebf2ba..2001b2c 100644 --- a/test/telemetry-report.test.mjs +++ b/test/telemetry-report.test.mjs @@ -138,6 +138,49 @@ test("an outcome on one Arcade server category does not settle an attempt on the assert.equal(unknownOutcomeCount([prompt, attempt("arcade"), attempt("arcade"), called("arcade")]), 0); }); +const copilotRow = (event, session, extra) => ({ + event, + distinct_id: session, + properties: { + session, + host: "copilot-cli", + plugin_version: "0.2.0", + telemetry_version: 2, + os: "linux", + arcade_used_before: false, + $process_person_profile: false, + $geoip_disable: true, + $ip: "0.0.0.0", + ...extra, + }, +}); + +test("operator_linked_sessions_excluded counts prompt sessions removed from the denominator", () => { + const parent = "bbbbbbbbbbbbbbbb"; + const child = "cccccccccccccccc"; + const prompt = (session) => + copilotRow("Plugin prompt submitted", session, { + could_use_arcade: true, + service_hints: [], + reminder_sent: false, + }); + const stop = copilotRow("Plugin subagent stopped", parent, { + agent: "arcade-operator", + status: "needs_auth", + subagent_session: child, + }); + + for (const row of [prompt(parent), prompt(child), stop]) assertMatchesContract(row); + + const orphanStop = buildReport([prompt(parent), stop]); + assert.equal(orphanStop.groups[0].operator_linked_sessions_excluded, 0); + assert.deepEqual(orphanStop.groups[0].denominators, { observed_relevant_sessions: 1 }); + + const linked = buildReport([prompt(parent), prompt(child), stop]); + assert.equal(linked.groups[0].operator_linked_sessions_excluded, 1); + assert.deepEqual(linked.groups[0].denominators, { observed_relevant_sessions: 1 }); +}); + test("report limits state that other_arcade pools connections and calls are not paired", () => { const limits = buildReport([]).limits.join("\n"); assert.match(limits, /other_arcade.*pools/i); From 31378b81ac1679331aae214719bd4b626d39e2de Mon Sep 17 00:00:00 2001 From: Teal Larson Date: Wed, 14 Oct 2026 13:59:31 -0700 Subject: [PATCH 04/10] docs: remove stale conditional from adapter test coverage --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f444dd3..41f5a8b 100644 --- a/README.md +++ b/README.md @@ -77,7 +77,7 @@ hooks in client manifests. [docs/telemetry.md](docs/telemetry.md) describes the event contract, what would be stored locally when enabled, opt-outs, and how to aggregate exports with `scripts/telemetry-report.mjs`. The Arcade MCP gateway remains the canonical source for request and tool-call telemetry. -Adapter behavior is covered by in-process tests when those files are present; +Adapter behavior is covered by in-process tests; see [docs/telemetry.md](docs/telemetry.md) for what CI does and does not exercise. ## Develop From 657ab52a79b61029b8f9b6af240e247019b72a41 Mon Sep 17 00:00:00 2001 From: Teal Larson Date: Wed, 14 Oct 2026 19:26:50 -0700 Subject: [PATCH 05/10] fix(telemetry-report): correct line numbers, legacy detection, tool_only_sessions, dead code, and wording - fix(3): split untrimmed text so blank lines count toward physical line numbers; use "the file is not valid JSON" for JSON array parse errors - fix(2): isLegacyRow now requires all property keys to be in allowedProperties; rows with extra properties (e.g. prompt, cwd) count as invalid - fix(4): move buildReport and JSON.stringify inside the try/catch so any build error prints the one-line message and exits 1 - fix(1): add tool_only_sessions to buildCopilotGroup, placed after multi_prompt_sessions, counting sessions not in observed_relevant_sessions that have at least one non-prompt row - fix(6): remove always-true .some check from toolOnlyTurns; simplify groupKey to drop unreachable legacy branch; remove unreachable unit:"unknown" branch; replace linkedChildren with childToParent.has and remove parent ?? session fallback - fix(7): replace "settles" and "pools" in comments and REPORT_LIMITS; update normalizeExportRow doc comment - fix(5): add fixture rows for plugin_version 0.3.0 (claude-code) to test group-by-version; add rows with server "other" for both hosts to test no_call_observed; add tests for all fixes Co-Authored-By: Claude Opus 5.5 --- scripts/telemetry-report.mjs | 77 +++++----- test/fixtures/telemetry-report/events.jsonl | 6 + .../telemetry-report/expected-report.json | 57 ++++++- test/telemetry-report.test.mjs | 140 ++++++++++++++++-- 4 files changed, 218 insertions(+), 62 deletions(-) diff --git a/scripts/telemetry-report.mjs b/scripts/telemetry-report.mjs index ce40628..16856b7 100644 --- a/scripts/telemetry-report.mjs +++ b/scripts/telemetry-report.mjs @@ -24,7 +24,7 @@ const validateRow = new Ajv2020({ allErrors: true }).compile(eventSchema()); const isValidRow = (row) => validateRow(row) === true; /** - * Drops export envelope fields and PostHog `$` noise; other properties must match the contract. + * Drops export fields outside event, distinct_id, and properties, and PostHog `$` properties not on the contract. * @param {unknown} raw * @returns {{ event: string, distinct_id: string, properties: Record } | null} */ @@ -49,13 +49,15 @@ export const normalizeExportRow = (raw) => { const isLegacyRow = (row) => { if (!row || typeof row !== "object") return false; const record = /** @type {{ event?: string, properties?: Record }} */ (row); - return ( - typeof record.event === "string" && - KNOWN_EVENTS.has(record.event) && - record.properties && - typeof record.properties === "object" && - record.properties.telemetry_version === undefined - ); + if ( + typeof record.event !== "string" || + !KNOWN_EVENTS.has(record.event) || + !record.properties || + typeof record.properties !== "object" || + record.properties.telemetry_version !== undefined + ) return false; + const allowed = new Set(allowedProperties(record.event)); + return Object.keys(record.properties).every((key) => allowed.has(key)); }; /** @@ -65,8 +67,7 @@ const isLegacyRow = (row) => { const groupKey = (properties) => ({ host: String(properties.host), plugin_version: String(properties.plugin_version), - telemetry_version: - properties.telemetry_version === undefined ? "legacy" : Number(properties.telemetry_version), + telemetry_version: Number(properties.telemetry_version), }); /** @param {string} server */ @@ -120,8 +121,8 @@ const authRow = (rows, needed) => ); /** - * An outcome only settles attempts on its own `server` category, so an `other_arcade` - * result cannot hide an `arcade` attempt. Within a category the check is turn-level. + * An outcome only counts as the outcome for attempts on its own `server` category, so an + * `other_arcade` result cannot hide an `arcade` attempt. Within a category the check is turn-level. * @param {{ event: string, properties: Record }[]} arcadeRows */ const attemptWithoutOutcome = (arcadeRows) => @@ -255,7 +256,7 @@ const buildClaudeGroup = (rows) => { } const toolOnlyTurns = [...byTurn.keys()].filter( - (turn) => !relevantTurns.has(turn) && (byTurn.get(turn) ?? []).some((row) => row.event !== "Plugin prompt submitted"), + (turn) => !relevantTurns.has(turn), ).length; return { @@ -279,19 +280,17 @@ const buildCopilotGroup = (rows) => { /** @type {Map} */ const childToParent = new Map(); - const linkedChildren = new Set(); for (const row of rows) { if (row.event !== "Plugin subagent stopped") continue; const parent = String(row.properties.session); const child = row.properties.subagent_session; if (typeof child === "string" && child) { childToParent.set(child, parent); - linkedChildren.add(child); } } const denominatorSessions = new Set( - [...promptSessions].filter((session) => !linkedChildren.has(session)), + [...promptSessions].filter((session) => !childToParent.has(session)), ); // Only prompt sessions actually removed from the denominator. A stop whose // child never sent a prompt does not count. @@ -313,30 +312,28 @@ const buildCopilotGroup = (rows) => { /** @type {Map }[]>} */ const bySession = new Map(); - /** @param {{ event: string, properties: Record }} row */ - const attributeSession = (row) => { - const session = String(row.properties.session); - if (linkedChildren.has(session)) { - const parent = childToParent.get(session); - return parent ?? session; - } - return session; - }; - for (const row of rows) { - const session = attributeSession(row); - const list = bySession.get(session) ?? []; + const session = String(row.properties.session); + const key = childToParent.get(session) ?? session; + const list = bySession.get(key) ?? []; list.push(row); - bySession.set(session, list); + bySession.set(key, list); } const sessionUnits = [...denominatorSessions].map((session) => bySession.get(session) ?? []); + const toolOnlySessions = [...bySession.keys()].filter( + (session) => + !denominatorSessions.has(session) && + (bySession.get(session) ?? []).some((row) => row.event !== "Plugin prompt submitted"), + ).length; + return { unit: "session", denominators: { observed_relevant_sessions: denominatorSessions.size }, operator_linked_sessions_excluded: operatorLinkedSessionsExcluded, multi_prompt_sessions: multiPromptSessions, + tool_only_sessions: toolOnlySessions, parent_attribution_unknown_sessions: parentAttributionUnknown, stages: sharedStageReport(countSharedStages(sessionUnits), "observed_relevant_sessions"), failure_kinds: failureKindCounts(rows), @@ -350,7 +347,7 @@ const REPORT_LIMITS = [ "Claude Code turn counts and Copilot CLI session counts must not be combined into one rate.", "Copilot CLI does not emit tool-attempt events; attempt stages apply only to Claude Code.", "Multiple tool calls in one turn or session cannot be paired without a tool-call ID.", - "Attempt outcomes are checked per server category; other_arcade pools every other Arcade connection, so an outcome there cannot be tied to the connection or call that was attempted.", + "Attempt outcomes are checked per server category; other_arcade groups every other Arcade connection together, so an outcome there cannot be tied to the connection or call that was attempted.", "The Arcade MCP gateway remains the canonical source for request, auth, discovery, and tool-call telemetry.", ]; @@ -388,11 +385,7 @@ export const buildReport = (events) => { for (const [key, rows] of [...groups.entries()].sort(([a], [b]) => a.localeCompare(b))) { const meta = groupKey(rows[0].properties); const body = - meta.host === "claude-code" - ? buildClaudeGroup(rows) - : meta.host === "copilot-cli" - ? buildCopilotGroup(rows) - : { unit: "unknown", denominators: {}, stages: {} }; + meta.host === "claude-code" ? buildClaudeGroup(rows) : buildCopilotGroup(rows); reportGroups.push({ ...meta, ...body }); } @@ -420,10 +413,10 @@ const parseJson = (text, where) => { * @returns {unknown[]} */ export const parseExportedEvents = (text) => { + if (!text.trim()) return []; const trimmed = text.trim(); - if (!trimmed) return []; - if (trimmed.startsWith("[")) return parseJson(trimmed, "the array"); - return trimmed + if (trimmed.startsWith("[")) return parseJson(trimmed, "the file"); + return text .split("\n") .map((line, index) => ({ line: line.trim(), number: index + 1 })) .filter(({ line }) => line) @@ -436,15 +429,13 @@ if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.ur console.error("usage: node scripts/telemetry-report.mjs "); process.exit(1); } - /** @type {unknown[]} */ - let events; try { - events = parseExportedEvents(readFileSync(file, "utf8")); + const events = parseExportedEvents(readFileSync(file, "utf8")); + console.log(JSON.stringify(buildReport(events), null, 2)); } catch (error) { // Error text from JSON.parse or the runtime can quote the input, so print only our own message. - const reason = error instanceof ExportParseError ? error.message : "the file could not be read"; + const reason = error instanceof ExportParseError ? error.message : "the report could not be built"; console.error(`telemetry-report: ${reason}; no report was produced`); process.exit(1); } - console.log(JSON.stringify(buildReport(events), null, 2)); } diff --git a/test/fixtures/telemetry-report/events.jsonl b/test/fixtures/telemetry-report/events.jsonl index b4b983e..7237168 100644 --- a/test/fixtures/telemetry-report/events.jsonl +++ b/test/fixtures/telemetry-report/events.jsonl @@ -19,3 +19,9 @@ {"event":"Plugin prompt submitted","distinct_id":"eeeeeeeeeeeeeeee","properties":{"session":"eeeeeeeeeeeeeeee","host":"copilot-cli","plugin_version":"0.2.0","telemetry_version":2,"os":"linux","arcade_used_before":false,"$process_person_profile":false,"$geoip_disable":true,"$ip":"0.0.0.0","could_use_arcade":true,"service_hints":["calendar"],"reminder_sent":false}} {"event":"Plugin prompt submitted","distinct_id":"aaaaaaaaaaaaaaaa","properties":{"session":"aaaaaaaaaaaaaaaa","host":"claude-code","plugin_version":"0.2.0","os":"linux","arcade_used_before":false,"$process_person_profile":false,"$geoip_disable":true,"$ip":"0.0.0.0","could_use_arcade":true,"service_hints":[],"reminder_sent":true}} {"event":"not-a-contract-event","distinct_id":"ffffffffffffffff","properties":{"session":"ffffffffffffffff","host":"claude-code","plugin_version":"0.2.0","telemetry_version":2,"os":"linux","arcade_used_before":false,"$process_person_profile":false,"$geoip_disable":true,"$ip":"0.0.0.0"}} +{"event":"Plugin prompt submitted","distinct_id":"aaaaaaaaaaaaaaaa","properties":{"session":"aaaaaaaaaaaaaaaa","turn":"6666666666666666","host":"claude-code","plugin_version":"0.2.0","telemetry_version":2,"os":"linux","arcade_used_before":true,"$process_person_profile":false,"$geoip_disable":true,"$ip":"0.0.0.0","could_use_arcade":true,"service_hints":[],"reminder_sent":false}} +{"event":"Plugin tool called","distinct_id":"aaaaaaaaaaaaaaaa","properties":{"session":"aaaaaaaaaaaaaaaa","turn":"6666666666666666","host":"claude-code","plugin_version":"0.2.0","telemetry_version":2,"os":"linux","arcade_used_before":true,"$process_person_profile":false,"$geoip_disable":true,"$ip":"0.0.0.0","server":"other"}} +{"event":"Plugin prompt submitted","distinct_id":"f0f0f0f0f0f0f0f0","properties":{"session":"f0f0f0f0f0f0f0f0","host":"copilot-cli","plugin_version":"0.2.0","telemetry_version":2,"os":"linux","arcade_used_before":false,"$process_person_profile":false,"$geoip_disable":true,"$ip":"0.0.0.0","could_use_arcade":true,"service_hints":["chat"],"reminder_sent":false}} +{"event":"Plugin tool called","distinct_id":"f0f0f0f0f0f0f0f0","properties":{"session":"f0f0f0f0f0f0f0f0","host":"copilot-cli","plugin_version":"0.2.0","telemetry_version":2,"os":"linux","arcade_used_before":false,"$process_person_profile":false,"$geoip_disable":true,"$ip":"0.0.0.0","server":"other"}} +{"event":"Plugin prompt submitted","distinct_id":"a1a1a1a1a1a1a1a1","properties":{"session":"a1a1a1a1a1a1a1a1","turn":"7777777777777777","host":"claude-code","plugin_version":"0.3.0","telemetry_version":2,"os":"linux","arcade_used_before":false,"$process_person_profile":false,"$geoip_disable":true,"$ip":"0.0.0.0","could_use_arcade":true,"service_hints":["email"],"reminder_sent":false}} +{"event":"Plugin tool called","distinct_id":"a1a1a1a1a1a1a1a1","properties":{"session":"a1a1a1a1a1a1a1a1","turn":"7777777777777777","host":"claude-code","plugin_version":"0.3.0","telemetry_version":2,"os":"linux","arcade_used_before":false,"$process_person_profile":false,"$geoip_disable":true,"$ip":"0.0.0.0","server":"arcade","tool":"Arcade_UseTool","service":"email"}} diff --git a/test/fixtures/telemetry-report/expected-report.json b/test/fixtures/telemetry-report/expected-report.json index 05662f6..9413aba 100644 --- a/test/fixtures/telemetry-report/expected-report.json +++ b/test/fixtures/telemetry-report/expected-report.json @@ -10,7 +10,7 @@ "telemetry_version": 2, "unit": "turn", "denominators": { - "observed_relevant_turns": 3 + "observed_relevant_turns": 4 }, "tool_only_turns": 1, "stages": { @@ -39,7 +39,7 @@ "denominator": "observed_relevant_turns" }, "no_call_observed": { - "count": 1, + "count": 2, "denominator": "observed_relevant_turns" } }, @@ -56,17 +56,62 @@ "completed": 1 } }, + { + "host": "claude-code", + "plugin_version": "0.3.0", + "telemetry_version": 2, + "unit": "turn", + "denominators": { + "observed_relevant_turns": 1 + }, + "tool_only_turns": 0, + "stages": { + "gateway_discovery_or_selection": { + "count": 0, + "denominator": "observed_relevant_turns" + }, + "authorization_check_auth_needed_true": { + "count": 0, + "denominator": "observed_relevant_turns" + }, + "authorization_check_auth_needed_false": { + "count": 0, + "denominator": "observed_relevant_turns" + }, + "app_action_called": { + "count": 1, + "denominator": "observed_relevant_turns" + }, + "app_action_failed": { + "count": 0, + "denominator": "observed_relevant_turns" + }, + "attempt_observed_outcome_unknown": { + "count": 0, + "denominator": "observed_relevant_turns" + }, + "no_call_observed": { + "count": 0, + "denominator": "observed_relevant_turns" + } + }, + "builtin_tools_called": {}, + "builtin_tools_failed": {}, + "failure_kinds": {}, + "operator_stop_status": {} + }, { "host": "copilot-cli", "plugin_version": "0.2.0", "telemetry_version": 2, "unit": "session", "denominators": { - "observed_relevant_sessions": 3 + "observed_relevant_sessions": 4 }, "operator_linked_sessions_excluded": 1, "multi_prompt_sessions": 1, - "parent_attribution_unknown_sessions": 2, + "tool_only_sessions": 0, + "parent_attribution_unknown_sessions": 3, "stages": { "gateway_discovery_or_selection": { "count": 1, @@ -89,7 +134,7 @@ "denominator": "observed_relevant_sessions" }, "no_call_observed": { - "count": 2, + "count": 3, "denominator": "observed_relevant_sessions" } }, @@ -105,7 +150,7 @@ "Claude Code turn counts and Copilot CLI session counts must not be combined into one rate.", "Copilot CLI does not emit tool-attempt events; attempt stages apply only to Claude Code.", "Multiple tool calls in one turn or session cannot be paired without a tool-call ID.", - "Attempt outcomes are checked per server category; other_arcade pools every other Arcade connection, so an outcome there cannot be tied to the connection or call that was attempted.", + "Attempt outcomes are checked per server category; other_arcade groups every other Arcade connection together, so an outcome there cannot be tied to the connection or call that was attempted.", "The Arcade MCP gateway remains the canonical source for request, auth, discovery, and tool-call telemetry." ] } diff --git a/test/telemetry-report.test.mjs b/test/telemetry-report.test.mjs index 2001b2c..835bea3 100644 --- a/test/telemetry-report.test.mjs +++ b/test/telemetry-report.test.mjs @@ -181,9 +181,9 @@ test("operator_linked_sessions_excluded counts prompt sessions removed from the assert.deepEqual(linked.groups[0].denominators, { observed_relevant_sessions: 1 }); }); -test("report limits state that other_arcade pools connections and calls are not paired", () => { +test("report limits state that other_arcade groups connections and calls are not paired", () => { const limits = buildReport([]).limits.join("\n"); - assert.match(limits, /other_arcade.*pools/i); + assert.match(limits, /other_arcade.*groups every other Arcade connection together/i); assert.match(limits, /cannot be paired/i); }); @@ -237,18 +237,18 @@ test("report keys avoid forbidden metric names and hosts stay separate", () => { assert.doesNotMatch(key, FORBIDDEN_KEY, key); } assert.deepEqual( - report.groups.map((group) => group.host), - ["claude-code", "copilot-cli"], + report.groups.map((group) => [group.host, group.plugin_version]), + [["claude-code", "0.2.0"], ["claude-code", "0.3.0"], ["copilot-cli", "0.2.0"]], ); - const byHost = Object.fromEntries(report.groups.map((group) => [group.host, group])); - const expectedByHost = Object.fromEntries(expected.groups.map((group) => [group.host, group])); - assert.equal(byHost["claude-code"].unit, "turn"); - assert.equal(byHost["copilot-cli"].unit, "session"); - assert.deepEqual(byHost["claude-code"].stages, expectedByHost["claude-code"].stages); - assert.deepEqual(byHost["copilot-cli"].stages, expectedByHost["copilot-cli"].stages); - assert.equal(byHost["copilot-cli"].stages.attempt_observed_outcome_unknown, undefined); - assert.ok(byHost["claude-code"].stages.attempt_observed_outcome_unknown); - assert.notDeepEqual(byHost["claude-code"].stages, byHost["copilot-cli"].stages); + const byGroup = Object.fromEntries(report.groups.map((group) => [`${group.host}/${group.plugin_version}`, group])); + const expectedByGroup = Object.fromEntries(expected.groups.map((group) => [`${group.host}/${group.plugin_version}`, group])); + assert.equal(byGroup["claude-code/0.2.0"].unit, "turn"); + assert.equal(byGroup["copilot-cli/0.2.0"].unit, "session"); + assert.deepEqual(byGroup["claude-code/0.2.0"].stages, expectedByGroup["claude-code/0.2.0"].stages); + assert.deepEqual(byGroup["copilot-cli/0.2.0"].stages, expectedByGroup["copilot-cli/0.2.0"].stages); + assert.equal(byGroup["copilot-cli/0.2.0"].stages.attempt_observed_outcome_unknown, undefined); + assert.ok(byGroup["claude-code/0.2.0"].stages.attempt_observed_outcome_unknown); + assert.notDeepEqual(byGroup["claude-code/0.2.0"].stages, byGroup["copilot-cli/0.2.0"].stages); }); const authStatusResponse = (statuses) => [ @@ -409,6 +409,120 @@ const copilotCrossCheckExpectation = (group) => { assert.deepEqual(group.operator_stop_status, { needs_auth: 1 }); }; +test("tool_only_sessions counts Copilot sessions with no prompt and at least one non-prompt row", () => { + const toolOnlySession = "a0a0a0a0a0a0a0a0"; + const row = copilotRow("Plugin tool called", toolOnlySession, { + server: "arcade", + tool: "Arcade_UseTool", + service: "email", + }); + assertMatchesContract(row); + const report = buildReport([row]); + assert.deepEqual(report.excluded, { invalid: 0, legacy: 0 }); + assert.equal(report.groups[0].denominators.observed_relevant_sessions, 0); + assert.equal(report.groups[0].tool_only_sessions, 1); +}); + +test("tool_only_sessions attributes child rows to parent before counting", () => { + const parent = "b0b0b0b0b0b0b0b0"; + const child = "c0c0c0c0c0c0c0c0"; + const stop = copilotRow("Plugin subagent stopped", parent, { + agent: "arcade-operator", + status: "needs_auth", + subagent_session: child, + }); + const childPrompt = copilotRow("Plugin prompt submitted", child, { + could_use_arcade: true, + service_hints: [], + reminder_sent: false, + }); + const childTool = copilotRow("Plugin tool called", child, { + server: "arcade", + tool: "Arcade_UseTool", + service: "email", + }); + for (const r of [stop, childPrompt, childTool]) assertMatchesContract(r); + const report = buildReport([stop, childPrompt, childTool]); + assert.deepEqual(report.excluded, { invalid: 0, legacy: 0 }); + assert.equal(report.groups[0].operator_linked_sessions_excluded, 1); + assert.equal(report.groups[0].denominators.observed_relevant_sessions, 0); + assert.equal(report.groups[0].tool_only_sessions, 1); +}); + +test("a legacy row with an extra property counts as invalid, not legacy", () => { + const base = { + event: "Plugin prompt submitted", + distinct_id: "aaaaaaaaaaaaaaaa", + properties: { + session: "aaaaaaaaaaaaaaaa", + host: "claude-code", + plugin_version: "0.2.0", + os: "linux", + arcade_used_before: false, + $process_person_profile: false, + $geoip_disable: true, + $ip: "0.0.0.0", + could_use_arcade: true, + service_hints: [], + reminder_sent: false, + }, + }; + // Without extra property: counts as legacy (no telemetry_version, all keys allowed). + assert.deepEqual(buildReport([base]).excluded, { invalid: 0, legacy: 1 }); + // With extra property: counts as invalid. + const withPrompt = { ...base, properties: { ...base.properties, prompt: "SECRET" } }; + assert.deepEqual(buildReport([withPrompt]).excluded, { invalid: 1, legacy: 0 }); + const withCwd = { ...base, properties: { ...base.properties, cwd: "/home/secret" } }; + assert.deepEqual(buildReport([withCwd]).excluded, { invalid: 1, legacy: 0 }); +}); + +test("parseExportedEvents reports the physical line number including blank lines", () => { + const blankLines = "\n\n\n"; + const validLine = '{"event":"Plugin prompt submitted","distinct_id":"aaaaaaaaaaaaaaaa","properties":{"session":"aaaaaaaaaaaaaaaa","host":"claude-code","plugin_version":"0.2.0","telemetry_version":2,"os":"linux","arcade_used_before":false,"$process_person_profile":false,"$geoip_disable":true,"$ip":"0.0.0.0","could_use_arcade":false,"service_hints":[],"reminder_sent":false}}'; + const badLine = "{bad"; + // 3 blank lines + valid row on line 4 + bad row on line 5 + assert.throws( + () => parseExportedEvents(`${blankLines}${validLine}\n${badLine}`), + (err) => err.message === "line 5 is not valid JSON", + ); +}); + +test("parseExportedEvents reports 'the file is not valid JSON' for a bad JSON array", () => { + assert.throws( + () => parseExportedEvents("[{bad}]"), + (err) => err.message === "the file is not valid JSON", + ); +}); + +test("a Copilot session whose only tool event is server other counts as no_call_observed", () => { + const session = "d0d0d0d0d0d0d0d0"; + const prompt = copilotRow("Plugin prompt submitted", session, { + could_use_arcade: true, + service_hints: ["chat"], + reminder_sent: false, + }); + const otherTool = copilotRow("Plugin tool called", session, { server: "other" }); + for (const r of [prompt, otherTool]) assertMatchesContract(r); + const report = buildReport([prompt, otherTool]); + assert.deepEqual(report.excluded, { invalid: 0, legacy: 0 }); + const group = report.groups[0]; + assert.equal(group.denominators.observed_relevant_sessions, 1); + assert.equal(group.stages.no_call_observed.count, 1); + assert.equal(group.stages.app_action_called.count, 0); +}); + +test("a Claude turn whose only tool event is server other counts as no_call_observed", () => { + const prompt = claudeRow("Plugin prompt submitted", { could_use_arcade: true, service_hints: ["chat"], reminder_sent: false }); + const otherTool = claudeRow("Plugin tool called", { server: "other" }); + for (const r of [prompt, otherTool]) assertMatchesContract(r); + const report = buildReport([prompt, otherTool]); + assert.deepEqual(report.excluded, { invalid: 0, legacy: 0 }); + const group = report.groups[0]; + assert.equal(group.denominators.observed_relevant_turns, 1); + assert.equal(group.stages.no_call_observed.count, 1); + assert.equal(group.stages.app_action_called.count, 0); +}); + for (const host of TELEMETRY_HOSTS) { test(`adapter cross-check buildReport counts for ${host}`, async () => { const adapter = await loadTelemetryAdapter(host); From ed884fe367e4cc4fe958a9f2e22156971188624e Mon Sep 17 00:00:00 2001 From: Teal Larson Date: Wed, 14 Oct 2026 19:22:25 -0700 Subject: [PATCH 06/10] fix(telemetry-report): correct coverage claims, test coverage, wording, and contract gaps - docs/telemetry.md CI table, docs/install/claude-code.md, docs/install/copilot.md: clarify that claude plugin validate and verify:copilot run on committed manifests with no telemetry hooks; no real CLI has validated a manifest with telemetry hooks - test/telemetry-docs.test.mjs: assert every EVENTS key and every COMMON_PROPERTIES key appears in buildTelemetryTables() output, not just one example of each - ARCHITECTURE.md: add hooks/telemetry-contract.mjs to the Source table - AGENTS.md: add docs/telemetry.md to the list of hand-written files with a generated block - docs/telemetry.md: add tool_only_sessions field definition after tool_only_turns - docs/telemetry.md: split the parse-error sentence so JSONL errors name the line and JSON array errors report no line number - docs/telemetry.md: clarify that a row with no telemetry_version counts as legacy only if every property is on the contract; extra properties make it invalid - docs/telemetry.md: replace "in the shape", "envelope fields", "settles", "pools", "roots", and "transmission gate" with plain literal words Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 7 +++--- ARCHITECTURE.md | 1 + docs/install/claude-code.md | 4 ++- docs/install/copilot.md | 7 ++++-- docs/telemetry.md | 48 +++++++++++++++++++++--------------- test/telemetry-docs.test.mjs | 9 +++++-- 6 files changed, 48 insertions(+), 28 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 406b289..c565407 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -5,9 +5,10 @@ For agents editing **arcade-plugin** itself. Client-specific files are generated. Edit the source, run `npm run generate`, then `npm run verify` (it fails if a generated file was edited by hand). [ARCHITECTURE.md](ARCHITECTURE.md) lists the sources; `.gitattributes` lists -every fully generated file. `skills/try-arcade/SKILL.md` and -`agents/arcade-operator.agent.md` are hand-written except -for a marked rules block. +every fully generated file. `skills/try-arcade/SKILL.md`, +`agents/arcade-operator.agent.md`, and `docs/telemetry.md` are hand-written +except for a marked generated block (the event tables in docs/telemetry.md +come from `hooks/telemetry-contract.mjs`; never edit inside those markers). - Routing rules: edit `hooks/routing-guidance.mjs`, never the rules text in the Cursor rule, arcade-operator, or try-arcade. diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index eadc0c7..43b357f 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -10,6 +10,7 @@ edited by hand. The package ships no credentials. | `plugin.json`, `mcp.json`, `VERSION` | identity, gateway URL, version, Codex listing metadata | | `hooks/routing-guidance.mjs` | the routing rules, as sentences | | `hooks/hook-hosts.mjs` | which hook script runs on which event in which client | +| `hooks/telemetry-contract.mjs` | the telemetry event contract; also generates the event tables in docs/telemetry.md | | `agents/arcade-operator.agent.md` | the operator, outside its generated rules block | | `skills/` | the skills, outside the generated rules block in try-arcade | diff --git a/docs/install/claude-code.md b/docs/install/claude-code.md index e0678b5..470d51e 100644 --- a/docs/install/claude-code.md +++ b/docs/install/claude-code.md @@ -48,7 +48,9 @@ prefixes, and stores scope under `CLAUDE_PLUGIN_DATA`. See `DISABLE_TELEMETRY`, `CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC`) and limits. **Repo coverage:** in-process hook tests and `claude plugin validate --strict` -(**2.1.258**). Not live sessions, IDE extensions, desktop Code tab, or Cowork. +(**2.1.258**) on the committed manifest, which has no telemetry hooks. No CLI +has validated a manifest with telemetry hooks enabled. Not live sessions, IDE +extensions, desktop Code tab, or Cowork. ## First steps diff --git a/docs/install/copilot.md b/docs/install/copilot.md index 6ad5682..c5200db 100644 --- a/docs/install/copilot.md +++ b/docs/install/copilot.md @@ -34,8 +34,11 @@ records MCP tools as `-`, operator stops with `subagent_session`, and session-scoped prompt state under `COPILOT_PLUGIN_DATA` (no `turn`, no `PreToolUse`, no built-in CLI/web events). See [telemetry.md](../telemetry.md). -**Repo coverage:** in-process fixtures and `npm run verify:copilot` (**1.0.88**). -Not live sessions, Windows PowerShell hook commands, or VS Code agent sessions. +**Repo coverage:** in-process fixtures and `npm run verify:copilot` (**1.0.88**) +on the committed manifest, which has no telemetry hooks; `verify:copilot` checks +only MCP servers and skills. No CLI has validated a manifest with telemetry hooks +enabled. Not live sessions, Windows PowerShell hook commands, or VS Code agent +sessions. Shared `com.github.copilot/hooks/hooks.json` uses `runOnlyIfScriptExists` so VS Code exits quietly without a plugin path. diff --git a/docs/telemetry.md b/docs/telemetry.md index 5a4af1e..40a547d 100644 --- a/docs/telemetry.md +++ b/docs/telemetry.md @@ -107,8 +107,8 @@ from this package. | Adapter | Tested in this repo | Not covered here | | --- | --- | --- | -| Claude Code | In-process hook captures, contract tests, `scripts/telemetry-report.mjs` cross-check, `claude plugin validate --strict` on **2.1.258** | Live user sessions, IDE extensions, desktop Code tab, Cowork delivery and opt-out | -| Copilot CLI | In-process captures, fixture JSON, report cross-check, `npm run verify:copilot` on **1.0.88** | Live sessions, Windows PowerShell hook commands, VS Code agent sessions | +| Claude Code | In-process hook captures, contract tests, `scripts/telemetry-report.mjs` cross-check, `claude plugin validate --strict` on **2.1.258** (runs on the committed manifest, which has no telemetry hooks; no CLI has validated a manifest with telemetry hooks enabled) | Live user sessions, IDE extensions, desktop Code tab, Cowork delivery and opt-out | +| Copilot CLI | In-process captures, fixture JSON, report cross-check, `npm run verify:copilot` on **1.0.88** (runs on the committed manifest, which has no telemetry hooks; checks only MCP servers and skills; no CLI has validated a manifest with telemetry hooks enabled) | Live sessions, Windows PowerShell hook commands, VS Code agent sessions | Those checks do not prove end-user transmission while `TELEMETRY_ENABLED` is `false`; they validate the adapters and reporting math for an enabled build. @@ -208,25 +208,29 @@ their own. ## Reading the numbers -Export PostHog rows in the shape `{ event, distinct_id, properties, timestamp? }` -and aggregate them locally: +Export PostHog rows with the fields `event`, `distinct_id`, `properties`, and +optionally `timestamp`, then aggregate them locally: ```bash node scripts/telemetry-report.mjs path/to/export.jsonl ``` The script prints JSON with counts, denominators, and a `limits` list. Before -validating, it ignores export envelope fields outside +validating, it ignores export fields outside `{ event, distinct_id, properties }` (such as `timestamp` and `uuid`) and PostHog-added properties whose names start with `$` but are not on the contract for that event. Every other property must match the contract; leaked hook -fields such as `prompt` or `cwd` make a row invalid. It counts invalid rows and -legacy rows (no `telemetry_version`) separately, and excludes both from grouped -counts. Groups are split by `host`, `plugin_version`, and -`telemetry_version`. Claude Code and Copilot CLI are never combined into one -denominator. If the file is not valid JSON or JSON Lines, the script exits -nonzero with a short message naming the failing line, prints no report, and -does not echo the file's contents. +fields such as `prompt` or `cwd` make a row invalid. A row with a known event +and no `telemetry_version` counts as legacy only if every property key is on +the contract for that event; a row that carries any additional property (such +as `prompt` or `cwd`) counts as invalid instead. It counts invalid rows and +legacy rows separately, and excludes both from grouped counts. Groups are split +by `host`, `plugin_version`, and `telemetry_version`. Claude Code and Copilot +CLI are never combined into one denominator. For JSONL input, a line that is +not valid JSON causes the script to exit nonzero with a message naming that +line number; for JSON array input that fails to parse, the message says the +file is not valid JSON without naming a line. Either way the script prints no +report and does not echo the file's contents. These events measure what plugin hooks observed, not whether Arcade was needed or whether the user's task succeeded. `could_use_arcade` and `service_hints` @@ -267,13 +271,17 @@ Report field meanings (from `scripts/telemetry-report.mjs`): `subagent_session`. A stop whose child sent no prompt does not increase it. - `tool_only_turns`: Claude turns with tool events but no prompt event in that turn (reported outside the turn denominator). +- `tool_only_sessions`: Copilot sessions (after operator child sessions are + attributed to their parent) that are not in `observed_relevant_sessions` and + have at least one row whose event is not `Plugin prompt submitted` (reported + outside the session denominator). Mirrors `tool_only_turns` for Claude Code. - `attempt_observed_outcome_unknown`: Claude turns with a `Plugin tool attempted` on a `server` category (`arcade` or `other_arcade`) but no `Plugin tool called` or `Plugin tool failed` on that same category in the same turn. - An outcome on one category never settles an attempt on the other. - `other_arcade` pools every other Arcade connection, and one outcome settles - every attempt on its category in the turn, so the check cannot pair a result - with a particular connection or call. + An outcome on one category never counts as the outcome for an attempt on the other. + `other_arcade` groups every other Arcade connection together, and one outcome + counts as the outcome for every attempt on its category in the turn, so the + check cannot pair a result with a particular connection or call. - `no_call_observed`: relevant turns or sessions with no Arcade MCP tool event (`Plugin tool attempted`, `Plugin tool called`, or `Plugin tool failed` on `server: arcade` or `other_arcade`). @@ -327,14 +335,14 @@ same machine or prove that the gateway or a particular app was unconnected. Copilot CLI supplies no `prompt_id` or `turn`. Count distinct sessions with a scoped `Plugin prompt submitted` event as **observed relevant sessions**. Report multiple-prompt sessions separately. This is not a count of all sessions, user -tasks, or turns. Session starts are not transmitted and cannot define roots. +tasks, or turns. Session starts are not transmitted and cannot define top-level sessions. An Arcade operator's stop report carries `subagent_session`, equal to that operator's own event `session`. Exclude known linked operator prompts from the parent-session denominator and include their tool events with the parent. Other subagents do not send stop reports, so their prompt sessions cannot be -reliably distinguished from roots. Report that parent attribution is unknown -rather than labeling every unlinked session as a root. +reliably distinguished from top-level sessions. Report that parent attribution +is unknown rather than labeling every unlinked session as a top-level session. Detached event delivery can change arrival order. A missing operator stop or a rolling-window boundary can also remove a parent link. Do not reconstruct @@ -391,4 +399,4 @@ shipping telemetry hooks. Confirm, outside this repository: These are release acceptance requirements; the current code and CI do not enforce them. A repository install can consume a branch before a GitHub release -is tagged, so a later release PR is not a transmission gate. +is tagged, so a branch install can transmit data before any release is tagged. diff --git a/test/telemetry-docs.test.mjs b/test/telemetry-docs.test.mjs index e955f15..71ea502 100644 --- a/test/telemetry-docs.test.mjs +++ b/test/telemetry-docs.test.mjs @@ -10,12 +10,17 @@ import { TELEMETRY_BLOCK_BEGIN, TELEMETRY_BLOCK_END, } from "../scripts/telemetry-docs.mjs"; +import { COMMON_PROPERTIES, EVENTS } from "../hooks/telemetry-contract.mjs"; import { ROOT } from "./helpers.mjs"; test("buildTelemetryTables lists every contract event and common property", () => { const tables = buildTelemetryTables(); - assert.match(tables, /Plugin prompt submitted/); - assert.match(tables, /telemetry_version/); + for (const eventName of Object.keys(EVENTS)) { + assert.ok(tables.includes(eventName), `tables must include event "${eventName}"`); + } + for (const propName of Object.keys(COMMON_PROPERTIES)) { + assert.ok(tables.includes(propName), `tables must include common property "${propName}"`); + } assert.match(tables, /Service categories:/); }); From e5ec76909ac907d7aaff0b34b5e9de278e33ad94 Mon Sep 17 00:00:00 2001 From: Teal Larson Date: Wed, 14 Oct 2026 19:34:42 -0700 Subject: [PATCH 07/10] fix(telemetry-report): tighten tests and messages from the second review - The docs test matches whole table rows, so a missing `os` or `turn` row fails instead of matching inside `subagent_session`. - A missing or unreadable file says so in the one-line error; a CLI test covers it. - docs/telemetry.md: the CI paragraph no longer credits the CLI validators with checking the adapters, and tool_only_sessions notes that child sessions are attributed one level up only. - The Copilot cross-check asserts tool_only_sessions. - Plain wording in test names and comments. Co-Authored-By: Claude Opus 5.5 --- docs/telemetry.md | 9 ++++++--- scripts/telemetry-report.mjs | 6 +++--- test/telemetry-docs.test.mjs | 4 ++-- test/telemetry-report.test.mjs | 20 +++++++++++++++++--- 4 files changed, 28 insertions(+), 11 deletions(-) diff --git a/docs/telemetry.md b/docs/telemetry.md index 40a547d..eb117c9 100644 --- a/docs/telemetry.md +++ b/docs/telemetry.md @@ -110,8 +110,9 @@ from this package. | Claude Code | In-process hook captures, contract tests, `scripts/telemetry-report.mjs` cross-check, `claude plugin validate --strict` on **2.1.258** (runs on the committed manifest, which has no telemetry hooks; no CLI has validated a manifest with telemetry hooks enabled) | Live user sessions, IDE extensions, desktop Code tab, Cowork delivery and opt-out | | Copilot CLI | In-process captures, fixture JSON, report cross-check, `npm run verify:copilot` on **1.0.88** (runs on the committed manifest, which has no telemetry hooks; checks only MCP servers and skills; no CLI has validated a manifest with telemetry hooks enabled) | Live sessions, Windows PowerShell hook commands, VS Code agent sessions | -Those checks do not prove end-user transmission while `TELEMETRY_ENABLED` is -`false`; they validate the adapters and reporting math for an enabled build. +None of these checks prove end-user transmission while `TELEMETRY_ENABLED` is +`false`. The in-process tests and report cross-check validate the adapters and +reporting math for an enabled build. ## What is stored on your machine @@ -274,7 +275,9 @@ Report field meanings (from `scripts/telemetry-report.mjs`): - `tool_only_sessions`: Copilot sessions (after operator child sessions are attributed to their parent) that are not in `observed_relevant_sessions` and have at least one row whose event is not `Plugin prompt submitted` (reported - outside the session denominator). Mirrors `tool_only_turns` for Claude Code. + outside the session denominator). The Copilot counterpart of `tool_only_turns`. + Child sessions are attributed one level up only, so rows from an operator + started inside another operator are counted under the middle session. - `attempt_observed_outcome_unknown`: Claude turns with a `Plugin tool attempted` on a `server` category (`arcade` or `other_arcade`) but no `Plugin tool called` or `Plugin tool failed` on that same category in the same turn. diff --git a/scripts/telemetry-report.mjs b/scripts/telemetry-report.mjs index 16856b7..915535b 100644 --- a/scripts/telemetry-report.mjs +++ b/scripts/telemetry-report.mjs @@ -62,7 +62,7 @@ const isLegacyRow = (row) => { /** * @param {Record} properties - * @returns {{ host: string, plugin_version: string, telemetry_version: number | "legacy" }} + * @returns {{ host: string, plugin_version: string, telemetry_version: number }} */ const groupKey = (properties) => ({ host: String(properties.host), @@ -413,8 +413,8 @@ const parseJson = (text, where) => { * @returns {unknown[]} */ export const parseExportedEvents = (text) => { - if (!text.trim()) return []; const trimmed = text.trim(); + if (!trimmed) return []; if (trimmed.startsWith("[")) return parseJson(trimmed, "the file"); return text .split("\n") @@ -434,7 +434,7 @@ if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.ur console.log(JSON.stringify(buildReport(events), null, 2)); } catch (error) { // Error text from JSON.parse or the runtime can quote the input, so print only our own message. - const reason = error instanceof ExportParseError ? error.message : "the report could not be built"; + const reason = error instanceof ExportParseError ? error.message : "the file could not be read or the report could not be built"; console.error(`telemetry-report: ${reason}; no report was produced`); process.exit(1); } diff --git a/test/telemetry-docs.test.mjs b/test/telemetry-docs.test.mjs index 71ea502..162366f 100644 --- a/test/telemetry-docs.test.mjs +++ b/test/telemetry-docs.test.mjs @@ -16,10 +16,10 @@ import { ROOT } from "./helpers.mjs"; test("buildTelemetryTables lists every contract event and common property", () => { const tables = buildTelemetryTables(); for (const eventName of Object.keys(EVENTS)) { - assert.ok(tables.includes(eventName), `tables must include event "${eventName}"`); + assert.ok(tables.includes(`| \`${eventName}\` |`), `tables must include a row for event "${eventName}"`); } for (const propName of Object.keys(COMMON_PROPERTIES)) { - assert.ok(tables.includes(propName), `tables must include common property "${propName}"`); + assert.ok(tables.includes(`| \`${propName}\` |`), `tables must include a row for common property "${propName}"`); } assert.match(tables, /Service categories:/); }); diff --git a/test/telemetry-report.test.mjs b/test/telemetry-report.test.mjs index 835bea3..f407781 100644 --- a/test/telemetry-report.test.mjs +++ b/test/telemetry-report.test.mjs @@ -119,7 +119,7 @@ const unknownOutcomeCount = (rows) => { return report.groups[0].stages.attempt_observed_outcome_unknown.count; }; -test("an outcome on one Arcade server category does not settle an attempt on the other", () => { +test("an outcome on one Arcade server category does not count as the outcome for an attempt on the other", () => { const prompt = claudeRow("Plugin prompt submitted", { could_use_arcade: true, service_hints: ["email"], reminder_sent: false }); const attempt = (server) => claudeRow("Plugin tool attempted", { server, tool: "Arcade_UseTool" }); const called = (server) => claudeRow("Plugin tool called", { server, tool: "Arcade_SelectTools" }); @@ -130,11 +130,11 @@ test("an outcome on one Arcade server category does not settle an attempt on the assert.equal(unknownOutcomeCount([prompt, attempt("arcade"), called("arcade")]), 0); assert.equal(unknownOutcomeCount([prompt, attempt("other_arcade"), failed("other_arcade")]), 0); assert.equal(unknownOutcomeCount([prompt, attempt("arcade"), attempt("other_arcade"), called("arcade")]), 1); - // Order within the turn does not matter: an earlier outcome still settles its category. + // Order within the turn does not matter: an earlier outcome still counts for its category. assert.equal(unknownOutcomeCount([prompt, called("arcade"), attempt("arcade")]), 0); assert.equal(unknownOutcomeCount([prompt, failed("other_arcade"), attempt("other_arcade")]), 0); assert.equal(unknownOutcomeCount([prompt, called("other_arcade"), attempt("arcade")]), 1); - // Turn-level only: one outcome on a category settles every attempt on that category. + // Turn-level only: one outcome on a category counts for every attempt on that category. assert.equal(unknownOutcomeCount([prompt, attempt("arcade"), attempt("arcade"), called("arcade")]), 0); }); @@ -230,6 +230,19 @@ test("empty input produces a report with no groups", () => { assert.deepEqual(JSON.parse(result.stdout), report); }); +test("a missing input file exits 1 with one stderr line and no report", () => { + const file = path.join(tempDataDir(), "does-not-exist.jsonl"); + const result = spawnSync(process.execPath, [path.join(ROOT, "scripts/telemetry-report.mjs"), file], { + encoding: "utf8", + }); + assert.equal(result.status, 1); + assert.equal(result.stdout, ""); + assert.equal( + result.stderr, + "telemetry-report: the file could not be read or the report could not be built; no report was produced\n", + ); +}); + test("report keys avoid forbidden metric names and hosts stay separate", () => { const { events, expected } = loadFixture(); const report = buildReport(events); @@ -396,6 +409,7 @@ const copilotCrossCheckExpectation = (group) => { assert.deepEqual(group.denominators, { observed_relevant_sessions: 1 }); assert.equal(group.operator_linked_sessions_excluded, 1); assert.equal(group.multi_prompt_sessions, 0); + assert.equal(group.tool_only_sessions, 0); assert.equal(group.parent_attribution_unknown_sessions, 0); assert.deepEqual(group.stages, { gateway_discovery_or_selection: { count: 1, denominator: "observed_relevant_sessions" }, From fe8c13df03f87a25a8286d024caa852a3c2be8d2 Mon Sep 17 00:00:00 2001 From: Teal Larson Date: Wed, 14 Oct 2026 19:47:58 -0700 Subject: [PATCH 08/10] fix(telemetry-report): accept exports where PostHog dropped its $ properties The report validated exported rows with the sending schema, which requires $ip, $geoip_disable, and $process_person_profile. PostHog can drop these on import (for example when a project discards client IPs), which would make every real row invalid. They are now optional in the report; when present they must still match the contract, so a row with a real IP address is still invalid. Co-Authored-By: Claude Opus 5.5 --- docs/telemetry.md | 5 ++++- scripts/telemetry-report.mjs | 16 +++++++++++++++- test/telemetry-report.test.mjs | 8 ++++++++ 3 files changed, 27 insertions(+), 2 deletions(-) diff --git a/docs/telemetry.md b/docs/telemetry.md index eb117c9..b54f7e1 100644 --- a/docs/telemetry.md +++ b/docs/telemetry.md @@ -220,7 +220,10 @@ The script prints JSON with counts, denominators, and a `limits` list. Before validating, it ignores export fields outside `{ event, distinct_id, properties }` (such as `timestamp` and `uuid`) and PostHog-added properties whose names start with `$` but are not on the contract -for that event. Every other property must match the contract; leaked hook +for that event. The contract's own `$` properties (`$ip`, `$geoip_disable`, +`$process_person_profile`) may be missing, because PostHog can drop them on +import, but when present they must match the contract, so a row with a real IP +address is invalid. Every other property must match the contract; leaked hook fields such as `prompt` or `cwd` make a row invalid. A row with a known event and no `telemetry_version` counts as legacy only if every property key is on the contract for that event; a row that carries any additional property (such diff --git a/scripts/telemetry-report.mjs b/scripts/telemetry-report.mjs index 915535b..5ddc155 100644 --- a/scripts/telemetry-report.mjs +++ b/scripts/telemetry-report.mjs @@ -18,7 +18,21 @@ const MCP_EVENTS = new Set(["Plugin tool attempted", "Plugin tool called", "Plug const OUTCOME_EVENTS = new Set(["Plugin tool called", "Plugin tool failed"]); const APP_ACTION_TOOLS = new Set(["Arcade_UseTool", "app_tool"]); -const validateRow = new Ajv2020({ allErrors: true }).compile(eventSchema()); +/** + * The sending schema, except PostHog's own `$` properties are optional: PostHog can + * strip them on import (for example when the project discards client IPs). When + * present they must still match the contract, so a row with a real IP is invalid. + */ +const exportSchema = () => { + const schema = eventSchema(); + for (const variant of schema.oneOf) { + const properties = variant.properties.properties; + properties.required = properties.required.filter((key) => !key.startsWith("$")); + } + return schema; +}; + +const validateRow = new Ajv2020({ allErrors: true }).compile(exportSchema()); /** @param {unknown} row */ const isValidRow = (row) => validateRow(row) === true; diff --git a/test/telemetry-report.test.mjs b/test/telemetry-report.test.mjs index f407781..fae12f8 100644 --- a/test/telemetry-report.test.mjs +++ b/test/telemetry-report.test.mjs @@ -463,6 +463,14 @@ test("tool_only_sessions attributes child rows to parent before counting", () => assert.equal(report.groups[0].tool_only_sessions, 1); }); +test("PostHog $ properties may be missing from an export but must match when present", () => { + const prompt = claudeRow("Plugin prompt submitted", { could_use_arcade: true, service_hints: [], reminder_sent: false }); + const { $ip, $geoip_disable, $process_person_profile, ...stripped } = prompt.properties; + assert.deepEqual(buildReport([{ ...prompt, properties: stripped }]).excluded, { invalid: 0, legacy: 0 }); + const realIp = { ...prompt, properties: { ...prompt.properties, $ip: "203.0.113.5" } }; + assert.deepEqual(buildReport([realIp]).excluded, { invalid: 1, legacy: 0 }); +}); + test("a legacy row with an extra property counts as invalid, not legacy", () => { const base = { event: "Plugin prompt submitted", From 85f5b29c8cee7f64d8a45cd2e562c78fd9c35ace Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Thu, 15 Oct 2026 19:40:03 +0000 Subject: [PATCH 09/10] docs(telemetry): make the report glossary authoritative and drop duplicated checks Point install guides at the shared CI section, reuse the Claude auth-status fixture, and let legacy detection trust a normalized row. Telemetry stays off. Co-authored-by: Teal Larson --- docs/install/claude-code.md | 13 ++++------ docs/install/copilot.md | 10 ++++---- docs/telemetry.md | 44 ++++++++++------------------------ scripts/telemetry-report.mjs | 20 +++++++--------- test/telemetry-docs.test.mjs | 31 +++++++++++++++++------- test/telemetry-report.test.mjs | 20 +++------------- 6 files changed, 55 insertions(+), 83 deletions(-) diff --git a/docs/install/claude-code.md b/docs/install/claude-code.md index 470d51e..d5be562 100644 --- a/docs/install/claude-code.md +++ b/docs/install/claude-code.md @@ -43,14 +43,11 @@ Telemetry is **off** in this build (`TELEMETRY_ENABLED` is `false`): no usage events are sent and generated manifests include no telemetry hooks. When telemetry is enabled, the adapter in `hooks/telemetry-adapters/claude-code.mjs` classifies prompts, records `PreToolUse` on the plugin and claude.ai Arcade -prefixes, and stores scope under `CLAUDE_PLUGIN_DATA`. See -[telemetry.md](../telemetry.md) for opt-outs (`ARCADE_PLUGIN_TELEMETRY=0`, -`DISABLE_TELEMETRY`, `CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC`) and limits. - -**Repo coverage:** in-process hook tests and `claude plugin validate --strict` -(**2.1.258**) on the committed manifest, which has no telemetry hooks. No CLI -has validated a manifest with telemetry hooks enabled. Not live sessions, IDE -extensions, desktop Code tab, or Cowork. +prefixes, and stores scope under `CLAUDE_PLUGIN_DATA`. Opt out with +`ARCADE_PLUGIN_TELEMETRY=0`, `DISABLE_TELEMETRY`, or +`CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC`. See +[telemetry.md](../telemetry.md) and +[what CI exercises](../telemetry.md#what-ci-exercises). ## First steps diff --git a/docs/install/copilot.md b/docs/install/copilot.md index c5200db..05bf18b 100644 --- a/docs/install/copilot.md +++ b/docs/install/copilot.md @@ -32,13 +32,11 @@ Telemetry is **off** in this build: generated manifests include no telemetry hooks and nothing is sent. When enabled, `hooks/telemetry-adapters/copilot-cli.mjs` records MCP tools as `-`, operator stops with `subagent_session`, and session-scoped prompt state under `COPILOT_PLUGIN_DATA` (no `turn`, no -`PreToolUse`, no built-in CLI/web events). See [telemetry.md](../telemetry.md). +`PreToolUse`, no built-in CLI/web events). See +[telemetry.md](../telemetry.md) for opt-outs (`ARCADE_PLUGIN_TELEMETRY=0` and +`COPILOT_OFFLINE`) and +[what CI exercises](../telemetry.md#what-ci-exercises). -**Repo coverage:** in-process fixtures and `npm run verify:copilot` (**1.0.88**) -on the committed manifest, which has no telemetry hooks; `verify:copilot` checks -only MCP servers and skills. No CLI has validated a manifest with telemetry hooks -enabled. Not live sessions, Windows PowerShell hook commands, or VS Code agent -sessions. Shared `com.github.copilot/hooks/hooks.json` uses `runOnlyIfScriptExists` so VS Code exits quietly without a plugin path. diff --git a/docs/telemetry.md b/docs/telemetry.md index b54f7e1..deccd53 100644 --- a/docs/telemetry.md +++ b/docs/telemetry.md @@ -264,7 +264,8 @@ observed prompt units and sessions visible even when a chart has no app actions. Do not extrapolate rates from a test sample or telemetry-enabled sessions to all users. Do not mix Claude turns with Copilot session counts in one rate. -Report field meanings (from `scripts/telemetry-report.mjs`): +Report field glossary (`scripts/telemetry-report.mjs`). These definitions are +authoritative. The host sections below add attribution and scope only. - `observed_relevant_turns` / `observed_relevant_sessions`: distinct Claude `turn` values or Copilot `session` values with a `Plugin prompt submitted` @@ -299,14 +300,11 @@ Report field meanings (from `scripts/telemetry-report.mjs`): ### Claude Code turns -The denominator is distinct `turn` values with a `Plugin prompt submitted` -event. This is a count of observed relevant turns, not all prompts or tasks. -Exclude tool-only turns: Claude Code filters background task results -from prompt events. Group tool events with the same `turn`, including an -arcade-operator's events, and report sessions and turns separately. A short -follow-up such as “yes, send it” is a separate observed turn while scope is -active, even when its keyword flag is false. Turn counts do not describe whole -tasks. Report tool-only turns separately from this denominator. +Claude Code filters background task results from prompt events. Group tool +events with the same `turn`, including an arcade-operator's events, and report +sessions and turns separately. A short follow-up such as “yes, send it” is a +separate observed turn while scope is active, even when its keyword flag is +false. `server: other_arcade` means the hook observed another connection's Arcade gateway tool. Results recognize exact gateway tool names on arbitrary MCP @@ -316,19 +314,6 @@ attempt event. Direct app tools on an unidentified connection may be classified as `other`. Gateway names are a recognition heuristic, not verified server provenance. -Events contain no tool-call ID. Multiple calls in the same turn cannot be -paired individually, even when their tool categories match. Count observed -events or turn-level stages; do not present a per-attempt completion rate. - -A tool invocation is not always followed by a PostToolUse or -PostToolUseFailure event. A `Plugin tool attempted` event can show the -invocation, but only `Plugin tool called` or `Plugin tool failed` records its -outcome. Count an attempt without either outcome as **attempt -observed, outcome unknown**, not app action success. It does not set -`arcade_used_before`. Label flagged turns with no Arcade tool event **no call -observed**, not **missed**. A direct app tool on another gateway may appear -as `server: other`, without an identifiable Arcade call. - A built-in CLI or web event is a tool observation, not evidence of fallback. Only call it a possible fallback after linking it to a labeled Arcade-eligible task and establishing that it served the same request. @@ -338,17 +323,14 @@ same machine or prove that the gateway or a particular app was unconnected. ### Copilot CLI sessions -Copilot CLI supplies no `prompt_id` or `turn`. Count distinct sessions with a -scoped `Plugin prompt submitted` event as **observed relevant sessions**. Report -multiple-prompt sessions separately. This is not a count of all sessions, user -tasks, or turns. Session starts are not transmitted and cannot define top-level sessions. +Copilot CLI supplies no `prompt_id` or `turn`. Report multiple-prompt sessions +separately. Session starts are not transmitted and cannot define top-level +sessions. An Arcade operator's stop report carries `subagent_session`, equal to that -operator's own event `session`. Exclude known linked operator prompts from the -parent-session denominator and include their tool events with the parent. -Other subagents do not send stop reports, so their prompt sessions cannot be -reliably distinguished from top-level sessions. Report that parent attribution -is unknown rather than labeling every unlinked session as a top-level session. +operator's own event `session`. Include that operator's tool events with the +parent. Other subagents do not send stop reports, so their prompt sessions +cannot be reliably distinguished from top-level sessions. Detached event delivery can change arrival order. A missing operator stop or a rolling-window boundary can also remove a parent link. Do not reconstruct diff --git a/scripts/telemetry-report.mjs b/scripts/telemetry-report.mjs index 5ddc155..3d9a603 100644 --- a/scripts/telemetry-report.mjs +++ b/scripts/telemetry-report.mjs @@ -59,19 +59,15 @@ export const normalizeExportRow = (raw) => { return { event, distinct_id, properties: trimmed }; }; -/** @param {unknown} row */ +/** + * `row` is a successfully normalized export row. A missing `telemetry_version` + * is legacy only when every property key is on the contract; extra keys stay invalid. + * @param {{ event: string, properties: Record }} row + */ const isLegacyRow = (row) => { - if (!row || typeof row !== "object") return false; - const record = /** @type {{ event?: string, properties?: Record }} */ (row); - if ( - typeof record.event !== "string" || - !KNOWN_EVENTS.has(record.event) || - !record.properties || - typeof record.properties !== "object" || - record.properties.telemetry_version !== undefined - ) return false; - const allowed = new Set(allowedProperties(record.event)); - return Object.keys(record.properties).every((key) => allowed.has(key)); + if (row.properties.telemetry_version !== undefined) return false; + const allowed = new Set(allowedProperties(row.event)); + return Object.keys(row.properties).every((key) => allowed.has(key)); }; /** diff --git a/test/telemetry-docs.test.mjs b/test/telemetry-docs.test.mjs index 162366f..405b8a8 100644 --- a/test/telemetry-docs.test.mjs +++ b/test/telemetry-docs.test.mjs @@ -1,8 +1,6 @@ // @ts-check import assert from "node:assert/strict"; -import { readFileSync } from "node:fs"; -import path from "node:path"; import { test } from "node:test"; import { buildTelemetryTables, @@ -11,7 +9,6 @@ import { TELEMETRY_BLOCK_END, } from "../scripts/telemetry-docs.mjs"; import { COMMON_PROPERTIES, EVENTS } from "../hooks/telemetry-contract.mjs"; -import { ROOT } from "./helpers.mjs"; test("buildTelemetryTables lists every contract event and common property", () => { const tables = buildTelemetryTables(); @@ -25,10 +22,26 @@ test("buildTelemetryTables lists every contract event and common property", () = }); test("fillTelemetryTables replaces only the generated block", () => { - const docPath = path.join(ROOT, "docs/telemetry.md"); - const source = readFileSync(docPath, "utf8"); - const filled = fillTelemetryTables(source); - assert.ok(filled.includes(TELEMETRY_BLOCK_BEGIN)); - assert.ok(filled.includes(TELEMETRY_BLOCK_END)); - assert.ok(filled.includes(buildTelemetryTables().split("\n")[0])); + const tables = buildTelemetryTables(); + const source = [ + "before the block", + "", + TELEMETRY_BLOCK_BEGIN, + "stale table row", + TELEMETRY_BLOCK_END, + "", + "after the block", + "", + ].join("\n"); + const expected = [ + "before the block", + "", + TELEMETRY_BLOCK_BEGIN, + tables, + TELEMETRY_BLOCK_END, + "", + "after the block", + "", + ].join("\n"); + assert.equal(fillTelemetryTables(source), expected); }); diff --git a/test/telemetry-report.test.mjs b/test/telemetry-report.test.mjs index fae12f8..927d06b 100644 --- a/test/telemetry-report.test.mjs +++ b/test/telemetry-report.test.mjs @@ -8,6 +8,7 @@ import { test } from "node:test"; import { TELEMETRY_HOSTS } from "../hooks/telemetry-contract.mjs"; import { loadTelemetryAdapter } from "../hooks/telemetry-adapter.mjs"; import { ARCADE_TOOL_PREFIX } from "../hooks/telemetry-adapters/claude-code.mjs"; +import { authStatusResponse } from "./fixtures/telemetry/claude-code/auth-status-response.mjs"; import { buildReport, normalizeExportRow, parseExportedEvents } from "../scripts/telemetry-report.mjs"; import { assertMatchesContract, captureTelemetry, hookInput, tempDataDir } from "./telemetry-helpers.mjs"; import { ROOT } from "./helpers.mjs"; @@ -244,7 +245,7 @@ test("a missing input file exits 1 with one stderr line and no report", () => { }); test("report keys avoid forbidden metric names and hosts stay separate", () => { - const { events, expected } = loadFixture(); + const { events } = loadFixture(); const report = buildReport(events); for (const key of collectKeys(report)) { assert.doesNotMatch(key, FORBIDDEN_KEY, key); @@ -254,26 +255,11 @@ test("report keys avoid forbidden metric names and hosts stay separate", () => { [["claude-code", "0.2.0"], ["claude-code", "0.3.0"], ["copilot-cli", "0.2.0"]], ); const byGroup = Object.fromEntries(report.groups.map((group) => [`${group.host}/${group.plugin_version}`, group])); - const expectedByGroup = Object.fromEntries(expected.groups.map((group) => [`${group.host}/${group.plugin_version}`, group])); assert.equal(byGroup["claude-code/0.2.0"].unit, "turn"); + assert.equal(byGroup["claude-code/0.3.0"].unit, "turn"); assert.equal(byGroup["copilot-cli/0.2.0"].unit, "session"); - assert.deepEqual(byGroup["claude-code/0.2.0"].stages, expectedByGroup["claude-code/0.2.0"].stages); - assert.deepEqual(byGroup["copilot-cli/0.2.0"].stages, expectedByGroup["copilot-cli/0.2.0"].stages); - assert.equal(byGroup["copilot-cli/0.2.0"].stages.attempt_observed_outcome_unknown, undefined); - assert.ok(byGroup["claude-code/0.2.0"].stages.attempt_observed_outcome_unknown); - assert.notDeepEqual(byGroup["claude-code/0.2.0"].stages, byGroup["copilot-cli/0.2.0"].stages); }); -const authStatusResponse = (statuses) => [ - { - type: "text", - text: JSON.stringify({ - message: statuses.includes("authorization_required") ? "Not yet authorized." : "All authorized.", - providers: statuses.map((status, index) => ({ provider: `provider${index}`, status })), - }), - }, -]; - const COPILOT_SESSION = "dd3beb80-4471-4513-99a4-a57d3d7c08df"; const COPILOT_SUBAGENT = "57946be7-1a73-40ca-a042-abb0f68d9445"; const COPILOT_OPERATOR = "arcade:arcade-operator"; From e8153a82a3f5609f94040cb80710f6f1de353244 Mon Sep 17 00:00:00 2001 From: Teal Larson Date: Thu, 15 Oct 2026 15:49:14 -0700 Subject: [PATCH 10/10] refactor(telemetry): remove offline export report --- AGENTS.md | 8 +- ARCHITECTURE.md | 6 +- README.md | 4 +- docs/telemetry.md | 186 +----- scripts/telemetry-report.mjs | 451 -------------- test/fixtures/telemetry-report/events.jsonl | 27 - .../telemetry-report/expected-report.json | 156 ----- test/telemetry-report.test.mjs | 548 ------------------ 8 files changed, 35 insertions(+), 1351 deletions(-) delete mode 100644 scripts/telemetry-report.mjs delete mode 100644 test/fixtures/telemetry-report/events.jsonl delete mode 100644 test/fixtures/telemetry-report/expected-report.json delete mode 100644 test/telemetry-report.test.mjs diff --git a/AGENTS.md b/AGENTS.md index c565407..5ed3a42 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -22,10 +22,10 @@ come from `hooks/telemetry-contract.mjs`; never edit inside those markers). that source). `telemetry.mjs` prints nothing; only `telemetry-send.mjs` may touch the network (a test enforces both). `TELEMETRY_ENABLED` in `hooks/telemetry-config.mjs` stays `false` without separate approval; while - false, `npm run generate` writes no telemetry hooks. Reporting and maintained - docs for exports live in `scripts/telemetry-report.mjs` and - `docs/telemetry.md`. `test/telemetry-report.test.mjs` cross-checks every - adapter named by the contract and fails if that adapter file is missing. + false, `npm run generate` writes no telemetry hooks. `docs/telemetry.md` + documents the event contract, privacy, opt-outs, and approval before enabling. + `scripts/telemetry-docs.mjs` generates its event tables from the contract; + keep its tests in `test/telemetry-docs.test.mjs`. - Codex hooks are blocked upstream ([docs/install/codex.md](docs/install/codex.md)). Don't remove the root `$schema` to force them; that breaks Agent Plugins conformance. diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 43b357f..adacd0d 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -92,6 +92,6 @@ Optional client hooks in `hooks/telemetry-adapters/` can send supplemental, scoped usage events when `TELEMETRY_ENABLED` is `true` and the user has not opted out. **This build keeps `TELEMETRY_ENABLED` false**, so no telemetry hooks are generated and nothing is sent. See [docs/telemetry.md](docs/telemetry.md) -for the contract, opt-outs, local state, and `scripts/telemetry-report.mjs` -for aggregating exports. Gateway telemetry remains authoritative; plugin events -do not establish task success. +for the contract, opt-outs, local state, and the separate approval required +before collection can be enabled. Gateway telemetry remains authoritative; +plugin events do not establish task success. diff --git a/README.md b/README.md index 41f5a8b..d8cbbe7 100644 --- a/README.md +++ b/README.md @@ -74,8 +74,8 @@ anything is sent, created, or deleted. Claude Code and Copilot CLI include telemetry adapters, but **`TELEMETRY_ENABLED` is `false`**: this package sends no usage events and generates no telemetry hooks in client manifests. [docs/telemetry.md](docs/telemetry.md) describes -the event contract, what would be stored locally when enabled, opt-outs, and how -to aggregate exports with `scripts/telemetry-report.mjs`. The Arcade MCP +the event contract, what would be stored locally when enabled, opt-outs, and +the separate approval required before collection can be enabled. The Arcade MCP gateway remains the canonical source for request and tool-call telemetry. Adapter behavior is covered by in-process tests; see [docs/telemetry.md](docs/telemetry.md) for what CI does and does not exercise. diff --git a/docs/telemetry.md b/docs/telemetry.md index deccd53..ddfba68 100644 --- a/docs/telemetry.md +++ b/docs/telemetry.md @@ -7,9 +7,8 @@ collection is separately approved after ingestion, opt-outs, privacy, and distribution requirements are verified. The sections below describe what **would** be sent when telemetry is turned on, -how to read exported events, and how that relates to gateway telemetry and -offline routing evaluation. Nothing here implies that this package currently -transmits data. +its privacy and opt-out controls, and the approval required before enabling it. +Nothing here implies that this package currently transmits data. When enabled, Claude Code and Copilot CLI hooks would inspect prompts locally in each session to recognize app-related work. Relevance state belongs to that @@ -107,12 +106,11 @@ from this package. | Adapter | Tested in this repo | Not covered here | | --- | --- | --- | -| Claude Code | In-process hook captures, contract tests, `scripts/telemetry-report.mjs` cross-check, `claude plugin validate --strict` on **2.1.258** (runs on the committed manifest, which has no telemetry hooks; no CLI has validated a manifest with telemetry hooks enabled) | Live user sessions, IDE extensions, desktop Code tab, Cowork delivery and opt-out | -| Copilot CLI | In-process captures, fixture JSON, report cross-check, `npm run verify:copilot` on **1.0.88** (runs on the committed manifest, which has no telemetry hooks; checks only MCP servers and skills; no CLI has validated a manifest with telemetry hooks enabled) | Live sessions, Windows PowerShell hook commands, VS Code agent sessions | +| Claude Code | In-process hook captures, contract tests, `claude plugin validate --strict` on **2.1.258** (runs on the committed manifest, which has no telemetry hooks; no CLI has validated a manifest with telemetry hooks enabled) | Live user sessions, IDE extensions, desktop Code tab, Cowork delivery and opt-out | +| Copilot CLI | In-process captures, fixture JSON, contract tests, `npm run verify:copilot` on **1.0.88** (runs on the committed manifest, which has no telemetry hooks; checks only MCP servers and skills; no CLI has validated a manifest with telemetry hooks enabled) | Live sessions, Windows PowerShell hook commands, VS Code agent sessions | None of these checks prove end-user transmission while `TELEMETRY_ENABLED` is -`false`. The in-process tests and report cross-check validate the adapters and -reporting math for an enabled build. +`false`. The in-process tests exercise adapter behavior for an enabled build. ## What is stored on your machine @@ -179,9 +177,8 @@ picked from are never sent. During an app-related observation period, a Bash command sends an event only when one of its commands starts with a listed program; commands that call a program by its full path, such as `/opt/homebrew/bin/gh`, are not counted. A command that runs -two listed programs, such as `gh … && curl …`, sends one event for each, so -count CLI use by turn, not by event. Claude Code starts these hooks only for -commands that match (the hook `if` field, Claude Code 2.1.246 and later). +two listed programs, such as `gh … && curl …`, sends one event for each. +Claude Code starts these hooks only for commands that match (the hook `if` field, Claude Code 2.1.246 and later). ## Never sent @@ -207,159 +204,28 @@ completion telemetry. Plugin events are supplemental observations from client hooks. They do not replace gateway records and cannot prove task success on their own. -## Reading the numbers +## Observation limits -Export PostHog rows with the fields `event`, `distinct_id`, `properties`, and -optionally `timestamp`, then aggregate them locally: +Plugin events describe what hooks observed. They do not establish whether Arcade +was needed or whether a user's task succeeded. `could_use_arcade` and +`service_hints` are local keyword classifications, and unrelated prompts are +excluded. Neither host sends a task ID. -```bash -node scripts/telemetry-report.mjs path/to/export.jsonl -``` +Claude Code supplies a hashed `turn`; Copilot CLI supplies only a hashed +`session`, which can contain multiple prompts. Detached delivery can change +event arrival order. Tool attempts, completions, and failures are separate +observations; timestamps alone cannot pair calls or establish a task outcome. -The script prints JSON with counts, denominators, and a `limits` list. Before -validating, it ignores export fields outside -`{ event, distinct_id, properties }` (such as `timestamp` and `uuid`) and -PostHog-added properties whose names start with `$` but are not on the contract -for that event. The contract's own `$` properties (`$ip`, `$geoip_disable`, -`$process_person_profile`) may be missing, because PostHog can drop them on -import, but when present they must match the contract, so a row with a real IP -address is invalid. Every other property must match the contract; leaked hook -fields such as `prompt` or `cwd` make a row invalid. A row with a known event -and no `telemetry_version` counts as legacy only if every property key is on -the contract for that event; a row that carries any additional property (such -as `prompt` or `cwd`) counts as invalid instead. It counts invalid rows and -legacy rows separately, and excludes both from grouped counts. Groups are split -by `host`, `plugin_version`, and `telemetry_version`. Claude Code and Copilot -CLI are never combined into one denominator. For JSONL input, a line that is -not valid JSON causes the script to exit nonzero with a message naming that -line number; for JSON array input that fails to parse, the message says the -file is not valid JSON without naming a line. Either way the script prints no -report and does not echo the file's contents. - -These events measure what plugin hooks observed, not whether Arcade was needed -or whether the user's task succeeded. `could_use_arcade` and `service_hints` -come from a local keyword classifier. A flagged prompt is a candidate for -review, not a confirmed opportunity. Prompt events are selected by relevance, -so they cannot measure the share of all prompts needing Arcade; unrelated -prompts are deliberately absent. Use labeled task evaluations to score routing, and compare the -plugin with a baseline or variant before attributing a change to it. -Neither host emits a task ID. - -Keep the observation stages separate: - -| Stage | Evidence | What it establishes | -| --- | --- | --- | -| Tool attempt (Claude Code) | `Plugin tool attempted` on PreToolUse | The model invoked a tool. An attempt alone has no observed outcome. | -| Gateway discovery or selection | `Plugin tool called` or `Plugin tool failed` for `Arcade_ListApps` or `Arcade_SelectTools` | The discovery or selection call completed or failed; a successful selection is not an app action. | -| Authorization check | `System_ManageAuthorization`, reported separately | `auth_needed: true` means its answer said sign-in was needed. A check alone says nothing about app use; `false` does not prove every app is connected. | -| App action | `Arcade_UseTool` or `app_tool`, split by `Plugin tool called` and `Plugin tool failed` | The hook observed a tool completion or failure. Tools with an unrecognized service category are `other` and cannot be assigned to this stage. Neither outcome proves the user's task succeeded. | -| No Arcade call observed | A prompt with no Arcade tool event in the observable group | The hooks saw no call. This is not a routing miss without an independently labeled need and complete tool visibility. | - -Count each unit once at each stage, and show the denominator, host, date range, -plugin version, telemetry version, and observation coverage beside every rate. -`telemetry_version: 2` identifies scoped events; a missing value identifies the -legacy contract. Keep those populations separate, even at the same plugin -version. Keep the number of -observed prompt units and sessions visible even when a chart has no app actions. -Do not extrapolate rates from a test sample or telemetry-enabled sessions to all users. -Do not mix Claude turns with Copilot session counts in one rate. - -Report field glossary (`scripts/telemetry-report.mjs`). These definitions are -authoritative. The host sections below add attribution and scope only. - -- `observed_relevant_turns` / `observed_relevant_sessions`: distinct Claude - `turn` values or Copilot `session` values with a `Plugin prompt submitted` - event, after Copilot operator-linked child sessions are excluded from the - session denominator. -- `operator_linked_sessions_excluded`: Copilot prompt sessions left out of - that session denominator because an operator stop named them as - `subagent_session`. A stop whose child sent no prompt does not increase it. -- `tool_only_turns`: Claude turns with tool events but no prompt event in that - turn (reported outside the turn denominator). -- `tool_only_sessions`: Copilot sessions (after operator child sessions are - attributed to their parent) that are not in `observed_relevant_sessions` and - have at least one row whose event is not `Plugin prompt submitted` (reported - outside the session denominator). The Copilot counterpart of `tool_only_turns`. - Child sessions are attributed one level up only, so rows from an operator - started inside another operator are counted under the middle session. -- `attempt_observed_outcome_unknown`: Claude turns with a `Plugin tool - attempted` on a `server` category (`arcade` or `other_arcade`) but no `Plugin - tool called` or `Plugin tool failed` on that same category in the same turn. - An outcome on one category never counts as the outcome for an attempt on the other. - `other_arcade` groups every other Arcade connection together, and one outcome - counts as the outcome for every attempt on its category in the turn, so the - check cannot pair a result with a particular connection or call. -- `no_call_observed`: relevant turns or sessions with no Arcade MCP tool event - (`Plugin tool attempted`, `Plugin tool called`, or `Plugin tool failed` on - `server: arcade` or `other_arcade`). -- `parent_attribution_unknown_sessions`: Copilot sessions with a prompt event - that are not operator-linked children and have no operator stop on that - session to confirm parent/child linkage. -- Stage counts use the denominators above; they are not paired attempt/result - metrics and do not imply recall, precision, or task success. - -### Claude Code turns - -Claude Code filters background task results from prompt events. Group tool -events with the same `turn`, including an arcade-operator's events, and report -sessions and turns separately. A short follow-up such as “yes, send it” is a -separate observed turn while scope is active, even when its keyword flag is -false. - -`server: other_arcade` means the hook observed another connection's Arcade -gateway tool. Results recognize exact gateway tool names on arbitrary MCP -server aliases; attempts recognize only the two configured lowercase Arcade -prefixes. UUID or capitalized aliases can therefore have results without an -attempt event. Direct app tools on an unidentified connection may be classified -as `other`. Gateway names are a recognition heuristic, not verified server -provenance. - -A built-in CLI or web event is a tool observation, not evidence of fallback. -Only call it a possible fallback after linking it to a labeled Arcade-eligible -task and establishing that it served the same request. -`arcade_used_before: false` says no Arcade call has previously succeeded for -that client plugin installation; it does not describe another client on the -same machine or prove that the gateway or a particular app was unconnected. - -### Copilot CLI sessions - -Copilot CLI supplies no `prompt_id` or `turn`. Report multiple-prompt sessions -separately. Session starts are not transmitted and cannot define top-level -sessions. - -An Arcade operator's stop report carries `subagent_session`, equal to that -operator's own event `session`. Include that operator's tool events with the -parent. Other subagents do not send stop reports, so their prompt sessions -cannot be reliably distinguished from top-level sessions. - -Detached event delivery can change arrival order. A missing operator stop or a -rolling-window boundary can also remove a parent link. Do not reconstruct -per-prompt outcomes from timestamps alone or link resumed tasks across sessions. - -Copilot names MCP tools `-` with no plugin prefix, so an MCP -server named `arcade` counts as `server: arcade` even if this plugin did not -install it. Copilot sends no built-in CLI or web events; its `arcade-operator` -can report `status: unknown`. Do not infer a fallback or task result from -either absence. - -### Failure and outcome limits - -`failure_kind` is a local classifier of error text, not a server error code. -An app sign-in failure can be `tool_error` when its wording does not match the -classifier; bad input, rate limits, and upstream errors can also get that -value. - -`auth_needed: true` on an authorization check reports a service still needing -sign-in; operator `status: needs_auth` is the model's report, not a verified -server status. `auth_required`, `session_expired`, `timeout`, and `unreachable` -match known client or MCP SDK error wording; other wording can classify -differently. Copilot does not report interrupted calls, so its `interrupted` -category is empty. - -Operator status describes its report, not the parent task's result. The hooks -do not send final answers or satisfaction signals. Task success, true routing -misses, and improvement from this plugin require labeled evaluations outside -this telemetry. +MCP server and gateway tool names are recognition heuristics, not verified +connection provenance. Copilot's server names do not identify which plugin +installed them. Operator stop events can link a child session to its parent; +missing stops and other subagents leave that attribution incomplete. Session +hashes do not link activity across sessions. + +`failure_kind` is classified locally from error text, and `auth_needed` reflects +the authorization tool's response. An operator's `status` is the model's report +of its own work. These values are not verified server status or task-success +signals. Routing quality requires labeled evaluations outside this telemetry. ## Routing evaluation diff --git a/scripts/telemetry-report.mjs b/scripts/telemetry-report.mjs deleted file mode 100644 index 3d9a603..0000000 --- a/scripts/telemetry-report.mjs +++ /dev/null @@ -1,451 +0,0 @@ -// @ts-check -/** Aggregate exported PostHog plugin telemetry rows into count-only reports. */ - -import { readFileSync } from "node:fs"; -import { resolve } from "node:path"; -import { fileURLToPath } from "node:url"; -import Ajv2020Module from "ajv/dist/2020.js"; -import { allowedProperties, EVENTS, eventSchema } from "../hooks/telemetry-contract.mjs"; - -const Ajv2020 = /** @type {new (options?: object) => import("ajv").default} */ ( - /** @type {any} */ (Ajv2020Module).default ?? Ajv2020Module -); - -const KNOWN_EVENTS = new Set(Object.keys(EVENTS)); - -const DISCOVERY_TOOLS = new Set(["Arcade_ListApps", "Arcade_SelectTools"]); -const MCP_EVENTS = new Set(["Plugin tool attempted", "Plugin tool called", "Plugin tool failed"]); -const OUTCOME_EVENTS = new Set(["Plugin tool called", "Plugin tool failed"]); -const APP_ACTION_TOOLS = new Set(["Arcade_UseTool", "app_tool"]); - -/** - * The sending schema, except PostHog's own `$` properties are optional: PostHog can - * strip them on import (for example when the project discards client IPs). When - * present they must still match the contract, so a row with a real IP is invalid. - */ -const exportSchema = () => { - const schema = eventSchema(); - for (const variant of schema.oneOf) { - const properties = variant.properties.properties; - properties.required = properties.required.filter((key) => !key.startsWith("$")); - } - return schema; -}; - -const validateRow = new Ajv2020({ allErrors: true }).compile(exportSchema()); - -/** @param {unknown} row */ -const isValidRow = (row) => validateRow(row) === true; - -/** - * Drops export fields outside event, distinct_id, and properties, and PostHog `$` properties not on the contract. - * @param {unknown} raw - * @returns {{ event: string, distinct_id: string, properties: Record } | null} - */ -export const normalizeExportRow = (raw) => { - if (!raw || typeof raw !== "object") return null; - const record = /** @type {Record} */ (raw); - const { event, distinct_id, properties } = record; - if (typeof event !== "string" || typeof distinct_id !== "string") return null; - if (!properties || typeof properties !== "object" || Array.isArray(properties)) return null; - if (!KNOWN_EVENTS.has(event)) return null; - const props = /** @type {Record} */ (properties); - const allowed = new Set(allowedProperties(event)); - /** @type {Record} */ - const trimmed = { ...props }; - for (const key of Object.keys(trimmed)) { - if (key.startsWith("$") && !allowed.has(key)) delete trimmed[key]; - } - return { event, distinct_id, properties: trimmed }; -}; - -/** - * `row` is a successfully normalized export row. A missing `telemetry_version` - * is legacy only when every property key is on the contract; extra keys stay invalid. - * @param {{ event: string, properties: Record }} row - */ -const isLegacyRow = (row) => { - if (row.properties.telemetry_version !== undefined) return false; - const allowed = new Set(allowedProperties(row.event)); - return Object.keys(row.properties).every((key) => allowed.has(key)); -}; - -/** - * @param {Record} properties - * @returns {{ host: string, plugin_version: string, telemetry_version: number }} - */ -const groupKey = (properties) => ({ - host: String(properties.host), - plugin_version: String(properties.plugin_version), - telemetry_version: Number(properties.telemetry_version), -}); - -/** @param {string} server */ -const isArcadeServer = (server) => server === "arcade" || server === "other_arcade"; - -/** - * @param {{ event: string, properties: Record }} row - */ -const isArcadeMcpRow = (row) => - MCP_EVENTS.has(row.event) && isArcadeServer(String(row.properties.server ?? "")); - -/** - * @param {{ event: string, properties: Record }} row - */ -const toolName = (row) => (typeof row.properties.tool === "string" ? row.properties.tool : ""); - -/** - * @param {{ event: string, properties: Record }} row - */ -const isDiscoveryRow = (row) => - OUTCOME_EVENTS.has(row.event) && DISCOVERY_TOOLS.has(toolName(row)); - -/** - * @param {{ event: string, properties: Record }} row - */ -const isAuthRow = (row) => - OUTCOME_EVENTS.has(row.event) && toolName(row) === "System_ManageAuthorization"; - -/** - * @param {{ event: string, properties: Record }} row - */ -const isAppActionCalledRow = (row) => - row.event === "Plugin tool called" && APP_ACTION_TOOLS.has(toolName(row)); - -/** - * @param {{ event: string, properties: Record }} row - */ -const isAppActionFailedRow = (row) => - row.event === "Plugin tool failed" && APP_ACTION_TOOLS.has(toolName(row)); - -/** @param {{ event: string, properties: Record }[]} rows */ -const gatewayRow = (rows) => rows.some(isDiscoveryRow); - -/** - * @param {{ event: string, properties: Record }[]} rows - * @param {boolean} needed - */ -const authRow = (rows, needed) => - rows.some( - (row) => isAuthRow(row) && row.properties.auth_needed === needed, - ); - -/** - * An outcome only counts as the outcome for attempts on its own `server` category, so an - * `other_arcade` result cannot hide an `arcade` attempt. Within a category the check is turn-level. - * @param {{ event: string, properties: Record }[]} arcadeRows - */ -const attemptWithoutOutcome = (arcadeRows) => - ["arcade", "other_arcade"].some((server) => { - const onServer = arcadeRows.filter((row) => row.properties.server === server); - const hasAttempt = onServer.some((row) => row.event === "Plugin tool attempted"); - const hasOutcome = onServer.some((row) => OUTCOME_EVENTS.has(row.event)); - return hasAttempt && !hasOutcome; - }); - -/** - * The six stages both hosts share. Each entry is one Claude turn or one Copilot session. - * @param {Iterable<{ event: string, properties: Record }[]>} units - */ -const countSharedStages = (units) => { - let gateway = 0; - let authNeededTrue = 0; - let authNeededFalse = 0; - let appCalled = 0; - let appFailed = 0; - let noCallObserved = 0; - - for (const rows of units) { - const arcadeRows = rows.filter(isArcadeMcpRow); - if (gatewayRow(rows)) gateway += 1; - if (authRow(rows, true)) authNeededTrue += 1; - if (authRow(rows, false)) authNeededFalse += 1; - if (rows.some(isAppActionCalledRow)) appCalled += 1; - if (rows.some(isAppActionFailedRow)) appFailed += 1; - if (arcadeRows.length === 0) noCallObserved += 1; - } - - return { gateway, authNeededTrue, authNeededFalse, appCalled, appFailed, noCallObserved }; -}; - -/** - * @param {ReturnType} counts - * @param {string} denominator - * @param {Record} [beforeNoCall] - * Claude inserts attempt_observed_outcome_unknown here. Copilot omits it. - */ -const sharedStageReport = (counts, denominator, beforeNoCall = {}) => ({ - gateway_discovery_or_selection: { count: counts.gateway, denominator }, - authorization_check_auth_needed_true: { count: counts.authNeededTrue, denominator }, - authorization_check_auth_needed_false: { count: counts.authNeededFalse, denominator }, - app_action_called: { count: counts.appCalled, denominator }, - app_action_failed: { count: counts.appFailed, denominator }, - ...beforeNoCall, - no_call_observed: { count: counts.noCallObserved, denominator }, -}); - -/** - * @param {Set} turnIds - * @param {Map }[]>} byTurn - */ -const countTurnStages = (turnIds, byTurn) => { - const denominator = "observed_relevant_turns"; - const units = [...turnIds].map((turn) => byTurn.get(turn) ?? []); - let attemptOutcomeUnknown = 0; - for (const rows of units) { - if (attemptWithoutOutcome(rows.filter(isArcadeMcpRow))) attemptOutcomeUnknown += 1; - } - return sharedStageReport(countSharedStages(units), denominator, { - attempt_observed_outcome_unknown: { count: attemptOutcomeUnknown, denominator }, - }); -}; - -/** - * @param {{ event: string, properties: Record }[]} rows - * @param {string} eventName - */ -const builtinCounts = (rows, eventName) => { - /** @type {Record>} */ - const out = {}; - for (const row of rows) { - if (row.event !== eventName) continue; - const tool = String(row.properties.tool ?? "unknown"); - if (!out[tool]) out[tool] = {}; - const cli = row.properties.cli; - const key = typeof cli === "string" ? cli : "_"; - out[tool][key] = (out[tool][key] ?? 0) + 1; - } - return out; -}; - -/** - * @param {{ event: string, properties: Record }[]} rows - */ -const failureKindCounts = (rows) => { - /** @type {Record} */ - const out = {}; - for (const row of rows) { - if (row.event !== "Plugin tool failed") continue; - const kind = String(row.properties.failure_kind ?? "unknown"); - out[kind] = (out[kind] ?? 0) + 1; - } - return out; -}; - -/** - * @param {{ event: string, properties: Record }[]} rows - */ -const operatorStatusCounts = (rows) => { - /** @type {Record} */ - const out = {}; - for (const row of rows) { - if (row.event !== "Plugin subagent stopped") continue; - const status = String(row.properties.status ?? "unknown"); - out[status] = (out[status] ?? 0) + 1; - } - return out; -}; - -/** - * @param {{ event: string, properties: Record }[]} rows - */ -const buildClaudeGroup = (rows) => { - const prompts = rows.filter((row) => row.event === "Plugin prompt submitted"); - const relevantTurns = new Set( - prompts.map((row) => String(row.properties.turn ?? "")).filter(Boolean), - ); - - /** @type {Map }[]>} */ - const byTurn = new Map(); - for (const row of rows) { - const turn = row.properties.turn; - if (typeof turn !== "string" || !turn) continue; - const list = byTurn.get(turn) ?? []; - list.push(row); - byTurn.set(turn, list); - } - - const toolOnlyTurns = [...byTurn.keys()].filter( - (turn) => !relevantTurns.has(turn), - ).length; - - return { - unit: "turn", - denominators: { observed_relevant_turns: relevantTurns.size }, - tool_only_turns: toolOnlyTurns, - stages: countTurnStages(relevantTurns, byTurn), - builtin_tools_called: builtinCounts(rows, "Plugin built-in tool called"), - builtin_tools_failed: builtinCounts(rows, "Plugin built-in tool failed"), - failure_kinds: failureKindCounts(rows), - operator_stop_status: operatorStatusCounts(rows), - }; -}; - -/** - * @param {{ event: string, properties: Record }[]} rows - */ -const buildCopilotGroup = (rows) => { - const prompts = rows.filter((row) => row.event === "Plugin prompt submitted"); - const promptSessions = new Set(prompts.map((row) => String(row.properties.session))); - - /** @type {Map} */ - const childToParent = new Map(); - for (const row of rows) { - if (row.event !== "Plugin subagent stopped") continue; - const parent = String(row.properties.session); - const child = row.properties.subagent_session; - if (typeof child === "string" && child) { - childToParent.set(child, parent); - } - } - - const denominatorSessions = new Set( - [...promptSessions].filter((session) => !childToParent.has(session)), - ); - // Only prompt sessions actually removed from the denominator. A stop whose - // child never sent a prompt does not count. - const operatorLinkedSessionsExcluded = promptSessions.size - denominatorSessions.size; - - const multiPromptSessions = [...denominatorSessions].filter((session) => { - const count = prompts.filter((row) => String(row.properties.session) === session).length; - return count > 1; - }).length; - - const parentSessionsWithStop = new Set( - rows - .filter((row) => row.event === "Plugin subagent stopped" && row.properties.subagent_session) - .map((row) => String(row.properties.session)), - ); - const parentAttributionUnknown = [...denominatorSessions].filter( - (session) => !parentSessionsWithStop.has(session), - ).length; - - /** @type {Map }[]>} */ - const bySession = new Map(); - for (const row of rows) { - const session = String(row.properties.session); - const key = childToParent.get(session) ?? session; - const list = bySession.get(key) ?? []; - list.push(row); - bySession.set(key, list); - } - - const sessionUnits = [...denominatorSessions].map((session) => bySession.get(session) ?? []); - - const toolOnlySessions = [...bySession.keys()].filter( - (session) => - !denominatorSessions.has(session) && - (bySession.get(session) ?? []).some((row) => row.event !== "Plugin prompt submitted"), - ).length; - - return { - unit: "session", - denominators: { observed_relevant_sessions: denominatorSessions.size }, - operator_linked_sessions_excluded: operatorLinkedSessionsExcluded, - multi_prompt_sessions: multiPromptSessions, - tool_only_sessions: toolOnlySessions, - parent_attribution_unknown_sessions: parentAttributionUnknown, - stages: sharedStageReport(countSharedStages(sessionUnits), "observed_relevant_sessions"), - failure_kinds: failureKindCounts(rows), - operator_stop_status: operatorStatusCounts(rows), - }; -}; - -const REPORT_LIMITS = [ - "Counts describe plugin hook observations only, not task outcomes or whether Arcade was needed.", - "Prompt events are limited to locally classified app-related work; unrelated prompts are absent.", - "Claude Code turn counts and Copilot CLI session counts must not be combined into one rate.", - "Copilot CLI does not emit tool-attempt events; attempt stages apply only to Claude Code.", - "Multiple tool calls in one turn or session cannot be paired without a tool-call ID.", - "Attempt outcomes are checked per server category; other_arcade groups every other Arcade connection together, so an outcome there cannot be tied to the connection or call that was attempted.", - "The Arcade MCP gateway remains the canonical source for request, auth, discovery, and tool-call telemetry.", -]; - -/** - * @param {unknown[]} events - */ -export const buildReport = (events) => { - let invalid = 0; - let legacy = 0; - /** @type {Map }[]>} */ - const groups = new Map(); - - for (const raw of events) { - const row = normalizeExportRow(raw); - if (row === null) { - invalid += 1; - continue; - } - if (isLegacyRow(row)) { - legacy += 1; - continue; - } - if (!isValidRow(row)) { - invalid += 1; - continue; - } - const key = JSON.stringify(groupKey(row.properties)); - const list = groups.get(key) ?? []; - list.push(row); - groups.set(key, list); - } - - /** @type {object[]} */ - const reportGroups = []; - for (const [key, rows] of [...groups.entries()].sort(([a], [b]) => a.localeCompare(b))) { - const meta = groupKey(rows[0].properties); - const body = - meta.host === "claude-code" ? buildClaudeGroup(rows) : buildCopilotGroup(rows); - reportGroups.push({ ...meta, ...body }); - } - - return { - excluded: { invalid, legacy }, - groups: reportGroups, - limits: REPORT_LIMITS, - }; -}; - -/** Thrown for unparseable input; the message never contains the input text. */ -export class ExportParseError extends Error {} - -/** @param {string} text @param {string} where */ -const parseJson = (text, where) => { - try { - return JSON.parse(text); - } catch { - throw new ExportParseError(`${where} is not valid JSON`); - } -}; - -/** - * @param {string} text - * @returns {unknown[]} - */ -export const parseExportedEvents = (text) => { - const trimmed = text.trim(); - if (!trimmed) return []; - if (trimmed.startsWith("[")) return parseJson(trimmed, "the file"); - return text - .split("\n") - .map((line, index) => ({ line: line.trim(), number: index + 1 })) - .filter(({ line }) => line) - .map(({ line, number }) => parseJson(line, `line ${number}`)); -}; - -if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { - const file = process.argv[2]; - if (!file) { - console.error("usage: node scripts/telemetry-report.mjs "); - process.exit(1); - } - try { - const events = parseExportedEvents(readFileSync(file, "utf8")); - console.log(JSON.stringify(buildReport(events), null, 2)); - } catch (error) { - // Error text from JSON.parse or the runtime can quote the input, so print only our own message. - const reason = error instanceof ExportParseError ? error.message : "the file could not be read or the report could not be built"; - console.error(`telemetry-report: ${reason}; no report was produced`); - process.exit(1); - } -} diff --git a/test/fixtures/telemetry-report/events.jsonl b/test/fixtures/telemetry-report/events.jsonl deleted file mode 100644 index 7237168..0000000 --- a/test/fixtures/telemetry-report/events.jsonl +++ /dev/null @@ -1,27 +0,0 @@ -{"event":"Plugin prompt submitted","distinct_id":"aaaaaaaaaaaaaaaa","properties":{"session":"aaaaaaaaaaaaaaaa","turn":"1111111111111111","host":"claude-code","plugin_version":"0.2.0","telemetry_version":2,"os":"linux","arcade_used_before":false,"$process_person_profile":false,"$geoip_disable":true,"$ip":"0.0.0.0","could_use_arcade":true,"service_hints":["email"],"reminder_sent":true}} -{"event":"Plugin tool called","distinct_id":"aaaaaaaaaaaaaaaa","properties":{"session":"aaaaaaaaaaaaaaaa","turn":"1111111111111111","host":"claude-code","plugin_version":"0.2.0","telemetry_version":2,"os":"linux","arcade_used_before":false,"$process_person_profile":false,"$geoip_disable":true,"$ip":"0.0.0.0","server":"arcade","tool":"Arcade_SelectTools"}} -{"event":"Plugin tool called","distinct_id":"aaaaaaaaaaaaaaaa","properties":{"session":"aaaaaaaaaaaaaaaa","turn":"1111111111111111","host":"claude-code","plugin_version":"0.2.0","telemetry_version":2,"os":"linux","arcade_used_before":false,"$process_person_profile":false,"$geoip_disable":true,"$ip":"0.0.0.0","server":"arcade","tool":"System_ManageAuthorization","auth_needed":false}} -{"event":"Plugin tool called","distinct_id":"aaaaaaaaaaaaaaaa","properties":{"session":"aaaaaaaaaaaaaaaa","turn":"1111111111111111","host":"claude-code","plugin_version":"0.2.0","telemetry_version":2,"os":"linux","arcade_used_before":true,"$process_person_profile":false,"$geoip_disable":true,"$ip":"0.0.0.0","server":"arcade","tool":"app_tool","service":"email"}} -{"event":"Plugin built-in tool called","distinct_id":"aaaaaaaaaaaaaaaa","properties":{"session":"aaaaaaaaaaaaaaaa","turn":"1111111111111111","host":"claude-code","plugin_version":"0.2.0","telemetry_version":2,"os":"linux","arcade_used_before":true,"$process_person_profile":false,"$geoip_disable":true,"$ip":"0.0.0.0","tool":"Bash","cli":"gh","service":"code_hosting"}} -{"event":"Plugin prompt submitted","distinct_id":"aaaaaaaaaaaaaaaa","properties":{"session":"aaaaaaaaaaaaaaaa","turn":"2222222222222222","host":"claude-code","plugin_version":"0.2.0","telemetry_version":2,"os":"linux","arcade_used_before":true,"$process_person_profile":false,"$geoip_disable":true,"$ip":"0.0.0.0","could_use_arcade":true,"service_hints":[],"reminder_sent":true}} -{"event":"Plugin tool attempted","distinct_id":"aaaaaaaaaaaaaaaa","properties":{"session":"aaaaaaaaaaaaaaaa","turn":"2222222222222222","host":"claude-code","plugin_version":"0.2.0","telemetry_version":2,"os":"linux","arcade_used_before":true,"$process_person_profile":false,"$geoip_disable":true,"$ip":"0.0.0.0","server":"arcade","tool":"Arcade_UseTool"}} -{"event":"Plugin tool failed","distinct_id":"aaaaaaaaaaaaaaaa","properties":{"session":"aaaaaaaaaaaaaaaa","turn":"1111111111111111","host":"claude-code","plugin_version":"0.2.0","telemetry_version":2,"os":"linux","arcade_used_before":true,"$process_person_profile":false,"$geoip_disable":true,"$ip":"0.0.0.0","server":"arcade","tool":"app_tool","service":"chat","failure_kind":"timeout"}} -{"event":"Plugin prompt submitted","distinct_id":"aaaaaaaaaaaaaaaa","properties":{"session":"aaaaaaaaaaaaaaaa","turn":"3333333333333333","host":"claude-code","plugin_version":"0.2.0","telemetry_version":2,"os":"linux","arcade_used_before":true,"$process_person_profile":false,"$geoip_disable":true,"$ip":"0.0.0.0","could_use_arcade":false,"service_hints":[],"reminder_sent":true}} -{"event":"Plugin tool called","distinct_id":"aaaaaaaaaaaaaaaa","properties":{"session":"aaaaaaaaaaaaaaaa","turn":"4444444444444444","host":"claude-code","plugin_version":"0.2.0","telemetry_version":2,"os":"linux","arcade_used_before":true,"$process_person_profile":false,"$geoip_disable":true,"$ip":"0.0.0.0","server":"arcade","tool":"Arcade_ListApps"}} -{"event":"Plugin subagent stopped","distinct_id":"aaaaaaaaaaaaaaaa","properties":{"session":"aaaaaaaaaaaaaaaa","host":"claude-code","plugin_version":"0.2.0","telemetry_version":2,"os":"linux","arcade_used_before":true,"$process_person_profile":false,"$geoip_disable":true,"$ip":"0.0.0.0","agent":"arcade-operator","status":"completed"}} -{"event":"Plugin prompt submitted","distinct_id":"bbbbbbbbbbbbbbbb","properties":{"session":"bbbbbbbbbbbbbbbb","host":"copilot-cli","plugin_version":"0.2.0","telemetry_version":2,"os":"linux","arcade_used_before":false,"$process_person_profile":false,"$geoip_disable":true,"$ip":"0.0.0.0","could_use_arcade":true,"service_hints":["issues"],"reminder_sent":false}} -{"event":"Plugin tool called","distinct_id":"bbbbbbbbbbbbbbbb","properties":{"session":"bbbbbbbbbbbbbbbb","host":"copilot-cli","plugin_version":"0.2.0","telemetry_version":2,"os":"linux","arcade_used_before":false,"$process_person_profile":false,"$geoip_disable":true,"$ip":"0.0.0.0","server":"arcade","tool":"Arcade_UseTool","service":"issues"}} -{"event":"Plugin subagent stopped","distinct_id":"bbbbbbbbbbbbbbbb","properties":{"session":"bbbbbbbbbbbbbbbb","host":"copilot-cli","plugin_version":"0.2.0","telemetry_version":2,"os":"linux","arcade_used_before":false,"$process_person_profile":false,"$geoip_disable":true,"$ip":"0.0.0.0","agent":"arcade-operator","status":"needs_auth","subagent_session":"cccccccccccccccc"}} -{"event":"Plugin prompt submitted","distinct_id":"cccccccccccccccc","properties":{"session":"cccccccccccccccc","host":"copilot-cli","plugin_version":"0.2.0","telemetry_version":2,"os":"linux","arcade_used_before":false,"$process_person_profile":false,"$geoip_disable":true,"$ip":"0.0.0.0","could_use_arcade":true,"service_hints":[],"reminder_sent":false}} -{"event":"Plugin tool called","distinct_id":"cccccccccccccccc","properties":{"session":"cccccccccccccccc","host":"copilot-cli","plugin_version":"0.2.0","telemetry_version":2,"os":"linux","arcade_used_before":false,"$process_person_profile":false,"$geoip_disable":true,"$ip":"0.0.0.0","server":"arcade","tool":"Arcade_SelectTools"}} -{"event":"Plugin prompt submitted","distinct_id":"dddddddddddddddd","properties":{"session":"dddddddddddddddd","host":"copilot-cli","plugin_version":"0.2.0","telemetry_version":2,"os":"linux","arcade_used_before":false,"$process_person_profile":false,"$geoip_disable":true,"$ip":"0.0.0.0","could_use_arcade":true,"service_hints":["docs"],"reminder_sent":false}} -{"event":"Plugin prompt submitted","distinct_id":"dddddddddddddddd","properties":{"session":"dddddddddddddddd","host":"copilot-cli","plugin_version":"0.2.0","telemetry_version":2,"os":"linux","arcade_used_before":false,"$process_person_profile":false,"$geoip_disable":true,"$ip":"0.0.0.0","could_use_arcade":false,"service_hints":[],"reminder_sent":false}} -{"event":"Plugin prompt submitted","distinct_id":"eeeeeeeeeeeeeeee","properties":{"session":"eeeeeeeeeeeeeeee","host":"copilot-cli","plugin_version":"0.2.0","telemetry_version":2,"os":"linux","arcade_used_before":false,"$process_person_profile":false,"$geoip_disable":true,"$ip":"0.0.0.0","could_use_arcade":true,"service_hints":["calendar"],"reminder_sent":false}} -{"event":"Plugin prompt submitted","distinct_id":"aaaaaaaaaaaaaaaa","properties":{"session":"aaaaaaaaaaaaaaaa","host":"claude-code","plugin_version":"0.2.0","os":"linux","arcade_used_before":false,"$process_person_profile":false,"$geoip_disable":true,"$ip":"0.0.0.0","could_use_arcade":true,"service_hints":[],"reminder_sent":true}} -{"event":"not-a-contract-event","distinct_id":"ffffffffffffffff","properties":{"session":"ffffffffffffffff","host":"claude-code","plugin_version":"0.2.0","telemetry_version":2,"os":"linux","arcade_used_before":false,"$process_person_profile":false,"$geoip_disable":true,"$ip":"0.0.0.0"}} -{"event":"Plugin prompt submitted","distinct_id":"aaaaaaaaaaaaaaaa","properties":{"session":"aaaaaaaaaaaaaaaa","turn":"6666666666666666","host":"claude-code","plugin_version":"0.2.0","telemetry_version":2,"os":"linux","arcade_used_before":true,"$process_person_profile":false,"$geoip_disable":true,"$ip":"0.0.0.0","could_use_arcade":true,"service_hints":[],"reminder_sent":false}} -{"event":"Plugin tool called","distinct_id":"aaaaaaaaaaaaaaaa","properties":{"session":"aaaaaaaaaaaaaaaa","turn":"6666666666666666","host":"claude-code","plugin_version":"0.2.0","telemetry_version":2,"os":"linux","arcade_used_before":true,"$process_person_profile":false,"$geoip_disable":true,"$ip":"0.0.0.0","server":"other"}} -{"event":"Plugin prompt submitted","distinct_id":"f0f0f0f0f0f0f0f0","properties":{"session":"f0f0f0f0f0f0f0f0","host":"copilot-cli","plugin_version":"0.2.0","telemetry_version":2,"os":"linux","arcade_used_before":false,"$process_person_profile":false,"$geoip_disable":true,"$ip":"0.0.0.0","could_use_arcade":true,"service_hints":["chat"],"reminder_sent":false}} -{"event":"Plugin tool called","distinct_id":"f0f0f0f0f0f0f0f0","properties":{"session":"f0f0f0f0f0f0f0f0","host":"copilot-cli","plugin_version":"0.2.0","telemetry_version":2,"os":"linux","arcade_used_before":false,"$process_person_profile":false,"$geoip_disable":true,"$ip":"0.0.0.0","server":"other"}} -{"event":"Plugin prompt submitted","distinct_id":"a1a1a1a1a1a1a1a1","properties":{"session":"a1a1a1a1a1a1a1a1","turn":"7777777777777777","host":"claude-code","plugin_version":"0.3.0","telemetry_version":2,"os":"linux","arcade_used_before":false,"$process_person_profile":false,"$geoip_disable":true,"$ip":"0.0.0.0","could_use_arcade":true,"service_hints":["email"],"reminder_sent":false}} -{"event":"Plugin tool called","distinct_id":"a1a1a1a1a1a1a1a1","properties":{"session":"a1a1a1a1a1a1a1a1","turn":"7777777777777777","host":"claude-code","plugin_version":"0.3.0","telemetry_version":2,"os":"linux","arcade_used_before":false,"$process_person_profile":false,"$geoip_disable":true,"$ip":"0.0.0.0","server":"arcade","tool":"Arcade_UseTool","service":"email"}} diff --git a/test/fixtures/telemetry-report/expected-report.json b/test/fixtures/telemetry-report/expected-report.json deleted file mode 100644 index 9413aba..0000000 --- a/test/fixtures/telemetry-report/expected-report.json +++ /dev/null @@ -1,156 +0,0 @@ -{ - "excluded": { - "invalid": 1, - "legacy": 1 - }, - "groups": [ - { - "host": "claude-code", - "plugin_version": "0.2.0", - "telemetry_version": 2, - "unit": "turn", - "denominators": { - "observed_relevant_turns": 4 - }, - "tool_only_turns": 1, - "stages": { - "gateway_discovery_or_selection": { - "count": 1, - "denominator": "observed_relevant_turns" - }, - "authorization_check_auth_needed_true": { - "count": 0, - "denominator": "observed_relevant_turns" - }, - "authorization_check_auth_needed_false": { - "count": 1, - "denominator": "observed_relevant_turns" - }, - "app_action_called": { - "count": 1, - "denominator": "observed_relevant_turns" - }, - "app_action_failed": { - "count": 1, - "denominator": "observed_relevant_turns" - }, - "attempt_observed_outcome_unknown": { - "count": 1, - "denominator": "observed_relevant_turns" - }, - "no_call_observed": { - "count": 2, - "denominator": "observed_relevant_turns" - } - }, - "builtin_tools_called": { - "Bash": { - "gh": 1 - } - }, - "builtin_tools_failed": {}, - "failure_kinds": { - "timeout": 1 - }, - "operator_stop_status": { - "completed": 1 - } - }, - { - "host": "claude-code", - "plugin_version": "0.3.0", - "telemetry_version": 2, - "unit": "turn", - "denominators": { - "observed_relevant_turns": 1 - }, - "tool_only_turns": 0, - "stages": { - "gateway_discovery_or_selection": { - "count": 0, - "denominator": "observed_relevant_turns" - }, - "authorization_check_auth_needed_true": { - "count": 0, - "denominator": "observed_relevant_turns" - }, - "authorization_check_auth_needed_false": { - "count": 0, - "denominator": "observed_relevant_turns" - }, - "app_action_called": { - "count": 1, - "denominator": "observed_relevant_turns" - }, - "app_action_failed": { - "count": 0, - "denominator": "observed_relevant_turns" - }, - "attempt_observed_outcome_unknown": { - "count": 0, - "denominator": "observed_relevant_turns" - }, - "no_call_observed": { - "count": 0, - "denominator": "observed_relevant_turns" - } - }, - "builtin_tools_called": {}, - "builtin_tools_failed": {}, - "failure_kinds": {}, - "operator_stop_status": {} - }, - { - "host": "copilot-cli", - "plugin_version": "0.2.0", - "telemetry_version": 2, - "unit": "session", - "denominators": { - "observed_relevant_sessions": 4 - }, - "operator_linked_sessions_excluded": 1, - "multi_prompt_sessions": 1, - "tool_only_sessions": 0, - "parent_attribution_unknown_sessions": 3, - "stages": { - "gateway_discovery_or_selection": { - "count": 1, - "denominator": "observed_relevant_sessions" - }, - "authorization_check_auth_needed_true": { - "count": 0, - "denominator": "observed_relevant_sessions" - }, - "authorization_check_auth_needed_false": { - "count": 0, - "denominator": "observed_relevant_sessions" - }, - "app_action_called": { - "count": 1, - "denominator": "observed_relevant_sessions" - }, - "app_action_failed": { - "count": 0, - "denominator": "observed_relevant_sessions" - }, - "no_call_observed": { - "count": 3, - "denominator": "observed_relevant_sessions" - } - }, - "failure_kinds": {}, - "operator_stop_status": { - "needs_auth": 1 - } - } - ], - "limits": [ - "Counts describe plugin hook observations only, not task outcomes or whether Arcade was needed.", - "Prompt events are limited to locally classified app-related work; unrelated prompts are absent.", - "Claude Code turn counts and Copilot CLI session counts must not be combined into one rate.", - "Copilot CLI does not emit tool-attempt events; attempt stages apply only to Claude Code.", - "Multiple tool calls in one turn or session cannot be paired without a tool-call ID.", - "Attempt outcomes are checked per server category; other_arcade groups every other Arcade connection together, so an outcome there cannot be tied to the connection or call that was attempted.", - "The Arcade MCP gateway remains the canonical source for request, auth, discovery, and tool-call telemetry." - ] -} diff --git a/test/telemetry-report.test.mjs b/test/telemetry-report.test.mjs deleted file mode 100644 index 927d06b..0000000 --- a/test/telemetry-report.test.mjs +++ /dev/null @@ -1,548 +0,0 @@ -// @ts-check - -import assert from "node:assert/strict"; -import { readFileSync, writeFileSync } from "node:fs"; -import { spawnSync } from "node:child_process"; -import path from "node:path"; -import { test } from "node:test"; -import { TELEMETRY_HOSTS } from "../hooks/telemetry-contract.mjs"; -import { loadTelemetryAdapter } from "../hooks/telemetry-adapter.mjs"; -import { ARCADE_TOOL_PREFIX } from "../hooks/telemetry-adapters/claude-code.mjs"; -import { authStatusResponse } from "./fixtures/telemetry/claude-code/auth-status-response.mjs"; -import { buildReport, normalizeExportRow, parseExportedEvents } from "../scripts/telemetry-report.mjs"; -import { assertMatchesContract, captureTelemetry, hookInput, tempDataDir } from "./telemetry-helpers.mjs"; -import { ROOT } from "./helpers.mjs"; - -const FIXTURE_DIR = path.join(ROOT, "test/fixtures/telemetry-report"); -const FORBIDDEN_KEY = /recall|precision|success.?rate|task.?success|routing.?miss|matched.?attempt/i; - -const loadFixture = () => { - const events = parseExportedEvents(readFileSync(path.join(FIXTURE_DIR, "events.jsonl"), "utf8")); - for (const event of events) { - if (event.event?.startsWith("Plugin") && event.properties?.telemetry_version === 2) { - assertMatchesContract(event); - } - } - const expected = JSON.parse(readFileSync(path.join(FIXTURE_DIR, "expected-report.json"), "utf8")); - return { events, expected }; -}; - -/** @param {unknown} value @param {string[]} keys */ -const collectKeys = (value, keys = []) => { - if (value && typeof value === "object") { - if (Array.isArray(value)) { - for (const item of value) collectKeys(item, keys); - } else { - for (const [key, child] of Object.entries(value)) { - keys.push(key); - collectKeys(child, keys); - } - } - } - return keys; -}; - -test("buildReport matches the telemetry-report fixture", () => { - const { events, expected } = loadFixture(); - assert.deepEqual(buildReport(events), expected); -}); - -test("buildReport strips PostHog export fields before validating", () => { - const { events, expected } = loadFixture(); - const noisy = events.map((row) => { - if (!row || typeof row !== "object" || !row.properties) return row; - return { - ...row, - timestamp: "2026-01-01T00:00:00.000Z", - uuid: "550e8400-e29b-41d4-a716-446655440000", - properties: { - ...row.properties, - $lib: "posthog-node", - $lib_version: "5.0.0", - }, - }; - }); - for (const row of noisy) { - if (row?.event?.startsWith("Plugin")) { - const trimmed = normalizeExportRow(row); - assert.ok(trimmed); - assert.equal(trimmed.properties.$lib, undefined); - } - } - assert.deepEqual(buildReport(noisy), expected); -}); - -test("buildReport rejects leaked hook fields in export properties", () => { - const { events } = loadFixture(); - const sample = events.find( - (row) => row.event === "Plugin prompt submitted" && row.properties?.telemetry_version === 2, - ); - assert.ok(sample); - const withPrompt = { ...sample, properties: { ...sample.properties, prompt: "secret text" } }; - const withCwd = { ...sample, properties: { ...sample.properties, cwd: "/Users/secret" } }; - assert.equal(buildReport([withPrompt]).excluded.invalid, 1); - assert.equal(buildReport([withCwd]).excluded.invalid, 1); -}); - -test("CLI prints the same JSON as buildReport", () => { - const { events, expected } = loadFixture(); - const file = path.join(FIXTURE_DIR, "events.jsonl"); - const result = spawnSync(process.execPath, [path.join(ROOT, "scripts/telemetry-report.mjs"), file], { - encoding: "utf8", - }); - assert.equal(result.status, 0, result.stderr); - assert.deepEqual(JSON.parse(result.stdout), expected); - assert.deepEqual(buildReport(events).excluded, { invalid: 1, legacy: 1 }); -}); - -const claudeRow = (event, extra) => ({ - event, - distinct_id: "aaaaaaaaaaaaaaaa", - properties: { - session: "aaaaaaaaaaaaaaaa", - turn: "1111111111111111", - host: "claude-code", - plugin_version: "0.2.0", - telemetry_version: 2, - os: "linux", - arcade_used_before: false, - $process_person_profile: false, - $geoip_disable: true, - $ip: "0.0.0.0", - ...extra, - }, -}); - -const unknownOutcomeCount = (rows) => { - for (const row of rows) assertMatchesContract(row); - const report = buildReport(rows); - assert.deepEqual(report.excluded, { invalid: 0, legacy: 0 }); - return report.groups[0].stages.attempt_observed_outcome_unknown.count; -}; - -test("an outcome on one Arcade server category does not count as the outcome for an attempt on the other", () => { - const prompt = claudeRow("Plugin prompt submitted", { could_use_arcade: true, service_hints: ["email"], reminder_sent: false }); - const attempt = (server) => claudeRow("Plugin tool attempted", { server, tool: "Arcade_UseTool" }); - const called = (server) => claudeRow("Plugin tool called", { server, tool: "Arcade_SelectTools" }); - const failed = (server) => claudeRow("Plugin tool failed", { server, tool: "Arcade_UseTool", failure_kind: "timeout" }); - - assert.equal(unknownOutcomeCount([prompt, attempt("arcade"), called("other_arcade")]), 1); - assert.equal(unknownOutcomeCount([prompt, attempt("other_arcade"), failed("arcade")]), 1); - assert.equal(unknownOutcomeCount([prompt, attempt("arcade"), called("arcade")]), 0); - assert.equal(unknownOutcomeCount([prompt, attempt("other_arcade"), failed("other_arcade")]), 0); - assert.equal(unknownOutcomeCount([prompt, attempt("arcade"), attempt("other_arcade"), called("arcade")]), 1); - // Order within the turn does not matter: an earlier outcome still counts for its category. - assert.equal(unknownOutcomeCount([prompt, called("arcade"), attempt("arcade")]), 0); - assert.equal(unknownOutcomeCount([prompt, failed("other_arcade"), attempt("other_arcade")]), 0); - assert.equal(unknownOutcomeCount([prompt, called("other_arcade"), attempt("arcade")]), 1); - // Turn-level only: one outcome on a category counts for every attempt on that category. - assert.equal(unknownOutcomeCount([prompt, attempt("arcade"), attempt("arcade"), called("arcade")]), 0); -}); - -const copilotRow = (event, session, extra) => ({ - event, - distinct_id: session, - properties: { - session, - host: "copilot-cli", - plugin_version: "0.2.0", - telemetry_version: 2, - os: "linux", - arcade_used_before: false, - $process_person_profile: false, - $geoip_disable: true, - $ip: "0.0.0.0", - ...extra, - }, -}); - -test("operator_linked_sessions_excluded counts prompt sessions removed from the denominator", () => { - const parent = "bbbbbbbbbbbbbbbb"; - const child = "cccccccccccccccc"; - const prompt = (session) => - copilotRow("Plugin prompt submitted", session, { - could_use_arcade: true, - service_hints: [], - reminder_sent: false, - }); - const stop = copilotRow("Plugin subagent stopped", parent, { - agent: "arcade-operator", - status: "needs_auth", - subagent_session: child, - }); - - for (const row of [prompt(parent), prompt(child), stop]) assertMatchesContract(row); - - const orphanStop = buildReport([prompt(parent), stop]); - assert.equal(orphanStop.groups[0].operator_linked_sessions_excluded, 0); - assert.deepEqual(orphanStop.groups[0].denominators, { observed_relevant_sessions: 1 }); - - const linked = buildReport([prompt(parent), prompt(child), stop]); - assert.equal(linked.groups[0].operator_linked_sessions_excluded, 1); - assert.deepEqual(linked.groups[0].denominators, { observed_relevant_sessions: 1 }); -}); - -test("report limits state that other_arcade groups connections and calls are not paired", () => { - const limits = buildReport([]).limits.join("\n"); - assert.match(limits, /other_arcade.*groups every other Arcade connection together/i); - assert.match(limits, /cannot be paired/i); -}); - -test("CLI rejects malformed JSON without echoing the input", (t) => { - const marker = "SYNTHETIC-PRIVATE-MARKER-7f3a"; - const dir = tempDataDir(); - const valid = readFileSync(path.join(FIXTURE_DIR, "events.jsonl"), "utf8").split("\n")[0]; - const inputs = { - "lines.jsonl": `${valid}\n{"event":"Plugin prompt submitted","properties":{"prompt":"${marker}"\n`, - "array.json": `[{"event":"${marker}",}]`, - "bare.jsonl": `${marker} not json`, - }; - for (const [name, contents] of Object.entries(inputs)) { - const file = path.join(dir, name); - writeFileSync(file, contents); - const result = spawnSync(process.execPath, [path.join(ROOT, "scripts/telemetry-report.mjs"), file], { - encoding: "utf8", - }); - t.diagnostic(`${name}: ${result.stderr.trim()}`); - assert.notEqual(result.status, 0, name); - assert.equal(result.stdout, "", name); - assert.match(result.stderr, /^telemetry-report: /, name); - for (const stream of [result.stdout, result.stderr]) { - assert.equal(stream.includes(marker), false, `${name}: ${stream}`); - assert.equal(stream.includes("Plugin prompt submitted"), false, `${name}: ${stream}`); - } - } -}); - -test("empty input produces a report with no groups", () => { - assert.deepEqual(parseExportedEvents(""), []); - assert.deepEqual(parseExportedEvents(" \n\n"), []); - const report = buildReport([]); - assert.deepEqual(report.excluded, { invalid: 0, legacy: 0 }); - assert.deepEqual(report.groups, []); - assert.ok(report.limits.length > 0); - - const file = path.join(tempDataDir(), "empty.jsonl"); - writeFileSync(file, ""); - const result = spawnSync(process.execPath, [path.join(ROOT, "scripts/telemetry-report.mjs"), file], { - encoding: "utf8", - }); - assert.equal(result.status, 0, result.stderr); - assert.deepEqual(JSON.parse(result.stdout), report); -}); - -test("a missing input file exits 1 with one stderr line and no report", () => { - const file = path.join(tempDataDir(), "does-not-exist.jsonl"); - const result = spawnSync(process.execPath, [path.join(ROOT, "scripts/telemetry-report.mjs"), file], { - encoding: "utf8", - }); - assert.equal(result.status, 1); - assert.equal(result.stdout, ""); - assert.equal( - result.stderr, - "telemetry-report: the file could not be read or the report could not be built; no report was produced\n", - ); -}); - -test("report keys avoid forbidden metric names and hosts stay separate", () => { - const { events } = loadFixture(); - const report = buildReport(events); - for (const key of collectKeys(report)) { - assert.doesNotMatch(key, FORBIDDEN_KEY, key); - } - assert.deepEqual( - report.groups.map((group) => [group.host, group.plugin_version]), - [["claude-code", "0.2.0"], ["claude-code", "0.3.0"], ["copilot-cli", "0.2.0"]], - ); - const byGroup = Object.fromEntries(report.groups.map((group) => [`${group.host}/${group.plugin_version}`, group])); - assert.equal(byGroup["claude-code/0.2.0"].unit, "turn"); - assert.equal(byGroup["claude-code/0.3.0"].unit, "turn"); - assert.equal(byGroup["copilot-cli/0.2.0"].unit, "session"); -}); - -const COPILOT_SESSION = "dd3beb80-4471-4513-99a4-a57d3d7c08df"; -const COPILOT_SUBAGENT = "57946be7-1a73-40ca-a042-abb0f68d9445"; -const COPILOT_OPERATOR = "arcade:arcade-operator"; - -/** @param {import("../hooks/telemetry-adapter.mjs").TelemetryAdapter} adapter */ -const runClaudeCrossCheck = async (adapter) => { - const dataDir = tempDataDir(); - const session = "cross-check-claude-session"; - /** @type {object[]} */ - const events = []; - const cap = async (fields, argv = []) => { - const { sent } = await captureTelemetry({ - adapter, - dataDir, - input: hookInput({ session_id: session, ...fields }), - argv, - }); - events.push(...sent); - }; - const p1 = "turn-one-11111111"; - const p2 = "turn-two-22222222"; - const p3 = "turn-three-33333333"; - const p4 = "turn-four-44444444"; - await cap({ hook_event_name: "UserPromptSubmit", prompt: "What is on my calendar tomorrow?", prompt_id: p1 }); - await cap({ hook_event_name: "PostToolUse", tool_name: `${ARCADE_TOOL_PREFIX}Arcade_SelectTools`, prompt_id: p1 }); - await cap({ - hook_event_name: "PostToolUse", - tool_name: `${ARCADE_TOOL_PREFIX}System_ManageAuthorization`, - tool_response: authStatusResponse(["authorized"]), - prompt_id: p1, - }); - await cap({ - hook_event_name: "PostToolUse", - tool_name: `${ARCADE_TOOL_PREFIX}Arcade_UseTool`, - tool_input: { tool_name: "Gmail.ListEmails" }, - prompt_id: p1, - }); - await cap({ - hook_event_name: "PostToolUseFailure", - tool_name: `${ARCADE_TOOL_PREFIX}Slack_SendMessage`, - error: "rate limited", - prompt_id: p1, - }); - await cap({ hook_event_name: "UserPromptSubmit", prompt: "Fix the parser in src/main.ts", prompt_id: p2 }); - await cap({ - hook_event_name: "PostToolUse", - tool_name: "mcp__granola__Granola_ListMeetings", - prompt_id: p2, - }); - await cap({ hook_event_name: "UserPromptSubmit", prompt: "What is on my calendar tomorrow?", prompt_id: p3 }); - await cap({ - hook_event_name: "PreToolUse", - tool_name: `${ARCADE_TOOL_PREFIX}Arcade_UseTool`, - tool_input: { tool_name: "Gmail.ListEmails" }, - prompt_id: p3, - }); - await cap({ hook_event_name: "UserPromptSubmit", prompt: "Summarize unread email from this week", prompt_id: p4 }); - return events; -}; - -/** @param {import("../hooks/telemetry-adapter.mjs").TelemetryAdapter} adapter */ -const runCopilotCrossCheck = async (adapter) => { - const dataDir = tempDataDir(); - /** @type {object[]} */ - const events = []; - const cap = async (fields) => { - const { sent } = await captureTelemetry({ adapter, dataDir, input: fields }); - events.push(...sent); - }; - const parent = (fields) => ({ - session_id: COPILOT_SESSION, - timestamp: "2026-09-24T21:27:02.743Z", - cwd: "/Users/someone/private-repo", - ...fields, - }); - const child = (fields) => parent({ session_id: COPILOT_SUBAGENT, ...fields }); - await cap(parent({ hook_event_name: "UserPromptSubmit", prompt: "What is on my calendar tomorrow?" })); - await cap(parent({ hook_event_name: "PostToolUse", tool_name: "arcade-Arcade_SelectTools", tool_input: {}, tool_result: { result_type: "success", text_result_for_llm: "[]" } })); - await cap(parent({ - hook_event_name: "PostToolUse", - tool_name: "arcade-System_ManageAuthorization", - tool_input: {}, - tool_result: { result_type: "success", text_result_for_llm: '{"providers":[{"status":"authorized"}]}' }, - })); - await cap(parent({ - hook_event_name: "PostToolUse", - tool_name: "arcade-Arcade_UseTool", - tool_input: { tool_name: "Gmail.ListEmails" }, - tool_result: { result_type: "success", text_result_for_llm: "ok" }, - })); - await cap(parent({ - hook_event_name: "PostToolUseFailure", - tool_name: "arcade-Slack_SendMessage", - tool_input: {}, - error: "MCP server 'arcade': Something went wrong in the upstream service", - })); - await cap(parent({ hook_event_name: "UserPromptSubmit", prompt: "Fix the parser in src/main.ts" })); - await cap(parent({ - hook_event_name: "PostToolUse", - tool_name: "granola-Granola_ListMeetings", - tool_input: {}, - tool_result: { result_type: "success", text_result_for_llm: "[]" }, - })); - await cap(parent({ - hook_event_name: "SubagentStop", - agent_type: COPILOT_OPERATOR, - agent_id: COPILOT_SUBAGENT, - last_assistant_message: "status: needs_auth\nsummary: sign in", - })); - await cap(child({ hook_event_name: "UserPromptSubmit", prompt: "What is on my calendar tomorrow?" })); - return events; -}; - -/** @param {object} group */ -const claudeCrossCheckExpectation = (group) => { - assert.equal(group.unit, "turn"); - assert.deepEqual(group.denominators, { observed_relevant_turns: 3 }); - assert.equal(group.tool_only_turns, 0); - assert.deepEqual(group.stages, { - gateway_discovery_or_selection: { count: 1, denominator: "observed_relevant_turns" }, - authorization_check_auth_needed_true: { count: 0, denominator: "observed_relevant_turns" }, - authorization_check_auth_needed_false: { count: 1, denominator: "observed_relevant_turns" }, - app_action_called: { count: 1, denominator: "observed_relevant_turns" }, - app_action_failed: { count: 1, denominator: "observed_relevant_turns" }, - attempt_observed_outcome_unknown: { count: 1, denominator: "observed_relevant_turns" }, - no_call_observed: { count: 1, denominator: "observed_relevant_turns" }, - }); -}; - -/** @param {object} group */ -const copilotCrossCheckExpectation = (group) => { - assert.equal(group.unit, "session"); - assert.deepEqual(group.denominators, { observed_relevant_sessions: 1 }); - assert.equal(group.operator_linked_sessions_excluded, 1); - assert.equal(group.multi_prompt_sessions, 0); - assert.equal(group.tool_only_sessions, 0); - assert.equal(group.parent_attribution_unknown_sessions, 0); - assert.deepEqual(group.stages, { - gateway_discovery_or_selection: { count: 1, denominator: "observed_relevant_sessions" }, - authorization_check_auth_needed_true: { count: 0, denominator: "observed_relevant_sessions" }, - authorization_check_auth_needed_false: { count: 1, denominator: "observed_relevant_sessions" }, - app_action_called: { count: 1, denominator: "observed_relevant_sessions" }, - app_action_failed: { count: 1, denominator: "observed_relevant_sessions" }, - no_call_observed: { count: 0, denominator: "observed_relevant_sessions" }, - }); - assert.equal(group.stages.attempt_observed_outcome_unknown, undefined); - assert.deepEqual(group.operator_stop_status, { needs_auth: 1 }); -}; - -test("tool_only_sessions counts Copilot sessions with no prompt and at least one non-prompt row", () => { - const toolOnlySession = "a0a0a0a0a0a0a0a0"; - const row = copilotRow("Plugin tool called", toolOnlySession, { - server: "arcade", - tool: "Arcade_UseTool", - service: "email", - }); - assertMatchesContract(row); - const report = buildReport([row]); - assert.deepEqual(report.excluded, { invalid: 0, legacy: 0 }); - assert.equal(report.groups[0].denominators.observed_relevant_sessions, 0); - assert.equal(report.groups[0].tool_only_sessions, 1); -}); - -test("tool_only_sessions attributes child rows to parent before counting", () => { - const parent = "b0b0b0b0b0b0b0b0"; - const child = "c0c0c0c0c0c0c0c0"; - const stop = copilotRow("Plugin subagent stopped", parent, { - agent: "arcade-operator", - status: "needs_auth", - subagent_session: child, - }); - const childPrompt = copilotRow("Plugin prompt submitted", child, { - could_use_arcade: true, - service_hints: [], - reminder_sent: false, - }); - const childTool = copilotRow("Plugin tool called", child, { - server: "arcade", - tool: "Arcade_UseTool", - service: "email", - }); - for (const r of [stop, childPrompt, childTool]) assertMatchesContract(r); - const report = buildReport([stop, childPrompt, childTool]); - assert.deepEqual(report.excluded, { invalid: 0, legacy: 0 }); - assert.equal(report.groups[0].operator_linked_sessions_excluded, 1); - assert.equal(report.groups[0].denominators.observed_relevant_sessions, 0); - assert.equal(report.groups[0].tool_only_sessions, 1); -}); - -test("PostHog $ properties may be missing from an export but must match when present", () => { - const prompt = claudeRow("Plugin prompt submitted", { could_use_arcade: true, service_hints: [], reminder_sent: false }); - const { $ip, $geoip_disable, $process_person_profile, ...stripped } = prompt.properties; - assert.deepEqual(buildReport([{ ...prompt, properties: stripped }]).excluded, { invalid: 0, legacy: 0 }); - const realIp = { ...prompt, properties: { ...prompt.properties, $ip: "203.0.113.5" } }; - assert.deepEqual(buildReport([realIp]).excluded, { invalid: 1, legacy: 0 }); -}); - -test("a legacy row with an extra property counts as invalid, not legacy", () => { - const base = { - event: "Plugin prompt submitted", - distinct_id: "aaaaaaaaaaaaaaaa", - properties: { - session: "aaaaaaaaaaaaaaaa", - host: "claude-code", - plugin_version: "0.2.0", - os: "linux", - arcade_used_before: false, - $process_person_profile: false, - $geoip_disable: true, - $ip: "0.0.0.0", - could_use_arcade: true, - service_hints: [], - reminder_sent: false, - }, - }; - // Without extra property: counts as legacy (no telemetry_version, all keys allowed). - assert.deepEqual(buildReport([base]).excluded, { invalid: 0, legacy: 1 }); - // With extra property: counts as invalid. - const withPrompt = { ...base, properties: { ...base.properties, prompt: "SECRET" } }; - assert.deepEqual(buildReport([withPrompt]).excluded, { invalid: 1, legacy: 0 }); - const withCwd = { ...base, properties: { ...base.properties, cwd: "/home/secret" } }; - assert.deepEqual(buildReport([withCwd]).excluded, { invalid: 1, legacy: 0 }); -}); - -test("parseExportedEvents reports the physical line number including blank lines", () => { - const blankLines = "\n\n\n"; - const validLine = '{"event":"Plugin prompt submitted","distinct_id":"aaaaaaaaaaaaaaaa","properties":{"session":"aaaaaaaaaaaaaaaa","host":"claude-code","plugin_version":"0.2.0","telemetry_version":2,"os":"linux","arcade_used_before":false,"$process_person_profile":false,"$geoip_disable":true,"$ip":"0.0.0.0","could_use_arcade":false,"service_hints":[],"reminder_sent":false}}'; - const badLine = "{bad"; - // 3 blank lines + valid row on line 4 + bad row on line 5 - assert.throws( - () => parseExportedEvents(`${blankLines}${validLine}\n${badLine}`), - (err) => err.message === "line 5 is not valid JSON", - ); -}); - -test("parseExportedEvents reports 'the file is not valid JSON' for a bad JSON array", () => { - assert.throws( - () => parseExportedEvents("[{bad}]"), - (err) => err.message === "the file is not valid JSON", - ); -}); - -test("a Copilot session whose only tool event is server other counts as no_call_observed", () => { - const session = "d0d0d0d0d0d0d0d0"; - const prompt = copilotRow("Plugin prompt submitted", session, { - could_use_arcade: true, - service_hints: ["chat"], - reminder_sent: false, - }); - const otherTool = copilotRow("Plugin tool called", session, { server: "other" }); - for (const r of [prompt, otherTool]) assertMatchesContract(r); - const report = buildReport([prompt, otherTool]); - assert.deepEqual(report.excluded, { invalid: 0, legacy: 0 }); - const group = report.groups[0]; - assert.equal(group.denominators.observed_relevant_sessions, 1); - assert.equal(group.stages.no_call_observed.count, 1); - assert.equal(group.stages.app_action_called.count, 0); -}); - -test("a Claude turn whose only tool event is server other counts as no_call_observed", () => { - const prompt = claudeRow("Plugin prompt submitted", { could_use_arcade: true, service_hints: ["chat"], reminder_sent: false }); - const otherTool = claudeRow("Plugin tool called", { server: "other" }); - for (const r of [prompt, otherTool]) assertMatchesContract(r); - const report = buildReport([prompt, otherTool]); - assert.deepEqual(report.excluded, { invalid: 0, legacy: 0 }); - const group = report.groups[0]; - assert.equal(group.denominators.observed_relevant_turns, 1); - assert.equal(group.stages.no_call_observed.count, 1); - assert.equal(group.stages.app_action_called.count, 0); -}); - -for (const host of TELEMETRY_HOSTS) { - test(`adapter cross-check buildReport counts for ${host}`, async () => { - const adapter = await loadTelemetryAdapter(host); - const events = host === "claude-code" ? await runClaudeCrossCheck(adapter) : await runCopilotCrossCheck(adapter); - for (const event of events) assertMatchesContract(event); - const report = buildReport(events); - assert.deepEqual(report.excluded, { invalid: 0, legacy: 0 }); - assert.equal(report.groups.length, 1); - const group = report.groups[0]; - assert.equal(group.host, host); - if (host === "claude-code") claudeCrossCheckExpectation(group); - else if (host === "copilot-cli") copilotCrossCheckExpectation(group); - else assert.fail(`no cross-check for ${host}`); - }); -}