diff --git a/AGENTS.md b/AGENTS.md index bc09b6b..5ed3a42 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -5,15 +5,27 @@ For agents editing **arcade-plugin** itself. Client-specific files are generated. Edit the source, run `npm run generate`, then `npm run verify` (it fails if a generated file was edited by hand). [ARCHITECTURE.md](ARCHITECTURE.md) lists the sources; `.gitattributes` lists -every fully generated file. `skills/try-arcade/SKILL.md` and -`agents/arcade-operator.agent.md` are hand-written except -for a marked rules block. +every fully generated file. `skills/try-arcade/SKILL.md`, +`agents/arcade-operator.agent.md`, and `docs/telemetry.md` are hand-written +except for a marked generated block (the event tables in docs/telemetry.md +come from `hooks/telemetry-contract.mjs`; never edit inside those markers). - Routing rules: edit `hooks/routing-guidance.mjs`, never the rules text in the Cursor rule, arcade-operator, or try-arcade. - Hooks: add a hook or a client in `hooks/hook-hosts.mjs`, never in a `hooks.json`. A new client also needs its expected output in `test/hooks.test.mjs`. +- Telemetry: `hooks/telemetry-contract.mjs` defines every event, property, + and allowed value; change them there only. Client-specific hook input mapping + lives only in `hooks/telemetry-adapters/.mjs`. Wire a client through + `HOSTS.telemetry` in `hooks/hook-hosts.mjs` (generated manifests follow + that source). `telemetry.mjs` prints nothing; only `telemetry-send.mjs` may + touch the network (a test enforces both). `TELEMETRY_ENABLED` in + `hooks/telemetry-config.mjs` stays `false` without separate approval; while + false, `npm run generate` writes no telemetry hooks. `docs/telemetry.md` + documents the event contract, privacy, opt-outs, and approval before enabling. + `scripts/telemetry-docs.mjs` generates its event tables from the contract; + keep its tests in `test/telemetry-docs.test.mjs`. - Codex hooks are blocked upstream ([docs/install/codex.md](docs/install/codex.md)). Don't remove the root `$schema` to force them; that breaks Agent Plugins conformance. diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 207d7a3..adacd0d 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -10,6 +10,7 @@ edited by hand. The package ships no credentials. | `plugin.json`, `mcp.json`, `VERSION` | identity, gateway URL, version, Codex listing metadata | | `hooks/routing-guidance.mjs` | the routing rules, as sentences | | `hooks/hook-hosts.mjs` | which hook script runs on which event in which client | +| `hooks/telemetry-contract.mjs` | the telemetry event contract; also generates the event tables in docs/telemetry.md | | `agents/arcade-operator.agent.md` | the operator, outside its generated rules block | | `skills/` | the skills, outside the generated rules block in try-arcade | @@ -85,6 +86,12 @@ success or narrating tool internals. The Arcade MCP server is the canonical place to record request, authentication, tool-discovery, tool-call, and completion outcomes. This package does not ask a -model to self-report tokens, turns, or success, and it ships no telemetry hook. -If a host-specific hook later adds supplemental signals, it must be explicit, -opt-in, and documented as non-portable. +model to self-report tokens, turns, or success. + +Optional client hooks in `hooks/telemetry-adapters/` can send supplemental, +scoped usage events when `TELEMETRY_ENABLED` is `true` and the user has not +opted out. **This build keeps `TELEMETRY_ENABLED` false**, so no telemetry +hooks are generated and nothing is sent. See [docs/telemetry.md](docs/telemetry.md) +for the contract, opt-outs, local state, and the separate approval required +before collection can be enabled. Gateway telemetry remains authoritative; +plugin events do not establish task success. diff --git a/README.md b/README.md index e72108b..d8cbbe7 100644 --- a/README.md +++ b/README.md @@ -69,6 +69,17 @@ anything is sent, created, or deleted. - Privacy: tasks run through Arcade's hosted gateway and the apps you connect — [privacy policy](https://www.arcade.dev/privacy-policy). +## Plugin usage events (off in this build) + +Claude Code and Copilot CLI include telemetry adapters, but **`TELEMETRY_ENABLED` +is `false`**: this package sends no usage events and generates no telemetry +hooks in client manifests. [docs/telemetry.md](docs/telemetry.md) describes +the event contract, what would be stored locally when enabled, opt-outs, and +the separate approval required before collection can be enabled. The Arcade MCP +gateway remains the canonical source for request and tool-call telemetry. +Adapter behavior is covered by in-process tests; +see [docs/telemetry.md](docs/telemetry.md) for what CI does and does not exercise. + ## Develop Agents editing this repo should read [AGENTS.md](AGENTS.md). diff --git a/docs/install/claude-code.md b/docs/install/claude-code.md index b8b4bfa..d5be562 100644 --- a/docs/install/claude-code.md +++ b/docs/install/claude-code.md @@ -37,6 +37,18 @@ If your host has other Arcade MCP connectors too, Claude may pick the wrong one connected and prefer disabling other Arcade connectors while testing this plugin. +## Telemetry + +Telemetry is **off** in this build (`TELEMETRY_ENABLED` is `false`): no usage +events are sent and generated manifests include no telemetry hooks. When +telemetry is enabled, the adapter in `hooks/telemetry-adapters/claude-code.mjs` +classifies prompts, records `PreToolUse` on the plugin and claude.ai Arcade +prefixes, and stores scope under `CLAUDE_PLUGIN_DATA`. Opt out with +`ARCADE_PLUGIN_TELEMETRY=0`, `DISABLE_TELEMETRY`, or +`CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC`. See +[telemetry.md](../telemetry.md) and +[what CI exercises](../telemetry.md#what-ci-exercises). + ## First steps - "What's on my calendar tomorrow?" diff --git a/docs/install/copilot.md b/docs/install/copilot.md index 542ae24..05bf18b 100644 --- a/docs/install/copilot.md +++ b/docs/install/copilot.md @@ -26,6 +26,20 @@ from config files, so there's no per-prompt reminder here. VS Code reads the same file but can't run Agent Plugins hook commands yet, so it relies on the skills. +## Telemetry + +Telemetry is **off** in this build: generated manifests include no telemetry +hooks and nothing is sent. When enabled, `hooks/telemetry-adapters/copilot-cli.mjs` +records MCP tools as `-`, operator stops with `subagent_session`, +and session-scoped prompt state under `COPILOT_PLUGIN_DATA` (no `turn`, no +`PreToolUse`, no built-in CLI/web events). See +[telemetry.md](../telemetry.md) for opt-outs (`ARCADE_PLUGIN_TELEMETRY=0` and +`COPILOT_OFFLINE`) and +[what CI exercises](../telemetry.md#what-ci-exercises). + +Shared `com.github.copilot/hooks/hooks.json` uses `runOnlyIfScriptExists` so VS +Code exits quietly without a plugin path. + ## First steps - "What's on my calendar tomorrow?" diff --git a/docs/install/vscode.md b/docs/install/vscode.md index 35e824b..b57c9ed 100644 --- a/docs/install/vscode.md +++ b/docs/install/vscode.md @@ -17,6 +17,9 @@ VS Code loads root `plugin.json` as an Agent Plugin: 2 skills, the gateway, and `arcade-operator` from `com.github.copilot/agents/`. It does not read the `.cursor-plugin/` adapter. No lifecycle hooks: VS Code doesn't yet run Agent Plugins hook commands with the plugin's path or pass their output to the model. +Copilot hook commands check that the script exists at the plugin path and exit +when the path is missing; this package has no validated VS Code telemetry flow, +and nothing is sent from VS Code. If you already installed the plugin via Copilot CLI, VS Code may auto-discover it from `~/.copilot/installed-plugins/`. Install in one place. diff --git a/docs/support-matrix.md b/docs/support-matrix.md index c64034c..7e26775 100644 --- a/docs/support-matrix.md +++ b/docs/support-matrix.md @@ -27,6 +27,19 @@ session-start text doesn't reach the main conversation. The files are `clients/cursor/hooks/hooks.json` (Cursor CLI), and `com.github.copilot/hooks/hooks.json` (Copilot CLI). +Telemetry adapters are separate from routing. **`TELEMETRY_ENABLED` is false** +in this build, so manifests include no telemetry hooks and nothing is sent. +When enabled, wiring comes from `hooks/hook-hosts.mjs`: + +| Client adapter | Telemetry events (when enabled) | +| --- | --- | +| Claude Code | `UserPromptSubmit`, `PreToolUse` on `^(?:mcp__plugin_arcade_arcade__\|mcp__claude_ai_arcade__)`, MCP `PostToolUse` / `PostToolUseFailure`, built-in `WebFetch` / `WebSearch` / listed `Bash` CLIs, Arcade operator `SubagentStop` | +| Copilot CLI | `UserPromptSubmit`, MCP `PostToolUse` / `PostToolUseFailure` (`-`), Arcade operator `SubagentStop` with `subagent_session` (no `PreToolUse`, no `turn`, no built-in tools) | + +Session start clears local prompt scope through the routing hook; it sends no +telemetry event. No other adapter in this package sends telemetry. See +[what's sent and its limits](telemetry.md). + ¹ The Cursor IDE (3.21.18) lists the commands on the plugin page but not in the `/` menu. Other plugins' commands don't appear there either. ² The IDE (3.21.18) and Cloud Agents don't run plugin hooks, so the @@ -37,10 +50,12 @@ load the always-apply rule. ⁴ Cowork runs the prompt and subagent hooks but doesn't add the session-start text, so its main conversation gets the short reminder and the skill, not the full rules. -⁵ Copilot CLI drops the output of prompt hooks from config files, so it gets -session and subagent hooks only. +⁵ Copilot CLI drops the output of **routing** prompt hooks from config files, +so the main conversation gets session and subagent hooks only. Telemetry still +hooks `UserPromptSubmit` when enabled; `reminder_sent` is always `false` there. ⁶ VS Code reads `com.github.copilot/hooks/hooks.json` but doesn't expand -`${PLUGIN_ROOT}` for Agent Plugins hooks or pass their output to the model yet. +`${PLUGIN_ROOT}` for Agent Plugins hooks or pass their output to the model yet; +`runOnlyIfScriptExists` makes those commands no-ops without a plugin path. ⁷ Blocked upstream; see [codex.md](install/codex.md). Skills are `try-arcade` and `scale-arcade`. The subagent is diff --git a/docs/telemetry.md b/docs/telemetry.md new file mode 100644 index 0000000..ddfba68 --- /dev/null +++ b/docs/telemetry.md @@ -0,0 +1,256 @@ +# Plugin telemetry + +**Telemetry is off in this build.** The plugin sends no usage events, and +`npm run generate` writes no telemetry hooks into any client manifest. +`TELEMETRY_ENABLED` in `hooks/telemetry-config.mjs` stays `false` until +collection is separately approved after ingestion, opt-outs, privacy, and +distribution requirements are verified. + +The sections below describe what **would** be sent when telemetry is turned on, +its privacy and opt-out controls, and the approval required before enabling it. +Nothing here implies that this package currently transmits data. + +When enabled, Claude Code and Copilot CLI hooks would inspect prompts locally +in each session to recognize app-related work. Relevance state belongs to that +session; it does not carry between sessions. Prompts classified as unrelated +would send no event. Direct Arcade tool calls would remain observable even +without a classified prompt. Alternative MCP, CLI, and web tools would send +events only during app-related work. + +Events would carry hashed session IDs, with no ID lasting across sessions. They +would exclude prompt text, commands, app data, names, email addresses, and Arcade +account IDs. These observations do not establish task success or whether +Arcade was needed. Install and signed-in-user counts require gateway data. + +## Prompt scope + +A prompt matching the local app classifier or mentioning Arcade would open a +30-minute observation period for its session. A short explicit confirmation such +as “yes, send it” would continue that period without extending its expiry. An +unrelated substantive prompt would close it. Background task notifications would +leave the current period unchanged. Expired, absent, or invalid state would +produce no alternative-tool telemetry. + +`could_use_arcade` and `service_hints` describe keywords in the current prompt, +not the preceding task. A confirmation reply can therefore send a scoped prompt +event with `could_use_arcade: false` and no service hints. Keyword matching can +misclassify prompts; the labeled evaluation measures that limitation separately. + +Session starts send no usage event. Only the Arcade operator's stop reports +send subagent events. Claude Code adds its routing reminder on every prompt +except short acknowledgements and background task results, whether or not +telemetry is on. Copilot CLI has no routing reminder hook, so `reminder_sent` +stays false there. + +## Opt-outs when telemetry is on + +While `TELEMETRY_ENABLED` is `false`, these switches are inert but documented +for a future enabled build. They are implemented in `hooks/telemetry-run.mjs` +and each client's adapter in `hooks/telemetry-adapters/`. + +Set `ARCADE_PLUGIN_TELEMETRY=0` in your environment, or in Claude Code's +`settings.json`: + +```json +{ "env": { "ARCADE_PLUGIN_TELEMETRY": "0" } } +``` + +`false`, `off`, and `no` also work. Telemetry is also off when `DO_NOT_TRACK` +is set to any non-empty value other than `0`, `false`, `off`, or `no`. In +Claude Code, `DISABLE_TELEMETRY` and `CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC` +opt out when set to **any** non-empty value (including `0` and `false`), matching +Claude's own telemetry switches. + +With telemetry opted out, the client still invokes its configured Node hooks. +The telemetry hook exits before classifying the prompt or storing state. Claude +Code's routing hook still adds its reminder. An environment variable cannot +remove hooks from the manifest. + +In Copilot CLI, set `ARCADE_PLUGIN_TELEMETRY` to `0`, `false`, `off`, or `no` +in your shell before starting `copilot`. Any other value, including an unset +variable, leaves that switch inactive. `COPILOT_OFFLINE` turns telemetry off +when it is set to any non-empty value other than `0`, `false`, `off`, or `no`. +An unset or empty `COPILOT_OFFLINE`, and those four values, leave telemetry on +through this switch. + +For testing an enabled build, `ARCADE_PLUGIN_TELEMETRY_HOST` sends events to a +different host. + +## Where it runs + +The Claude Code and Copilot CLI adapters are the only ones wired for telemetry +in `hooks/hook-hosts.mjs` (`hooks/telemetry-adapters/claude-code.mjs` and +`hooks/telemetry-adapters/copilot-cli.mjs`). Claude maps MCP tools under +`mcp__plugin____` (this plugin's gateway) and +`mcp__claude_ai_arcade__` (the claude.ai Arcade connection). Attempt events +(`PreToolUse`) are emitted only for those two prefixes. Copilot maps MCP tools +as `-` (for example `arcade-Arcade_SelectTools` when the MCP +server name matches `mcp.json`). Copilot sends no `turn`, no `PreToolUse` +attempt events, and no built-in CLI or web tool events. + +Claude ties alternative-tool observations to the hashed `turn` for the +current `prompt_id` (`requiresTurn: true`). Copilot keeps prompt scope per +`session` without a prompt ID (`requiresTurn: false`). + +Local state uses `CLAUDE_PLUGIN_DATA` (Claude Code) and `COPILOT_PLUGIN_DATA` +(Copilot CLI) for `arcade-used` and `prompt-scope/` files. + +Copilot's manifest is shared with VS Code. Hook commands use +`runOnlyIfScriptExists`: each command checks that its script exists at +`${PLUGIN_ROOT}` and exits quietly when the path is missing, so VS Code does +not run telemetry today. Cursor is not wired up; its hook input can include the +user's email. claude.ai, ChatGPT, Codex, and OpenCode don't run telemetry hooks +from this package. + +### What CI exercises + +| Adapter | Tested in this repo | Not covered here | +| --- | --- | --- | +| Claude Code | In-process hook captures, contract tests, `claude plugin validate --strict` on **2.1.258** (runs on the committed manifest, which has no telemetry hooks; no CLI has validated a manifest with telemetry hooks enabled) | Live user sessions, IDE extensions, desktop Code tab, Cowork delivery and opt-out | +| Copilot CLI | In-process captures, fixture JSON, contract tests, `npm run verify:copilot` on **1.0.88** (runs on the committed manifest, which has no telemetry hooks; checks only MCP servers and skills; no CLI has validated a manifest with telemetry hooks enabled) | Live sessions, Windows PowerShell hook commands, VS Code agent sessions | + +None of these checks prove end-user transmission while `TELEMETRY_ENABLED` is +`false`. The in-process tests exercise adapter behavior for an enabled build. + +## What is stored on your machine + +When telemetry is enabled, each client's plugin data folder contains its own +`arcade-used` flag, readable only by you. It holds `true` after an Arcade call +succeeds in that client plugin installation and supplies `arcade_used_before`. + +Prompt relevance state lives in `prompt-scope/.json` in +the same folder. It contains a relevance boolean, expiry timestamp, and optional +hashed prompt ID that ties tool calls to their prompt. It contains no prompt text, +commands, tool arguments, or service content. Expired state cannot authorize +observation. The next prompt-state write removes expired or malformed entries; +at most 256 session state files are retained. Session start clears that +session's state, except after Claude Code compacts the conversation. Claude +alternative-tool observations require a matching hashed prompt ID; an absent +prompt ID cannot authorize those observations. + +- Claude Code: `~/.claude/plugins/data//` +- Copilot CLI: `~/.copilot/plugin-data/<…>/` + +If the client supplies no plugin data folder, the plugin sends nothing. + +## What is sent + + +Every event has these properties: + +| Property | Value | +| --- | --- | +| `distinct_id` | the same value as `session` | +| `session` | `sha256(session_id)`, first 16 hex characters, where `session_id` is the client's random ID for the session | +| `turn` | `sha256(session_id + ":" + prompt_id)`, first 16 hex characters; Claude Code only, because Copilot CLI has no prompt ID | +| `arcade_used_before` | whether an Arcade tool call had succeeded for this client plugin installation before this event (from the `arcade-used` file) | +| `host` | `claude-code` \| `copilot-cli` | +| `telemetry_version` | `2`, the scoped event contract; earlier events have no version | +| `plugin_version` | from `VERSION` | +| `os` | `darwin` \| `linux` \| `win32` \| `other` | +| `$process_person_profile` | `false` | +| `$geoip_disable` | `true` | +| `$ip` | `0.0.0.0`, so PostHog stores this instead of your real IP address | + +Events and their extra properties: + +| Event | When | Extra properties | +| --- | --- | --- | +| `Plugin prompt submitted` | UserPromptSubmit, only for locally classified app work and short confirmations of that work; background task results are excluded. In Copilot CLI a relevant subagent prompt also sends it | `could_use_arcade`: boolean, a local keyword guess (see below). `service_hints`: service categories the prompt mentions. `reminder_sent`: boolean, whether the routing reminder was added (always `false` in Copilot CLI). | +| `Plugin tool attempted` | PreToolUse, in Claude Code, before an MCP call through this plugin's Arcade gateway or the claude.ai Arcade connection; an attempt does not show whether the tool finished | `server`: `arcade` (this plugin's gateway) \| `other_arcade` (the claude.ai Arcade connection). `tool`: only for `arcade` and `other_arcade`: an exact gateway tool name, `app_tool` for a recognized service category, or `other`; app tool names are never sent. `service`: the service category, when the tool, the app tool passed to `Arcade_UseTool`, or the server name matches one. | +| `Plugin tool called` | PostToolUse, on Arcade tools, or alternative MCP tools while the current turn concerns app work | `server`: `arcade` (this plugin's gateway) \| `other_arcade` (another connection exposing Arcade's gateway tools) \| `other`. `tool`: only for `arcade` and `other_arcade`: an exact gateway tool name, `app_tool` for a recognized service category, or `other`; app tool names are never sent. `service`: the service category, when the tool, the app tool passed to `Arcade_UseTool`, or the server name matches one. `auth_needed`: only for `System_ManageAuthorization`: whether its answer says a service still needs sign-in. | +| `Plugin tool failed` | PostToolUseFailure, on Arcade tools, or alternative MCP tools while the current turn concerns app work | `server`: `arcade` (this plugin's gateway) \| `other_arcade` (another connection exposing Arcade's gateway tools) \| `other`. `tool`: only for `arcade` and `other_arcade`: an exact gateway tool name, `app_tool` for a recognized service category, or `other`; app tool names are never sent. `service`: the service category, when the tool, the app tool passed to `Arcade_UseTool`, or the server name matches one. `failure_kind`: picked on your machine from the error message; the message is not sent: `auth_required` \| `session_expired` \| `unreachable` \| `timeout` \| `http_error` \| `interrupted` \| `tool_error`. | +| `Plugin built-in tool called` | PostToolUse, Claude Code only, on `WebFetch` and `WebSearch`, and on `Bash` commands that run `gh` \| `glab` \| `curl` \| `wget` \| `http` \| `osascript`; only while the current turn concerns app work | `tool`: `Bash` \| `WebFetch` \| `WebSearch`. `cli`: only for `Bash`: the program the command runs, `gh` \| `glab` \| `curl` \| `wget` \| `http` \| `osascript`. `service`: the program's service category, only for `gh` \| `glab`: `code_hosting`. | +| `Plugin built-in tool failed` | PostToolUseFailure, Claude Code only, on `WebFetch` and `WebSearch`, and on `Bash` commands that run `gh` \| `glab` \| `curl` \| `wget` \| `http` \| `osascript`; only while the current turn concerns app work | `tool`: `Bash` \| `WebFetch` \| `WebSearch`. `cli`: only for `Bash`: the program the command runs, `gh` \| `glab` \| `curl` \| `wget` \| `http` \| `osascript`. `service`: the program's service category, only for `gh` \| `glab`: `code_hosting`. | +| `Plugin subagent stopped` | SubagentStop, only for arcade-operator | `agent`: `arcade-operator`. `status`: only for `arcade-operator`: the status line of its final report, `completed` \| `needs_auth` \| `needs_confirmation` \| `needs_clarification` \| `failed` \| `unknown`. `subagent_session`: `sha256(agent_id)`, first 16 hex characters. In Copilot CLI the subagent's own events carry this as `session`. | + +Service categories: `email`, `calendar`, `chat`, `issues`, `docs`, `meetings`, `crm`, `code_hosting`, `analytics`, `storage`. + + +`could_use_arcade` is a local keyword guess (in +`hooks/telemetry-classify.mjs`) at whether the prompt is a task Arcade could +do: email, calendar, chat, and the other categories above. + +`failure_kind`, `auth_needed`, and `cli` are values from fixed lists, picked +on your machine. The error text, the tool output, and the command they are +picked from are never sent. During an app-related observation period, a Bash +command sends an event only when one of its +commands starts with a listed program; commands that call a program by its +full path, such as `/opt/homebrew/bin/gh`, are not counted. A command that runs +two listed programs, such as `gh … && curl …`, sends one event for each. +Claude Code starts these hooks only for commands that match (the hook `if` field, Claude Code 2.1.246 and later). + +## Never sent + +- prompt text, or any text you or the model wrote +- tool inputs, tool outputs, or error messages +- commands, their arguments, URLs, and search queries +- file paths, the working directory, or transcript paths +- names of MCP servers other than Arcade's, or their tool names +- names of subagents other than Arcade's +- your email, username, hostname, repository, or Arcade account + +The plugin drops any property not listed on this page before sending. + +PostHog would see the IP address the request comes from, like any web request, +but would not store it: every event sets `$ip` to `0.0.0.0`, and location +lookup is off. + +## Gateway telemetry + +The Arcade MCP gateway (`https://api.arcade.dev/mcp/arcade`) remains the +canonical source for request, authentication, discovery, tool-call, and +completion telemetry. Plugin events are supplemental observations from client +hooks. They do not replace gateway records and cannot prove task success on +their own. + +## Observation limits + +Plugin events describe what hooks observed. They do not establish whether Arcade +was needed or whether a user's task succeeded. `could_use_arcade` and +`service_hints` are local keyword classifications, and unrelated prompts are +excluded. Neither host sends a task ID. + +Claude Code supplies a hashed `turn`; Copilot CLI supplies only a hashed +`session`, which can contain multiple prompts. Detached delivery can change +event arrival order. Tool attempts, completions, and failures are separate +observations; timestamps alone cannot pair calls or establish a task outcome. + +MCP server and gateway tool names are recognition heuristics, not verified +connection provenance. Copilot's server names do not identify which plugin +installed them. Operator stop events can link a child session to its parent; +missing stops and other subagents leave that attribution incomplete. Session +hashes do not link activity across sessions. + +`failure_kind` is classified locally from error text, and `auth_needed` reflects +the authorization tool's response. An operator's `status` is the model's report +of its own work. These values are not verified server status or task-success +signals. Routing quality requires labeled evaluations outside this telemetry. + +## Routing evaluation + +Evaluate classification and client routing on controlled, labeled cases separately +from production usage events. Include implicit app requests, contextual follow-ups, +and local coding tasks that name an app. Fixture accuracy does not establish +real-user routing quality. + +The existing [Tool Recommendation Cost Eval](https://github.com/ArcadeAI/tool-recommendation-cost-eval) +provides an offline cross-client harness and published results. Record the plugin +SHA and client configuration when evaluating the plugin; Tool Recommendation +on/off results do not establish the plugin's effect on routing or cost. Plugin +telemetry and that harness are complementary; neither replaces the other. + +## Before turning it on + +Separate approval is required before setting `TELEMETRY_ENABLED` to `true` and +shipping telemetry hooks. Confirm, outside this repository: + +- user-facing disclosure and privacy policy coverage for plugin usage events +- analytics retention and processor handling for the PostHog project +- verified ingestion per client and `telemetry_version` for the build under review +- opt-out switches verified on each supported client surface +- applicable store or distribution decisions for builds that transmit data + +These are release acceptance requirements; the current code and CI do not +enforce them. A repository install can consume a branch before a GitHub release +is tagged, so a branch install can transmit data before any release is tagged. diff --git a/scripts/generate-manifests.mjs b/scripts/generate-manifests.mjs index bb1a89a..d047de2 100644 --- a/scripts/generate-manifests.mjs +++ b/scripts/generate-manifests.mjs @@ -11,6 +11,7 @@ import { SKILL_RULES, } from "../hooks/routing-guidance.mjs"; import { HOOK_TIMEOUT_SEC, HOOKS, HOSTS } from "../hooks/hook-hosts.mjs"; +import { fillTelemetryTables, TELEMETRY_DOC } from "./telemetry-docs.mjs"; const ROOT = resolve(dirname(fileURLToPath(import.meta.url)), ".."); @@ -26,6 +27,9 @@ export const FILES_WITH_GENERATED_RULES = { "skills/try-arcade/SKILL.md": SKILL_RULES, }; +/** Hand-written files whose event tables are generated from hooks/telemetry-contract.mjs. */ +export const FILES_WITH_GENERATED_TABLES = [TELEMETRY_DOC]; + /** Generated copy → the file it is copied from. */ export const COPIED_FILES = { // Each skill folder has to work on its own. @@ -63,6 +67,7 @@ export const FILE_SOURCES = { "com.github.copilot/agents/arcade-operator.agent.md": [OPERATOR, "hooks/routing-guidance.mjs"], [OPERATOR]: ["hooks/routing-guidance.mjs"], "skills/try-arcade/SKILL.md": ["hooks/routing-guidance.mjs"], + [TELEMETRY_DOC]: ["hooks/telemetry-contract.mjs"], }; /** Formats an array of source paths as a human-readable list. */ @@ -89,6 +94,9 @@ const outOfDateError = (path) => { if (path in FILES_WITH_GENERATED_RULES) { return `${path}: the rules block is out of date. Run npm run generate. If you edited the block by hand, make the change in hooks/routing-guidance.mjs instead.`; } + if (FILES_WITH_GENERATED_TABLES.includes(path)) { + return `${path}: the telemetry tables are out of date. Run npm run generate. If you edited the tables by hand, make the change in hooks/telemetry-contract.mjs instead.`; + } const sources = FILE_SOURCES[path]; const joined = joinSources(sources); if (path in COPIED_FILES) { @@ -268,6 +276,10 @@ const buildFiles = (root) => { files.set(path, fillRulesBlock(readText(root, path), rules, path)); } + for (const path of FILES_WITH_GENERATED_TABLES) { + files.set(path, fillTelemetryTables(readText(root, path))); + } + // Every output must have a source entry so error messages can name it. for (const path of files.keys()) { requireSources(path, FILE_SOURCES); diff --git a/scripts/telemetry-docs.mjs b/scripts/telemetry-docs.mjs new file mode 100644 index 0000000..dd88f28 --- /dev/null +++ b/scripts/telemetry-docs.mjs @@ -0,0 +1,59 @@ +// @ts-check +/** Writes the event tables in docs/telemetry.md from hooks/telemetry-contract.mjs. */ + +import { COMMON_PROPERTIES, EVENTS, SERVICE_CATEGORIES } from "../hooks/telemetry-contract.mjs"; + +export const TELEMETRY_DOC = "docs/telemetry.md"; +export const TELEMETRY_BLOCK_BEGIN = + ""; +export const TELEMETRY_BLOCK_END = ""; + +const code = (/** @type {string} */ text) => `\`${text}\``; + +export const buildTelemetryTables = () => { + const common = [ + "| Property | Value |", + "| --- | --- |", + "| `distinct_id` | the same value as `session` |", + ...Object.entries(COMMON_PROPERTIES).map(([key, { doc }]) => `| ${code(key)} | ${doc} |`), + ]; + const events = [ + "| Event | When | Extra properties |", + "| --- | --- | --- |", + ...Object.entries(EVENTS).map(([name, spec]) => { + const when = spec.when ? `${spec.hook}, ${spec.when}` : spec.hook; + const extras = Object.entries(spec.properties) + .map(([key, { doc }]) => `${code(key)}: ${doc}.`) + .join(" "); + return `| ${code(name)} | ${when} | ${extras} |`; + }), + ]; + return [ + "Every event has these properties:", + "", + ...common, + "", + "Events and their extra properties:", + "", + ...events, + "", + `Service categories: ${SERVICE_CATEGORIES.map(code).join(", ")}.`, + ].join("\n"); +}; + +/** + * Replaces the generated block in docs/telemetry.md. + * @param {string} text + */ +export const fillTelemetryTables = (text) => { + const begin = text.indexOf(TELEMETRY_BLOCK_BEGIN); + const end = text.indexOf(TELEMETRY_BLOCK_END); + if (begin === -1 || end === -1 || end < begin) { + throw new Error(`${TELEMETRY_DOC} is missing the generated telemetry block markers`); + } + return ( + text.slice(0, begin + TELEMETRY_BLOCK_BEGIN.length) + + `\n${buildTelemetryTables()}\n` + + text.slice(end) + ); +}; diff --git a/test/generate-manifests.test.mjs b/test/generate-manifests.test.mjs index dc237f1..85140f6 100644 --- a/test/generate-manifests.test.mjs +++ b/test/generate-manifests.test.mjs @@ -5,6 +5,7 @@ import { test } from "node:test"; import { FILE_SOURCES, FILES_WITH_GENERATED_RULES, + FILES_WITH_GENERATED_TABLES, generateManifests, requireSources, } from "../scripts/generate-manifests.mjs"; @@ -24,6 +25,7 @@ const EXPECTED_SOURCES = { "com.github.copilot/agents/arcade-operator.agent.md": ["agents/arcade-operator.agent.md", "hooks/routing-guidance.mjs"], "agents/arcade-operator.agent.md": ["hooks/routing-guidance.mjs"], "skills/try-arcade/SKILL.md": ["hooks/routing-guidance.mjs"], + "docs/telemetry.md": ["hooks/telemetry-contract.mjs"], }; test("FILE_SOURCES lists the expected sources for every generated path", () => { @@ -48,6 +50,9 @@ test("check mode fails with a source-naming error when a generated file is hand- if (path in FILES_WITH_GENERATED_RULES) { // Edit inside the block so the rules-block path is specifically tested. writeFileSync(fullPath, readFileSync(fullPath, "utf8").replace("Gateway: Arcade is connected", "Gateway: Any server is connected")); + } else if (FILES_WITH_GENERATED_TABLES.includes(path)) { + // Only the tables are generated, so the edit has to be inside them. + writeFileSync(fullPath, readFileSync(fullPath, "utf8").replace("Service categories:", "Service kinds:")); } else { writeFileSync(fullPath, `${readFileSync(fullPath, "utf8")} `); } diff --git a/test/helpers.mjs b/test/helpers.mjs index 707de63..72dafeb 100644 --- a/test/helpers.mjs +++ b/test/helpers.mjs @@ -3,7 +3,7 @@ import { cpSync, mkdtempSync, readFileSync } from "node:fs"; import { tmpdir } from "node:os"; import path from "node:path"; import { fileURLToPath } from "node:url"; -import { COPIED_FILES, FILES_WITH_GENERATED_RULES } from "../scripts/generate-manifests.mjs"; +import { COPIED_FILES, FILES_WITH_GENERATED_RULES, FILES_WITH_GENERATED_TABLES } from "../scripts/generate-manifests.mjs"; export const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); @@ -52,7 +52,7 @@ export const skipOnWindows = { skip: process.platform === "win32" && "the comman /** A temp copy of the generator's source files, for tests that write. */ export const makeFixture = () => { const root = mkdtempSync(path.join(tmpdir(), "arcade-plugin-")); - const sources = ["VERSION", "plugin.json", "mcp.json", ...Object.keys(FILES_WITH_GENERATED_RULES), ...Object.values(COPIED_FILES).flat()]; + const sources = ["VERSION", "plugin.json", "mcp.json", ...Object.keys(FILES_WITH_GENERATED_RULES), ...FILES_WITH_GENERATED_TABLES, ...Object.values(COPIED_FILES).flat()]; for (const file of sources) { cpSync(path.join(ROOT, file), path.join(root, file)); } diff --git a/test/telemetry-docs.test.mjs b/test/telemetry-docs.test.mjs new file mode 100644 index 0000000..405b8a8 --- /dev/null +++ b/test/telemetry-docs.test.mjs @@ -0,0 +1,47 @@ +// @ts-check + +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { + buildTelemetryTables, + fillTelemetryTables, + TELEMETRY_BLOCK_BEGIN, + TELEMETRY_BLOCK_END, +} from "../scripts/telemetry-docs.mjs"; +import { COMMON_PROPERTIES, EVENTS } from "../hooks/telemetry-contract.mjs"; + +test("buildTelemetryTables lists every contract event and common property", () => { + const tables = buildTelemetryTables(); + for (const eventName of Object.keys(EVENTS)) { + assert.ok(tables.includes(`| \`${eventName}\` |`), `tables must include a row for event "${eventName}"`); + } + for (const propName of Object.keys(COMMON_PROPERTIES)) { + assert.ok(tables.includes(`| \`${propName}\` |`), `tables must include a row for common property "${propName}"`); + } + assert.match(tables, /Service categories:/); +}); + +test("fillTelemetryTables replaces only the generated block", () => { + const tables = buildTelemetryTables(); + const source = [ + "before the block", + "", + TELEMETRY_BLOCK_BEGIN, + "stale table row", + TELEMETRY_BLOCK_END, + "", + "after the block", + "", + ].join("\n"); + const expected = [ + "before the block", + "", + TELEMETRY_BLOCK_BEGIN, + tables, + TELEMETRY_BLOCK_END, + "", + "after the block", + "", + ].join("\n"); + assert.equal(fillTelemetryTables(source), expected); +});