diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index d537eb5..ccdca6d 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -10,7 +10,7 @@ { "name": "devscope", "description": "Real-time developer activity monitoring for Claude Code sessions", - "version": "0.34.0", + "version": "0.35.0", "author": { "name": "Lucas Dow", "email": "lucdow7@gmail.com" diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index cc200f6..d0507fa 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "devscope", - "version": "0.34.0", + "version": "0.35.0", "description": "Real-time developer activity monitoring for Claude Code sessions", "author": { "name": "Lucas Dow", diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f560389..0ffbef2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,6 +26,7 @@ jobs: tests/smoke/run.sh \ tests/queue/run.sh \ tests/hints/run.sh \ + tests/models/run.sh \ tests/recall/run.sh \ tests/errors/run.sh \ tests/voice/run.sh \ @@ -55,6 +56,14 @@ jobs: - name: Run hint hook tests (no backend required) run: bash tests/hints/run.sh + models: + name: first model use test + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Run first-model-use hook tests (stub server, no backend required) + run: bash tests/models/run.sh + usage: name: token usage test runs-on: ubuntu-latest diff --git a/CHANGELOG.md b/CHANGELOG.md index d5a1159..e31d36a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,24 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). +## [0.35.0] - 2026-10-08 + +### Added +- **First use of a model.** The first time a model shows up on this machine (at + session start or after `/model`), Claude is asked to offer, in one question, + to check your CLAUDE.md and memory files for instructions about model + selection and usage and update them for the new model; it does nothing + unless you agree. You see `DevScope: first time on .` and a + `model.first_use` event (`model`, `trigger`, `previousModel`) is sent, which + the dashboard shows in the live feed. Seen models are kept in + `~/.cache/devscope/models-seen`; the context-window suffix (`[1m]`) does not + make a model new, and the model in use when the plugin first runs is the + silent baseline. New synchronous hook `scripts/model-first-use.sh` on + `SessionStart` and `PostModelSwitch`: local only, the event is sent in the + background. `DEVSCOPE_HINTS=off` turns the message off; the event is still + sent. Older servers reject the new event type with a 400 and the plugin + drops it, so nothing else is affected. + ## [0.34.0] - 2026-10-08 ### Added diff --git a/CLAUDE.md b/CLAUDE.md index 3631908..cb457f1 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -41,6 +41,7 @@ scripts/ prompt-expansion.sh # UserPromptExpansion hook response-failed.sh # StopFailure hook model-switch.sh # PostModelSwitch hook + model-first-use.sh # SessionStart + PostModelSwitch (sync): first use of a model, asks Claude to offer a CLAUDE.md/memory review permission-denied.sh # PermissionDenied hook task-created.sh # TaskCreated hook cwd-changed.sh # CwdChanged hook @@ -279,6 +280,20 @@ claude plugin disable devscope@devscope # Disable The mod parses it in `hooks/voiceBar.ts` and treats a file past its piece's end + 15 s as dead; change both sides together. +## First use of a model + +`model-first-use.sh` runs synchronously (no `async`, `timeout: 5`) beside the async +`session-start.sh` and `model-switch.sh`, because Claude Code ignores async output and +this one must reach Claude as `additionalContext` (both events accept it). It keeps +`~/.cache/devscope/models-seen` (ids lowercased, `[1m]`-style suffix dropped); with no +file it seeds the model in use, or on a switch the model switched from, so existing +users are not asked about the model they already run. On a new model it prints the +`systemMessage` + `additionalContext` and sends `model.first_use` (`{model, trigger: +session_start|model_switch, previousModel?}`) from a detached subshell with every +stream closed, so a slow backend never delays the session. That also means smoke cannot +see a delivery failure for it; the backend contract test covers the schema. +`DEVSCOPE_HINTS=off` silences the message only. Tests: `tests/models/run.sh`. + ## Token usage `response.complete` and `session.end` carry two usage fields: diff --git a/README.md b/README.md index 24c0da0..8136898 100644 --- a/README.md +++ b/README.md @@ -148,6 +148,10 @@ DEVSCOPE_HINTS=off # no hints DEVSCOPE_HINT_AFTER_PR=/review-pr # suggest a different command (default: /code-review) ``` +## New model check + +The first time you use a model (at session start or after `/model`), DevScope shows `DevScope: first time on .` and asks Claude to offer, in one question, to check your CLAUDE.md and memory files for instructions about model selection and usage, such as rules that name an older model, and update them. Nothing is read or changed unless you say yes. Models already seen are listed in `~/.cache/devscope/models-seen`; the model you are on when the plugin first runs counts as seen. A `model.first_use` event appears in the dashboard's live feed. `DEVSCOPE_HINTS=off` turns the question off. + ## Token usage and cost DevScope shows each session's tokens and its **API-equivalent cost**: what those tokens would cost at Anthropic's API list prices, which is not what a Claude subscription plan charges you. The Stop and SessionEnd hooks sum every API call in the session transcript and its subagent transcripts, per model, and send the totals. Only token counts and model ids are sent, never transcript content. diff --git a/hooks/hooks.json b/hooks/hooks.json index 0d60212..279c129 100644 --- a/hooks/hooks.json +++ b/hooks/hooks.json @@ -9,6 +9,15 @@ "async": true } ] + }, + { + "hooks": [ + { + "type": "command", + "command": "${CLAUDE_PLUGIN_ROOT}/scripts/model-first-use.sh", + "timeout": 5 + } + ] } ], "SessionEnd": [ @@ -316,6 +325,15 @@ "async": true } ] + }, + { + "hooks": [ + { + "type": "command", + "command": "${CLAUDE_PLUGIN_ROOT}/scripts/model-first-use.sh", + "timeout": 5 + } + ] } ], "DirectoryAdded": [ diff --git a/scripts/model-first-use.sh b/scripts/model-first-use.sh new file mode 100755 index 0000000..e87d893 --- /dev/null +++ b/scripts/model-first-use.sh @@ -0,0 +1,74 @@ +#!/usr/bin/env bash +# First use of a model: offer to review model instructions in CLAUDE.md and memory. +# +# SessionStart (`model`) and PostModelSwitch (`to_model`). A CLAUDE.md or memory +# file often says which model to use for what, or carries advice written for an +# older model; a new model is the moment those go stale. The first time a model +# shows up on this machine, Claude is asked to offer a review of them (never to +# do it unasked), the user sees a one-line notice, and a `model.first_use` event +# is sent. +# +# Seen models live in ~/.cache/devscope/models-seen, one id per line, without +# the context-window suffix (`claude-opus-5-5[1m]` is `claude-opus-5-5`). When +# that file does not exist yet it is seeded silently with the model in use (on +# a switch, the model switched from), so installing the plugin or clearing the +# cache does not make every model look new. +# +# Synchronous by necessity: Claude Code ignores the output of async hooks. It +# is local, the event is sent in the background, and it is silent on any error. +# DEVSCOPE_HINTS=off drops the notice and the message to Claude; the event is +# still sent. +set -uo pipefail +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +# shellcheck disable=SC1091 +. "$SCRIPT_DIR/_helpers.sh" 2>/dev/null || exit 0 + +INPUT=$(cat) +HOOK=$(printf '%s' "$INPUT" | jq -r '.hook_event_name // ""' 2>/dev/null) || exit 0 + +# Lowercased, context-window suffix dropped; empty unless it looks like a model id. +normalize() { + local m + m=$(printf '%s' "$1" | tr '[:upper:]' '[:lower:]' | sed -E 's/\[[^]]*\]$//') + printf '%s' "$m" | grep -Eq '^[a-z0-9][a-z0-9._:@/-]{0,99}$' && printf '%s' "$m" +} + +case "$HOOK" in + SessionStart) + MODEL=$(normalize "$(printf '%s' "$INPUT" | jq -r '.model // ""' 2>/dev/null)") + PREVIOUS="" + TRIGGER="session_start" + ;; + PostModelSwitch) + MODEL=$(normalize "$(printf '%s' "$INPUT" | jq -r '.to_model // ""' 2>/dev/null)") + PREVIOUS=$(normalize "$(printf '%s' "$INPUT" | jq -r '.from_model // ""' 2>/dev/null)") + TRIGGER="model_switch" + ;; + *) exit 0 ;; +esac +[ -n "$MODEL" ] || exit 0 + +DIR="${HOME}/.cache/devscope" +SEEN="$DIR/models-seen" +mkdir -p -m 0700 "$DIR" 2>/dev/null || exit 0 +if [ ! -f "$SEEN" ]; then + # Baseline: what was in use before the plugin could see it is not new. + printf '%s\n' "${PREVIOUS:-$MODEL}" > "$SEEN" 2>/dev/null || exit 0 + [ -z "$PREVIOUS" ] && exit 0 +fi +grep -Fxq "$MODEL" "$SEEN" 2>/dev/null && exit 0 +printf '%s\n' "$MODEL" >> "$SEEN" 2>/dev/null || exit 0 + +PAYLOAD=$(jq -n --arg m "$MODEL" --arg t "$TRIGGER" --arg p "$PREVIOUS" \ + '{model: $m, trigger: $t} | if $p != "" then . + {previousModel: $p} else . end') || exit 0 +# Detached with every stream closed, so Claude Code does not wait on the network. +( printf '%s' "$INPUT" | "$SCRIPT_DIR/send-event.sh" "model.first_use" "$PAYLOAD" ) \ + /dev/null 2>&1 & + +[ "${DEVSCOPE_HINTS:-on}" = "off" ] && exit 0 + +jq -n --arg h "$HOOK" \ + --arg msg "DevScope: first time on ${MODEL}." \ + --arg ctx "DevScope: this is the first time the user has used the model ${MODEL}. Before starting on their request (or right away if there is none yet), ask the user in one short question whether they want you to check their CLAUDE.md files and memory files for instructions about model selection or model usage (for example rules that name a specific model, which model to use for subagents, or advice written for an older model) and update them for ${MODEL}. Do not read or change those files unless the user agrees." \ + '{systemMessage: $msg, hookSpecificOutput: {hookEventName: $h, additionalContext: $ctx}}' +exit 0 diff --git a/tests/models/run.sh b/tests/models/run.sh new file mode 100644 index 0000000..6003b33 --- /dev/null +++ b/tests/models/run.sh @@ -0,0 +1,107 @@ +#!/usr/bin/env bash +# Tests for scripts/model-first-use.sh (first use of a model). Stub server, no backend. +# +# bash tests/models/run.sh +set -euo pipefail +ROOT="$(cd "$(dirname "$0")/../.." && pwd)" +HOOK="$ROOT/scripts/model-first-use.sh" +TMP=$(mktemp -d) +export HOME="$TMP" XDG_CONFIG_HOME="$TMP/config" +unset DEVSCOPE_HINTS DEVSCOPE_PRIVACY +# shellcheck disable=SC1091 +. "$ROOT/tests/lib/stub.sh" +SEEN="$HOME/.cache/devscope/models-seen" +pass=0; fail=0 + +start() { # model [session] + jq -n --arg m "$1" --arg s "${2:-s1}" \ + '{session_id: $s, hook_event_name: "SessionStart", source: "startup", cwd: "/nonexistent", model: $m}' +} +switch() { # from to + jq -n --arg f "$1" --arg t "$2" \ + '{session_id: "s1", hook_event_name: "PostModelSwitch", cwd: "/nonexistent", + from_model: $f, to_model: $t, requested_model: null, source: "command", + context_tokens: 1000, prompt_cache_warm: true}' +} +check() { # name expected-substring-or-EMPTY actual + if { [ "$2" = "EMPTY" ] && [ -z "$3" ]; } || { [ "$2" != "EMPTY" ] && printf '%s' "$3" | grep -Fq "$2"; }; then + pass=$((pass + 1)); echo "ok $1" + else + fail=$((fail + 1)); echo "FAIL $1: expected '$2', got '$3'" + fi +} +# The event is sent in the background: wait for a POST to /api/events. +wait_event() { + for _ in $(seq 1 50); do paths | grep -q '/api/events' && return 0; sleep 0.1; done + return 1 +} +event() { last_at /api/events ".body$1"; } + +respond '{"ok":true}' +reset_hits +check "first model ever is the silent baseline" EMPTY "$(start claude-opus-5-5 | "$HOOK")" +check "baseline is recorded" "claude-opus-5-5" "$(cat "$SEEN")" +sleep 0.5 +check "baseline sends no event" EMPTY "$(paths)" +check "a seen model is silent" EMPTY "$(start claude-opus-5-5 s2 | "$HOOK")" +check "context-window suffix is the same model" EMPTY "$(start 'claude-opus-5-5[1m]' s3 | "$HOOK")" +check "case is ignored" EMPTY "$(start Claude-Opus-5-5 s4 | "$HOOK")" + +out=$(start claude-fable-5-1 s5 | "$HOOK") +check "new model tells the user" "DevScope: first time on claude-fable-5-1." "$(printf '%s' "$out" | jq -r .systemMessage)" +ctx=$(printf '%s' "$out" | jq -r .hookSpecificOutput.additionalContext) +check "new model asks Claude to offer the review" "ask the user" "$ctx" +check "it names CLAUDE.md and memory" "CLAUDE.md files and memory files" "$ctx" +check "it is about model selection and usage" "model selection or model usage" "$ctx" +check "it forbids doing it unasked" "unless the user agrees" "$ctx" +check "hook event name is SessionStart" "SessionStart" "$(printf '%s' "$out" | jq -r .hookSpecificOutput.hookEventName)" +wait_event || true +check "event type" "model.first_use" "$(event .eventType)" +check "event model" "claude-fable-5-1" "$(event .payload.model)" +check "event trigger" "session_start" "$(event .payload.trigger)" +check "no previousModel on session start" "null" "$(event .payload.previousModel)" +check "new model is recorded" "claude-fable-5-1" "$(cat "$SEEN")" +check "second time is silent" EMPTY "$(start claude-fable-5-1 s6 | "$HOOK")" + +reset_hits +out=$(switch claude-opus-5-5 'claude-sonnet-5-5[1m]' | "$HOOK") +check "switch to a new model tells Claude" "claude-sonnet-5-5" "$(printf '%s' "$out" | jq -r .hookSpecificOutput.additionalContext)" +check "hook event name is PostModelSwitch" "PostModelSwitch" "$(printf '%s' "$out" | jq -r .hookSpecificOutput.hookEventName)" +wait_event || true +check "switch event trigger" "model_switch" "$(event .payload.trigger)" +check "switch event previous model" "claude-opus-5-5" "$(event .payload.previousModel)" +check "switch to a seen model is silent" EMPTY "$(switch claude-sonnet-5-5 claude-opus-5-5 | "$HOOK")" + +# No seen file yet and the first thing seen is a switch: the model switched +# from is the baseline, the one switched to is new. +rm -f "$SEEN" +out=$(switch claude-opus-5-5 claude-haiku-5-5 | "$HOOK") +check "switch with no baseline still announces the new model" "claude-haiku-5-5" "$(printf '%s' "$out" | jq -r .systemMessage)" +check "switch with no baseline records both" "claude-opus-5-5 claude-haiku-5-5" "$(tr '\n' ' ' < "$SEEN")" + +reset_hits +out=$(start claude-new-1 s7 | DEVSCOPE_HINTS=off "$HOOK") +check "hints off is silent" EMPTY "$out" +wait_event || true +check "hints off still sends the event" "claude-new-1" "$(event .payload.model)" +check "hints off still records the model" "claude-new-1" "$(cat "$SEEN")" + +check "missing model is silent" EMPTY "$(start '' s8 | "$HOOK")" +check "unsafe model id is ignored" EMPTY "$(start 'x"; rm -rf ~' s9 | "$HOOK")" +check "other hook events are ignored" EMPTY "$(jq -n '{hook_event_name: "Stop", model: "claude-z"}' | "$HOOK")" +check "garbage stdin is silent" EMPTY "$(printf 'not json' | "$HOOK")" +check "empty stdin is silent" EMPTY "$(printf '' | "$HOOK")" + +# Runs synchronously at every session start: the seen path must be fast. +t0=$(date +%s%N); for _ in 1 2 3 4 5 6 7 8 9 10; do start claude-opus-5-5 | "$HOOK" >/dev/null; done; t1=$(date +%s%N) +avg_ms=$(( (t1 - t0) / 10000000 )) +if [ "$avg_ms" -lt 150 ]; then pass=$((pass + 1)); echo "ok seen path averages ${avg_ms}ms (< 150ms)"; else fail=$((fail + 1)); echo "FAIL seen path averages ${avg_ms}ms"; fi + +# A slow backend must not hold up the hook (the event is detached). +slow +t0=$(date +%s%N); start claude-slow-1 s10 | "$HOOK" >/dev/null; t1=$(date +%s%N) +ms=$(( (t1 - t0) / 1000000 )) +if [ "$ms" -lt 1000 ]; then pass=$((pass + 1)); echo "ok slow backend does not block (${ms}ms)"; else fail=$((fail + 1)); echo "FAIL slow backend blocked for ${ms}ms"; fi + +echo "---"; echo "$pass passed, $fail failed" +[ "$fail" -eq 0 ] diff --git a/tests/smoke/fixtures/model-first-use.json b/tests/smoke/fixtures/model-first-use.json new file mode 100644 index 0000000..990cf8d --- /dev/null +++ b/tests/smoke/fixtures/model-first-use.json @@ -0,0 +1,12 @@ +{ + "session_id": "smoke-session-0001", + "cwd": "__SMOKE_CWD__", + "hook_event_name": "PostModelSwitch", + "permission_mode": "default", + "from_model": "claude-opus-5-5", + "to_model": "claude-fable-5-1", + "requested_model": "fable", + "source": "command", + "context_tokens": 41234, + "prompt_cache_warm": true +}