diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index d22f955..aeb5076 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -10,7 +10,7 @@ { "name": "devscope", "description": "Real-time developer activity monitoring for Claude Code sessions", - "version": "0.37.0", + "version": "0.38.0", "author": { "name": "Lucas Dow", "email": "lucdow7@gmail.com" diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index bd297a5..2bedb33 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "devscope", - "version": "0.37.0", + "version": "0.38.0", "description": "Real-time developer activity monitoring for Claude Code sessions", "author": { "name": "Lucas Dow", diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0ffbef2..ccd2854 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,6 +30,7 @@ jobs: tests/recall/run.sh \ tests/errors/run.sh \ tests/voice/run.sh \ + tests/commands/run.sh \ tests/lib/stub.sh hooks-consistency: @@ -72,6 +73,14 @@ jobs: - name: Run token usage tests (stub server, no backend required) run: bash tests/usage/run.sh + commands: + name: command timing test + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Run command timing tests (stub server, no backend required) + run: bash tests/commands/run.sh + recall: name: prompt recall test runs-on: ubuntu-latest diff --git a/CHANGELOG.md b/CHANGELOG.md index f2b7c5d..c14174f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,31 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). +## [0.38.0] - 2026-11-05 + +### Added +- **Command timing.** Bash calls now carry what DevScope needs to show which + commands Claude waits on and whether they are getting slower: Claude Code's + `tool_use_id`, a command signature (`commandSignature`: the program and up to + two plain words for programs that take subcommands, such as `bun run build` + or `docker compose up`, and the program alone for any other (`cat ` is + `cat`); quoted text and heredoc bodies are never read, so the command, its + arguments, paths and messages are never sent; in `private` mode the program + only), and for a backgrounded command its task id. A background command's + PostToolUse fires when it is backgrounded, so its real duration is read from + the transcript at Stop and SessionEnd (`scripts/bg_commands.py`: when Claude + Code queued the notification that it finished, its status and exit code) and + sent as `command.background_complete`. Only sessions that backgrounded a + command scan their transcript, incrementally. Needs a DevScope server with + command timing; older servers ignore the new fields and drop the new event. + +### Fixed +- **Durations of parallel tool calls.** The start time of a call was kept per + session and tool name, so two Bash calls running at once overwrote each + other's start and one got the wrong duration. Durations now come from + Claude Code's own `duration_ms` (2.1.318+), and the fallback timer is keyed by + `tool_use_id`. + ## [0.37.0] - 2026-10-26 ### Added diff --git a/CLAUDE.md b/CLAUDE.md index 5e44672..3f52797 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -23,6 +23,7 @@ commands/ scripts/ _helpers.sh # Shared helpers (config loading, SHA256, timestamps) token_usage.py # Exact per-model token totals from transcripts (Stop/SessionEnd, /devscope:backfill-usage) + bg_commands.py # When background Bash commands settled, from the transcript (Stop/SessionEnd) send-event.sh # Core event sender (all hooks call this) session-start.sh # SessionStart hook session-end.sh # SessionEnd hook @@ -317,6 +318,40 @@ see a delivery failure for it; the backend contract test covers the schema. total. Still sent because older servers read it and the server's estimator uses it as the per-turn context size. +## Command timing + +Bash `tool.complete` / `tool.fail` carry `toolUseId`, `commandSignature` and, for a backgrounded +command, `backgroundTaskId` + `runInBackground` (0.38.0+; the server's `command_runs`). +`duration` is Claude Code's `duration_ms` when it sends one (2.1.318+), else the start time +`tool-use.sh` wrote under `~/.cache/devscope/timings/`. + +- **Signature** (`_ds_command_signature`, jq, bounded passes only since it runs on every Bash + call). Quoted spans and heredoc bodies are removed first, so nothing quoted can become a + program or word. Then the first segment of the chain that is not set-up (`cd`, `export`, + `echo`, `mkdir`, ...), without `VAR=val` and wrappers (`sudo`, `env`, `time`, `timeout`, + `nice`, `nohup`) with their options and option values; the program's basename. Only programs + in `multi` (those that take subcommands: `git`, `docker`, `bun`, `npm`, `gh`, `make`, ...) + keep words after it, matching `^[a-z][a-z0-9_:-]{0,23}$` up to the first that does not: two + words, three only when the second names a group (`run`, `compose`, `pr`, `stash`, ...), + because a third word is usually an argument (`git checkout `). Everything else is the + program alone (`cat ` is `cat`). Private: program only. The backend's `cleanSignature` + enforces the same `multi` list: change both. +- **Background commands**: PostToolUse fires when a command is backgrounded, and no hook fires + when it settles. `tool-complete.sh` touches `~/.cache/devscope/bg/.active`; + `_ds_report_bg_commands` (Stop, and SessionEnd with `final`, before `session.end` because + send-event.sh maps the session through the state file SessionEnd removes) runs + `bg_commands.py scan` for marked sessions only, in batches of at most 200 (up to 5). It reads + the transcript from a byte offset (`bg/.offset`, read and written under + `flock` on `.offset.lock`, so a concurrent Stop and SessionEnd never report twice; `final` + drops only the marker, and offsets go 7 days after their marker) for + `queue-operation` `enqueue` entries whose `` + summary starts with `Background command` (agents and monitors notify too) and sends + `{commands: [{toolUseId, taskId, status, exitCode?, finishedAt}]}` (exit code from the last + `exit code N` in the summary: the description comes first) as + `command.background_complete`. Never the summary (it holds the description) or the output path. + Subagent transcripts hold no queue operations. +- Tests: `tests/commands/run.sh`. + ## Making Changes 1. Edit scripts in `scripts/` diff --git a/scripts/_helpers.sh b/scripts/_helpers.sh index a6ab6e7..685a4f7 100755 --- a/scripts/_helpers.sh +++ b/scripts/_helpers.sh @@ -503,3 +503,121 @@ _ds_extract_subcommand() { ;; esac } + +# A command's signature for timing ("bun run build", "docker compose up", +# "git push"), so durations can be compared by what ran without sending the +# command. Only programs that take subcommands (MULTI below) keep words after +# the program, up to two, a third only after a group word; every other program +# is the program alone (`cat ` is "cat"), since its words are operands. +# Words stop at the first flag, path, number or `=`. Before splitting the chain, +# quoted text and heredoc bodies are removed, so nothing quoted can become a +# program or a word. The main segment is the first one that is not set-up +# (`cd x && bun test` is "bun test"); env assignments and wrappers (sudo, env, +# time, timeout, nice, nohup) are skipped with their options. Private mode keeps +# the program only, the same as toolSubcommand. The server re-checks the same +# rules (cleanSignature in the backend's commandQueries.ts): change both. +# Usage: _ds_command_signature "$RAW_TOOL_INPUT_JSON" +_ds_command_signature() { + local max_words=3 + [ "${DEVSCOPE_PRIVACY:-standard}" = "private" ] && max_words=1 + # Only bounded passes (reduce over the words): this runs on every Bash call. + printf '%s' "$1" | jq -r --argjson max "$max_words" ' + def words: [splits("[ \t]+")] | map(select(length > 0)); + # Everything after the line that starts a heredoc is its body. + def no_heredoc: + if test("<<") then + split("\n") as $l | ($l | map(test("<<")) | index(true)) as $i | $l[:$i + 1] | join("\n") + else . end; + # Quoted spans become a lone \u0001 token: never a program, never a word. + def no_quotes: + gsub("\"(\\\\.|[^\"\\\\])*\""; " \u0001 ") | gsub("\u0027[^\u0027]*\u0027"; " \u0001 "); + # Drop leading VAR=val and wrappers with their options (and option values). + def unwrap: + reduce .[] as $t ({prefix: true, wrap: false, skip: false, timeout: false, out: []}; + if .prefix | not then .out += [$t] + elif .skip then .skip = false + elif ($t | test("^[A-Za-z_][A-Za-z0-9_]*=")) then . + elif ($t | IN("sudo", "env", "time", "nice", "nohup", "command", "exec", "builtin", "timeout")) + then .wrap = true | .timeout = ($t == "timeout") + elif .wrap and ($t | startswith("-")) then + (if ($t | IN("-u", "-g", "-n", "-s", "-k", "-C", "-D", "-h", "-p", "-r", "-t", "-U", "-S", + "--user", "--group", "--signal", "--kill-after", "--adjustment", "--chdir")) + then .skip = true else . end) + elif .timeout and ($t | test("^[0-9]")) then .timeout = false + else .prefix = false | .out = [$t] end) + | .out; + def setup: IN("cd", "pushd", "popd", "export", "source", ".", "set", "unset", "shopt", + "true", ":", "echo", "printf", "trap", "local", "declare", "mkdir"); + def prog: split("/") | last | ascii_downcase | gsub("[^a-z0-9._+-]"; ""); + def plain: test("^[a-z][a-z0-9_:-]{0,23}$"); + # Programs whose next word names what they do (a subcommand, script, target + # or tool), so it is kept. + def multi: IN("git", "gh", "glab", "docker", "podman", "kubectl", "helm", "terraform", "tofu", + "npm", "pnpm", "yarn", "bun", "bunx", "npx", "deno", "cargo", "rustup", + "go", "uv", "uvx", "pip", "pip3", "poetry", "pdm", "hatch", "conda", + "brew", "apt", "apt-get", "dnf", "systemctl", "journalctl", "make", "just", "task", + "claude", "gcloud", "aws", "az", "fly", "flyctl", "vercel", "wrangler", "supabase", + "firebase", "dotnet", "mvn", "gradle", "gradlew", "mix", "rails", "bundle", "rake", + "composer", "swift", "xcodebuild", "zig", "nix"); + # A third word is kept only after a word that names a group of commands + # ("npm run test:unit", "docker compose up", "gh pr view"); otherwise it is + # usually an argument ("git checkout ", "docker logs "). + def group: IN("run", "run-script", "compose", "exec", "x", "workspace", "workspaces", + "pr", "issue", "repo", "release", "workflow", "api", "auth", + "container", "image", "volume", "network", "system", "buildx", "builder", + "stash", "worktree", "submodule", "remote", "bisect", "lfs", + "plugin", "plugins", "marketplace", "mcp", "config", "cache", "db", "migrate"); + (.command // "" | tostring | .[:2000] | no_heredoc | no_quotes) + | [splits("&&|\\|\\||;|\\||\n")] + | map(gsub("^[\\s({]+"; "") | gsub("[\\s)}]+$"; "") | words | unwrap | select(length > 0)) + | (map(select((.[0] | prog | setup) | not)) | first) // first // [] + | if length == 0 or (.[0] | prog) == "" then "" + else + (.[0] | prog) as $p + | ([$p] + + (if ($p | multi) then + .[1:] | reduce .[] as $w ({out: [], stop: false}; + if .stop or ($w | plain | not) then .stop = true else .out += [$w] end) + | .out + else [] end)) + | if length > 2 and (.[1] | group | not) then .[:2] else . end + | .[:$max] | join(" ") + end + ' 2>/dev/null || true +} + +# Report background Bash commands that settled since the last call, as +# `command.background_complete` events ({commands: [...]}, up to 200 each, see +# bg_commands.py). Runs only for sessions tool-complete.sh marked as having +# backgrounded a command, so other sessions never read their transcript for +# this. `final` (SessionEnd) drops the marker; the offset stays (a Stop still +# running must not rescan from the start) until the sweep below. Needs +# SCRIPT_DIR (hook scripts set it) and python3; does nothing without them. +# Usage: _ds_report_bg_commands "$INPUT" [final] +_ds_report_bg_commands() { + local input="$1" final="${2:-}" sid transcript dir found count batch f base + sid=$(printf '%s' "$input" | jq -r '.session_id // ""' 2>/dev/null | tr -cd 'a-zA-Z0-9_-') + transcript=$(printf '%s' "$input" | jq -r '.transcript_path // ""' 2>/dev/null) + dir="${XDG_CACHE_HOME:-$HOME/.cache}/devscope/bg" + [ -n "$sid" ] && [ -f "$dir/$sid.active" ] && [ -n "$transcript" ] || return 0 + command -v python3 >/dev/null 2>&1 && [ -n "${SCRIPT_DIR:-}" ] || return 0 + # A full batch means more may follow (the scan stops at 200); at most 5 rounds. + for batch in 1 2 3 4 5; do + found=$(python3 "$SCRIPT_DIR/bg_commands.py" scan "$transcript" 2>/dev/null || echo '[]') + count=$(printf '%s' "$found" | jq 'length' 2>/dev/null || echo 0) + [ "${count:-0}" -gt 0 ] 2>/dev/null || break + printf '%s' "$input" | "$SCRIPT_DIR/send-event.sh" "command.background_complete" \ + "$(jq -nc --argjson c "$found" '{commands: $c}')" >/dev/null 2>&1 || true + [ "$count" -ge 200 ] || break + done + [ "$final" = "final" ] && rm -f "$dir/$sid.active" 2>/dev/null + # Sweep: markers after 7 days, offsets (and locks) after 7 days once their + # session's marker is gone. + find "$dir" -type f -name '*.active' -mtime +7 -delete 2>/dev/null || true + find "$dir" -type f \( -name '*.offset' -o -name '*.offset.lock' \) -mtime +7 2>/dev/null \ + | while IFS= read -r f; do + base="${f%.lock}"; base="${base%.offset}" + [ -e "$base.active" ] || rm -f "$f" 2>/dev/null + done + return 0 +} diff --git a/scripts/bg_commands.py b/scripts/bg_commands.py new file mode 100644 index 0000000..85056c7 --- /dev/null +++ b/scripts/bg_commands.py @@ -0,0 +1,145 @@ +#!/usr/bin/env python3 +"""When background Bash commands finished, from the Claude Code transcript. + +A Bash command run with `run_in_background` (or moved to the background after +its timeout, or by the user) reports PostToolUse as soon as it is backgrounded, +and no hook fires when it settles. Claude Code does log the settling: it +enqueues a for the model, and the transcript records that +as a `queue-operation` entry with `operation: "enqueue"` and a timestamp: + + + b5r0fuwka + toolu_016AWJ... + /tmp/claude-1000/.../tasks/b5r0fuwka.output + completed + Background command "..." completed (exit code 0) + + + bg_commands.py scan + The commands that settled since the last scan, as a JSON array of + {toolUseId, taskId, status, exitCode?, finishedAt} ([] when none). + Reads incrementally: the byte offset of the last line reported is kept + in ~/.cache/devscope/bg/.offset, under an exclusive lock so + a Stop and a SessionEnd running at once never report twice. At most 200 + per call; the rest is read by the next call. + +Only ids, the status, the exit code and the time leave the machine: never the +command, its description (inside the summary) or its output path. +""" +import fcntl +import json +import os +import re +import sys +import tempfile + +MAX_ITEMS = 200 # the backend accepts at most 200 per event + +NOTIFICATION = re.compile(r"(.*?)", re.S) + + +def tag(body, name): + m = re.search(r"<%s>(.*?)" % (name, name), body, re.S) + return m.group(1).strip() if m else "" + + +def cache_dir(): + base = os.environ.get("XDG_CACHE_HOME") or os.path.join(os.path.expanduser("~"), ".cache") + return os.path.join(base, "devscope", "bg") + + +def offset_path(transcript): + name = re.sub(r"[^A-Za-z0-9_-]", "", os.path.basename(transcript).rsplit(".jsonl", 1)[0]) + return os.path.join(cache_dir(), (name or "transcript") + ".offset") + + +def parse_entry(entry): + """A settled background command from one transcript entry, else None.""" + if entry.get("type") != "queue-operation" or entry.get("operation") != "enqueue": + return None + content = entry.get("content") + if not isinstance(content, str) or "" not in content: + return None + m = NOTIFICATION.search(content) + if not m: + return None + body = m.group(1) + summary = tag(body, "summary") + # Agents and monitors send task notifications too; only shells count here. + if not summary.startswith("Background command"): + return None + tool_use_id = tag(body, "tool-use-id") + status = tag(body, "status") + finished_at = entry.get("timestamp") + if not tool_use_id or not status or not isinstance(finished_at, str): + return None + item = { + "toolUseId": tool_use_id[:200], + "taskId": tag(body, "task-id")[:100], + "status": status[:20], + "finishedAt": finished_at[:50], + } + # The real "(exit code N)" ends the summary; the description before it + # could contain the words too. + codes = re.findall(r"exit code (-?\d+)", summary) + if codes: + item["exitCode"] = int(codes[-1]) + return item + + +def scan(transcript): + if not os.path.isfile(transcript): + return [] + os.makedirs(cache_dir(), mode=0o700, exist_ok=True) + with open(offset_path(transcript) + ".lock", "w") as lock: + fcntl.flock(lock, fcntl.LOCK_EX) # the offset read-modify-write is one step + return scan_locked(transcript) + + +def scan_locked(transcript): + state = offset_path(transcript) + try: + with open(state) as f: + offset = int(f.read().strip() or 0) + except (OSError, ValueError): + offset = 0 + size = os.path.getsize(transcript) + if offset > size: # rewritten transcript: start over + offset = 0 + found, seen = [], set() + with open(transcript, "rb") as f: + f.seek(offset) + for raw in f: + if not raw.endswith(b"\n"): + break # half-written line: read it next time + if b'"queue-operation"' in raw: + try: + item = parse_entry(json.loads(raw)) + except (ValueError, AttributeError): + item = None + if item and item["toolUseId"] not in seen: + if len(found) >= MAX_ITEMS: + break # full: this line is the first of the next call + seen.add(item["toolUseId"]) + found.append(item) + offset += len(raw) + fd, tmp = tempfile.mkstemp(dir=cache_dir()) + with os.fdopen(fd, "w") as f: + f.write(str(offset)) + os.replace(tmp, state) + return found + + +def main(argv): + if len(argv) != 3 or argv[1] != "scan": + print(__doc__, file=sys.stderr) + return 2 + try: + print(json.dumps(scan(argv[2]), separators=(",", ":"))) + except Exception: # never break a hook + print("[]") + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/scripts/response-stop.sh b/scripts/response-stop.sh index b920ede..82b1faa 100755 --- a/scripts/response-stop.sh +++ b/scripts/response-stop.sh @@ -32,4 +32,7 @@ else | if ($us | length) > 0 then . + {usageSnapshot: $us} else . end') fi +# Background commands that settled during this turn (or woke it up). +_ds_report_bg_commands "$INPUT" + echo "$INPUT" | "$SCRIPT_DIR/send-event.sh" "response.complete" "$PAYLOAD" diff --git a/scripts/session-end.sh b/scripts/session-end.sh index 1024e62..a0a9415 100755 --- a/scripts/session-end.sh +++ b/scripts/session-end.sh @@ -27,6 +27,8 @@ if [ -n "$TRANSCRIPT_PATH" ]; then rm -f "$_DS_USAGE_CACHE/$(basename "$TRANSCRIPT_PATH" .jsonl).json" 2>/dev/null || true fi find "$_DS_USAGE_CACHE" -name '*.json' -mtime +30 -delete 2>/dev/null || true +# Start times of tool calls that never completed (e.g. denied in hard mode). +find "${HOME}/.cache/devscope/timings" -type f -mmin +1440 -delete 2>/dev/null || true # Read accumulated file changes FILES_CHANGED="[]" @@ -59,6 +61,10 @@ PAYLOAD=$(jq -n \ | if ($tu | length) > 0 then . + {tokenUsage: $tu} else . end | if ($us | length) > 0 then . + {usageSnapshot: $us} else . end') +# Background commands that settled since the last Stop. Before session.end: +# send-event.sh maps the session through the state file removed below. +_ds_report_bg_commands "$INPUT" final + echo "$INPUT" | "$SCRIPT_DIR/send-event.sh" "session.end" "$PAYLOAD" # Clean up all session state files diff --git a/scripts/tool-complete.sh b/scripts/tool-complete.sh index d9584b5..6d6ab4d 100755 --- a/scripts/tool-complete.sh +++ b/scripts/tool-complete.sh @@ -50,16 +50,40 @@ fi SESSION_ID_SAFE=$(echo "$SESSION_ID" | tr -cd 'a-zA-Z0-9_-') TOOL_NAME_SAFE=$(echo "$TOOL_NAME" | tr -cd 'a-zA-Z0-9_-') -# Calculate duration from start timestamp +# Duration: Claude Code's own `duration_ms` (2.1.318+) when present, else the +# start timestamp tool-use.sh wrote (keyed by tool_use_id, see there). The file +# is removed either way so it does not pile up. +TOOL_USE_ID=$(echo "$INPUT" | jq -r '.tool_use_id // ""') +TOOL_USE_ID_SAFE=$(printf '%s' "$TOOL_USE_ID" | tr -cd 'a-zA-Z0-9_-') DURATION_MS=0 TIMING_DIR="${HOME}/.cache/devscope/timings" -TIMING_FILE="${TIMING_DIR}/${SESSION_ID_SAFE}_${TOOL_NAME_SAFE}" +TIMING_FILE="${TIMING_DIR}/${TOOL_USE_ID_SAFE:-${SESSION_ID_SAFE}_${TOOL_NAME_SAFE}}" if [ -f "$TIMING_FILE" ]; then START_NS=$(cat "$TIMING_FILE") NOW_NS=$(_ds_now_ns) DURATION_MS=$(( (NOW_NS - START_NS) / 1000000 )) rm -f "$TIMING_FILE" fi +CC_DURATION_MS=$(echo "$INPUT" | jq -r '.duration_ms // empty | numbers | floor' 2>/dev/null || true) +[ -n "$CC_DURATION_MS" ] && DURATION_MS="$CC_DURATION_MS" + +# Bash command timing (command_runs on the server): a signature instead of the +# command, and whether it went to the background. A backgrounded command's +# PostToolUse fires when it is backgrounded, so its total time is only known +# when it settles; bg_commands.py finds that in the transcript at Stop and +# SessionEnd, and the marker below tells it this session has some to look for. +COMMAND_SIGNATURE="" +BG_TASK_ID="" +RUN_IN_BG=false +if [ "$TOOL_NAME" = "Bash" ]; then + COMMAND_SIGNATURE=$(_ds_command_signature "$(echo "$INPUT" | jq -c '.tool_input // {}')") + RUN_IN_BG=$(echo "$INPUT" | jq -r '.tool_input.run_in_background == true') + BG_TASK_ID=$(echo "$INPUT" | jq -r '(.tool_response // .tool_result // {}) | objects | .backgroundTaskId // ""' 2>/dev/null || true) + if [ -n "$BG_TASK_ID" ] && [ -n "$SESSION_ID_SAFE" ]; then + _DS_BG_DIR="${XDG_CACHE_HOME:-$HOME/.cache}/devscope/bg" + mkdir -p -m 0700 "$_DS_BG_DIR" 2>/dev/null && touch "$_DS_BG_DIR/${SESSION_ID_SAFE}.active" 2>/dev/null || true + fi +fi if [ "$HOOK_EVENT" = "PostToolUseFailure" ]; then SUCCESS=false @@ -112,6 +136,10 @@ PAYLOAD=$(jq -n \ --argjson intr "$IS_INTERRUPT" \ --arg ts "$TOOL_SUBCOMMAND" \ --arg tih "$TOOL_INPUT_HASH" \ + --arg tuid "$TOOL_USE_ID" \ + --arg cs "$COMMAND_SIGNATURE" \ + --arg bg "$BG_TASK_ID" \ + --argjson rib "$RUN_IN_BG" \ '{toolName: $tn, success: $s, duration: $d} | if $em != "" then . + {errorMessage: $em} else . end | if $ai != "" then . + {agentId: $ai} else . end @@ -119,7 +147,10 @@ PAYLOAD=$(jq -n \ | if $tr != null then . + {toolResult: $tr} else . end | . + {isInterrupt: $intr} | if $ts != "" then . + {toolSubcommand: $ts} else . end - | . + {toolInputHash: $tih}') + | . + {toolInputHash: $tih} + | if $tuid != "" then . + {toolUseId: $tuid} else . end + | if $cs != "" then . + {commandSignature: $cs} else . end + | if $bg != "" then . + {backgroundTaskId: $bg, runInBackground: $rib} else . end') # Send the event and capture the backend's response body so we can surface # soft nudges back into Claude's session via stderr (exit 2 → system message). diff --git a/scripts/tool-use.sh b/scripts/tool-use.sh index 3c29dc1..3cbd8eb 100755 --- a/scripts/tool-use.sh +++ b/scripts/tool-use.sh @@ -53,10 +53,14 @@ fi SESSION_ID_SAFE=$(echo "$SESSION_ID" | tr -cd 'a-zA-Z0-9_-') TOOL_NAME_SAFE=$(echo "$TOOL_NAME" | tr -cd 'a-zA-Z0-9_-') -# Write start timestamp for duration calculation +# Write start timestamp for duration calculation. Claude Code 2.1.318+ sends +# `duration_ms` on PostToolUse itself; this is the fallback for older versions. +# Keyed by tool_use_id so parallel calls of one tool do not overwrite each +# other's start (the session_tool key is for inputs without an id). +TOOL_USE_ID_SAFE=$(echo "$INPUT" | jq -r '.tool_use_id // ""' | tr -cd 'a-zA-Z0-9_-') TIMING_DIR="${HOME}/.cache/devscope/timings" mkdir -p -m 0700 "$TIMING_DIR" -echo "$(_ds_now_ns)" > "${TIMING_DIR}/${SESSION_ID_SAFE}_${TOOL_NAME_SAFE}" +echo "$(_ds_now_ns)" > "${TIMING_DIR}/${TOOL_USE_ID_SAFE:-${SESSION_ID_SAFE}_${TOOL_NAME_SAFE}}" # Extract privacy-safe subcommand from raw input (before sanitization) RAW_INPUT_JSON=$(echo "$INPUT" | jq -c '.tool_input // {}') diff --git a/tests/commands/run.sh b/tests/commands/run.sh new file mode 100644 index 0000000..dd5c849 --- /dev/null +++ b/tests/commands/run.sh @@ -0,0 +1,181 @@ +#!/usr/bin/env bash +# Tests for Bash command timing: the command signature, what tool-complete.sh +# sends (Claude Code's duration_ms, toolUseId, background), and how Stop and +# SessionEnd report background commands that settled (scripts/bg_commands.py), +# against a stub DevScope server. +# +# bash tests/commands/run.sh +set -euo pipefail +ROOT="$(cd "$(dirname "$0")/../.." && pwd)" +TMP=$(mktemp -d) +export HOME="$TMP" XDG_CONFIG_HOME="$TMP/config" XDG_CACHE_HOME="$TMP/cache" +unset DEVSCOPE_PRIVACY +# shellcheck disable=SC1091 +. "$ROOT/tests/lib/stub.sh" + +pass=0; fail=0 +ok() { pass=$((pass + 1)); echo "ok $1"; } +bad() { fail=$((fail + 1)); echo "FAIL $1: $2"; } + +# --- signature ------------------------------------------------------------- +sig() { # command [privacy] + DEVSCOPE_PRIVACY="${2:-standard}" timeout 5 bash -c '. "$1/scripts/_helpers.sh" 2>/dev/null; _ds_command_signature "$2"' \ + _ "$ROOT" "$(jq -nc --arg c "$1" '{command: $c}')" +} +check_sig() { # command expected [privacy] + local got; got=$(sig "$1" "${3:-standard}") + [ "$got" = "$2" ] && ok "signature: $1 -> $2" || bad "signature: $1" "got '$got', want '$2'" +} +check_sig 'bun test' 'bun test' +check_sig 'cd packages/backend && bun run build' 'bun run build' +check_sig '(cd packages/backend && bun test)' 'bun test' +check_sig 'sudo docker compose up -d' 'docker compose up' +check_sig 'FOO=1 BAR=2 timeout --foreground 5m npm run test:unit -- --watch' 'npm run test:unit' +check_sig 'git commit -m "fix: the thing"' 'git commit' +check_sig 'git checkout feature/secret-branch' 'git checkout' +check_sig 'git checkout my-branch' 'git checkout' +check_sig 'docker logs devscope-backend 2>&1 | tail -20' 'docker logs' +check_sig 'gh pr view 93' 'gh pr view' +check_sig 'echo ===; cargo test --release' 'cargo test' +check_sig 'python3 - < notes.md < "$T" +BG="$XDG_CACHE_HOME/devscope/bg" + +post() { # json -> run tool-complete.sh, print the payload it sent + reset_hits + printf '%s' "$1" | "$ROOT/scripts/tool-complete.sh" >/dev/null 2>&1 || true + events_of tool.complete | tail -1 | jq -c .payload +} +base() { # command extra-jq + jq -cn --arg s "$SID" --arg t "$T" --arg cwd "$TMP" --arg c "$1" \ + "{session_id: \$s, transcript_path: \$t, cwd: \$cwd, hook_event_name: \"PostToolUse\", + tool_name: \"Bash\", tool_input: {command: \$c}, tool_use_id: \"toolu_fg1\"} ${2:-}" +} +respond '{"ok": true}' + +p=$(post "$(base 'cd app && bun test' '+ {duration_ms: 4321}')") +[ "$(jq -r .duration <<<"$p")" = 4321 ] && ok "uses Claude Code's duration_ms" || bad "duration_ms" "$p" +[ "$(jq -r .toolUseId <<<"$p")" = toolu_fg1 ] && ok "sends toolUseId" || bad "toolUseId" "$p" +[ "$(jq -r .commandSignature <<<"$p")" = "bun test" ] && ok "sends the command signature" || bad "signature" "$p" +[ "$(jq -r .toolSubcommand <<<"$p")" = "cd" ] && ok "toolSubcommand is unchanged (first word)" || bad "toolSubcommand" "$p" +[ "$(jq 'has("backgroundTaskId")' <<<"$p")" = false ] && [ ! -e "$BG/$SID.active" ] \ + && ok "a foreground command is not marked as background" || bad "fg bg" "$p" + +# Fallback timer for Claude Code versions without duration_ms: keyed by +# tool_use_id, so a parallel call of the same tool does not overwrite it. +pre() { jq -cn --arg s "$SID" --arg cwd "$TMP" --arg id "$1" \ + '{session_id: $s, cwd: $cwd, tool_name: "Bash", tool_input: {command: "sleep 1"}, tool_use_id: $id}' \ + | "$ROOT/scripts/tool-use.sh" >/dev/null 2>&1; } +pre toolu_p1; sleep 0.3; pre toolu_p2 +p=$(post "$(base 'sleep 1' '| .tool_use_id = "toolu_p1"')") +d=$(jq -r .duration <<<"$p") +[ "$d" -ge 250 ] 2>/dev/null && ok "fallback timer survives a parallel call of the same tool (${d} ms)" || bad "parallel timer" "$p" +[ -f "$HOME/.cache/devscope/timings/toolu_p2" ] && [ ! -f "$HOME/.cache/devscope/timings/toolu_p1" ] \ + && ok "each call's start file is its own" || bad "timing files" "$(ls "$HOME/.cache/devscope/timings")" + +p=$(post "$(base 'bun run dev' '| .tool_use_id = "toolu_bg1" | .tool_input.run_in_background = true + | .tool_response = {stdout: "", backgroundTaskId: "b1"} | .duration_ms = 12')") +[ "$(jq -c '[.backgroundTaskId, .runInBackground, .duration]' <<<"$p")" = '["b1",true,12]' ] \ + && ok "a background command carries its task id" || bad "bg payload" "$p" +[ -e "$BG/$SID.active" ] && ok "and marks the session for the background scan" || bad "bg marker" "missing" +p=$(post "$(base 'bun test' '| .tool_use_id = "toolu_bg2" | .tool_response = {backgroundTaskId: "b2", timedOutAfterMs: 120000}')") +[ "$(jq -c '[.backgroundTaskId, .runInBackground]' <<<"$p")" = '["b2",false]' ] \ + && ok "a command moved to the background after its timeout is told apart" || bad "moved bg" "$p" + +# --- background completions -------------------------------------------------- +note() { # tool-use-id status summary-tail [ts] [operation] + local body + body=$(printf '\nt-%s\n%s\n/tmp/claude-1000/x/tasks/t.output\n%s\n%s\n' "$1" "$1" "$2" "$3") + jq -cn --arg b "$body" --arg ts "${4:-2026-11-05T10:00:00.000Z}" --arg op "${5:-enqueue}" --arg s "$SID" \ + '{type: "queue-operation", operation: $op, timestamp: $ts, sessionId: $s, content: $b}' +} +{ + note toolu_bg1 completed 'Background command "Run dev server, exit code 1 means port taken" completed (exit code 0)' 2026-11-05T10:01:00.000Z + note toolu_bg2 failed 'Background command "Tests" failed with exit code 1' 2026-11-05T10:02:00.000Z + note toolu_bg1 completed 'Background command "Run dev server" completed (exit code 0)' 2026-11-05T10:03:00.000Z remove + note toolu_ag completed 'Agent "Explore" finished' 2026-11-05T10:04:00.000Z + note toolu_bg3 killed 'Background command "Wait for it" was stopped' 2026-11-05T10:05:00.000Z + echo '{"type":"user","message":{"role":"user","content":" in a prompt, not a queue op"}}' + printf '%s' '{"type":"queue-operation","operation":"enqueue"' # half-written line +} > "$T" + +stop() { # hook script event + reset_hits + jq -cn --arg s "$SID" --arg t "$T" --arg cwd "$TMP" --arg e "$2" \ + '{session_id: $s, transcript_path: $t, cwd: $cwd, hook_event_name: $e, last_assistant_message: "done", reason: "other"}' \ + | "$ROOT/scripts/$1" >/dev/null 2>&1 || true +} +stop response-stop.sh Stop +c=$(events_of command.background_complete | jq -c '.payload.commands') +[ "$(jq -c '[.[] | [.toolUseId, .status, .exitCode, .finishedAt]]' <<<"$c")" = \ + '[["toolu_bg1","completed",0,"2026-11-05T10:01:00.000Z"],["toolu_bg2","failed",1,"2026-11-05T10:02:00.000Z"],["toolu_bg3","killed",null,"2026-11-05T10:05:00.000Z"]]' ] \ + && ok "Stop reports settled shells once each, not agents or removals; exit code from the suffix" || bad "bg report" "$c" +[ "$(jq -r '.[0].taskId' <<<"$c")" = "t-toolu_bg1" ] && ok "with the task id" || bad "task id" "$c" +[ "$(jq -c '[.[] | keys] | add | unique' <<<"$c")" = '["exitCode","finishedAt","status","taskId","toolUseId"]' ] \ + && ok "and nothing else (no description, no output path)" || bad "fields" "$c" +[ "$(events_of response.complete | wc -l | tr -d ' ')" = 1 ] && ok "Stop still sends response.complete" || bad "response" "$(paths)" + +stop response-stop.sh Stop +[ -z "$(events_of command.background_complete)" ] && ok "a second Stop sends nothing already reported" || bad "dedupe" "$(events_of command.background_complete)" + +echo ',"timestamp":"2026-11-05T10:06:00.000Z","content":"no notification"}' >> "$T" # completes the half line +note toolu_bg4 completed 'Background command "Build" completed (exit code 2: No matches found)' 2026-11-05T10:07:00.000Z >> "$T" +stop response-stop.sh Stop +c=$(events_of command.background_complete | jq -c '[.payload.commands[] | [.toolUseId, .exitCode]]') +[ "$c" = '[["toolu_bg4",2]]' ] && ok "incremental: only what was appended, after the half-written line" || bad "incremental" "$c" + +stop session-end.sh SessionEnd +[ ! -e "$BG/$SID.active" ] && [ -e "$BG/$SID.offset" ] \ + && ok "SessionEnd drops the marker, keeps the offset (a Stop still running must not rescan)" || bad "cleanup" "$(ls "$BG")" +[ "$(events_of session.end | wc -l | tr -d ' ')" = 1 ] && ok "SessionEnd still sends session.end" || bad "end" "$(paths)" + +note toolu_bg5 completed 'Background command "x" completed (exit code 0)' >> "$T" +stop response-stop.sh Stop +[ -z "$(events_of command.background_complete)" ] && ok "sessions without a background command never scan" || bad "unmarked" "$(events_of command.background_complete)" + +[ "$(python3 "$ROOT/scripts/bg_commands.py" scan "$TMP/missing.jsonl")" = "[]" ] && ok "missing transcript gives []" || bad "missing" "x" + +# More than 200 settle in one turn: every one is reported, in full batches. +SID2="bbbbbbbb-1111-2222-3333-444444444444"; T2="$PROJ/$SID2.jsonl" +for i in $(seq 1 205); do SID=$SID2 note "toolu_many_$i" completed 'Background command "x" completed (exit code 0)'; done > "$T2" +touch "$BG/$SID2.active" +reset_hits +jq -cn --arg s "$SID2" --arg t "$T2" --arg cwd "$TMP" '{session_id: $s, transcript_path: $t, cwd: $cwd, hook_event_name: "Stop", last_assistant_message: "x"}' \ + | "$ROOT/scripts/response-stop.sh" >/dev/null 2>&1 || true +sizes=$(events_of command.background_complete | jq -c '.payload.commands | length' | paste -sd, -) +total=$(events_of command.background_complete | jq -r '.payload.commands[].toolUseId' | sort -u | wc -l | tr -d ' ') +[ "$sizes" = "200,5" ] && [ "$total" = 205 ] && ok "past 200, the rest follows in a second batch" || bad "cap" "sizes=$sizes total=$total" + +echo "---"; echo "$pass passed, $fail failed" +[ "$fail" -eq 0 ] diff --git a/tests/lib/stub.sh b/tests/lib/stub.sh index afb0eb7..c186428 100644 --- a/tests/lib/stub.sh +++ b/tests/lib/stub.sh @@ -16,7 +16,8 @@ respond() { # body [content-type] [status] return 0 } slow() { touch "$STUB_DIR/slow"; } -reset_hits() { : > "$STUB_DIR/hits"; : > "$STUB_DIR/paths"; } +reset_hits() { : > "$STUB_DIR/hits"; : > "$STUB_DIR/paths"; : > "$STUB_DIR/all.jsonl"; } +events_of() { jq -c --arg t "$1" 'select(.path == "/api/events" and .body.eventType == $t) | .body' "$STUB_DIR/all.jsonl" 2>/dev/null; } # eventType paths() { cat "$STUB_DIR/paths" 2>/dev/null; } hits() { [ -f "$STUB_DIR/hits" ] && wc -c < "$STUB_DIR/hits" | tr -d ' ' || echo 0; } last() { jq -r "$1" "$STUB_DIR/last"; } diff --git a/tests/lib/stub_server.py b/tests/lib/stub_server.py index b5bb33b..94fafd6 100644 --- a/tests/lib/stub_server.py +++ b/tests/lib/stub_server.py @@ -5,7 +5,8 @@ Answers every request with the body in DIR/resp (sleeping 5 s first if DIR/slow exists), appends one byte per request to DIR/hits, and writes the last request's path, API key and JSON body to DIR/last (and per path to -DIR/last), and appends each path to DIR/paths. DIR/status and DIR/ctype, when present, override the +DIR/last), and appends each path to DIR/paths and each +request to DIR/all.jsonl. DIR/status and DIR/ctype, when present, override the response status (200) and content type (application/json). POSTs without the x-requested-with header get 403, like the backend's CSRF middleware (which exempts /api/events). @@ -35,6 +36,8 @@ def handle_any(self, method): open(os.path.join(d, "paths"), "a").write(self.path + "\n") req = {"method": method, "path": self.path, "key": self.headers.get("x-api-key"), "body": json.loads(raw) if raw else None} + with open(os.path.join(d, "all.jsonl"), "a") as f: + f.write(json.dumps(req) + "\n") for name in ("last", "last" + self.path.split("?")[0].replace("/", "_")): with open(os.path.join(d, name), "w") as f: json.dump(req, f)