diff --git a/README.md b/README.md index 3172bf2..43b888d 100644 --- a/README.md +++ b/README.md @@ -178,7 +178,7 @@ bd create "Fix the login timeout" --description "..." # Shep (the Grok bot) - **Prompt** — one line pointing at the issue (`Your job is beads issue . Run bd show …`), the same for every harness. - **Context** — before it starts the agent, bercail adds a `bd comment` to the issue with the worktree and, headless, the result file and session id. -- **Result** — a headless run leaves `{status: ready|blocked|failed, summary, session_id}`. Read that file, not terminal output. `--resume SESSION` continues the same session (`claude --resume`, `codex exec resume`, `cursor-agent --resume`, `opencode run --session`). +- **Result** — a headless run leaves `{status: ready|blocked|failed, summary, session_id}`. A plan usage limit also sets `reason` to `usage-limit`, so Shep can propose another plan. Bercail does not switch or rerun. Read that file, not terminal output. `--resume SESSION` continues the same session (`claude --resume`, `codex exec resume`, `cursor-agent --resume`, `opencode run --session`). - **Blocked, not switched** — if the stage's binary, `bd`, or the issue is missing, the result is `blocked` and nothing starts. Bercail never falls through to another harness. `--binary claude|codex|cursor-agent|opencode` runs a headless stage on another CLI only when you ask (with `opencode`, `start` stays on Claude and `codex` on GPT, through your opencode plans). - **Dry run** — `--dry-run` prints the exact argv for any stage, even when that binary is not installed. - `bercail harness` reports which of claude, codex, cursor-agent, and opencode are on PATH, with a local model hint. It does not choose. diff --git a/docs/bstack/07-handoff.md b/docs/bstack/07-handoff.md index 686f50d..37e6a94 100644 --- a/docs/bstack/07-handoff.md +++ b/docs/bstack/07-handoff.md @@ -19,7 +19,7 @@ spec. The agent's prompt is one line that points at it. review it in that space's tuicr. - **No branch:** a headless `opencode run --agent bstack`. It writes `.bercail/handoff-result.json` (`status` ready, blocked, or failed, a summary, - the session id) and nothing opens. + the session id; `reason` is `usage-limit` when a plan is exhausted) and nothing opens. It edits and stages but can't commit, push, or merge: a headless run rejects anything that would ask. Stage `autopilot` is the exception (see [Ship with autopilot](./06-autopilot.md)). diff --git a/skills/handoff/SKILL.md b/skills/handoff/SKILL.md index eb99d47..bbe0b0a 100644 --- a/skills/handoff/SKILL.md +++ b/skills/handoff/SKILL.md @@ -80,7 +80,9 @@ If the requested harness, bd, or the issue is missing, the stage writes `/.bercail/handoff-result.json` with status `blocked`, starts nothing, and exits nonzero. It does not fall through to another harness. The agent writes the same result file (`status`, `summary`, and bercail fills -`session_id`). Read that file. Do not scrape terminal output. For Codex, +`session_id`). When the run stops on a plan usage limit, bercail adds `reason`: +`usage-limit`. It does not rerun or switch plans. Read that file. Do not scrape +terminal output. For Codex, `session_id` is the JSONL `thread.started` thread id. For cursor-agent, it is the print JSON `session_id`. When there is no resumable session (blocked before codex or cursor-agent ran, or no id in its log), `session_id` is `none`; diff --git a/skills/handoff/scripts/handoff-stages.sh b/skills/handoff/scripts/handoff-stages.sh index 1c39105..73545b9 100644 --- a/skills/handoff/scripts/handoff-stages.sh +++ b/skills/handoff/scripts/handoff-stages.sh @@ -106,14 +106,18 @@ handoff_issue_prompt() { printf 'Your job is beads issue %s. Run `bd show %s` (read its comments too) and do the work it describes.' "$1" "$1" } +# reason is optional. Empty omits the field; Shep reads it, and unknown fields +# do not fail handoff_result_ok. The only value the runner writes is usage-limit. handoff_write_result() { - local dest="$1" status="$2" summary="$3" session="$4" + local dest="$1" status="$2" summary="$3" session="$4" reason="${5:-}" mkdir -p "$(dirname "$dest")" jq -nc \ --arg status "$status" \ --arg summary "$summary" \ --arg session_id "$session" \ - '{status:$status, summary:$summary, session_id:$session_id}' >"$dest" + --arg reason "$reason" \ + '{status:$status, summary:$summary, session_id:$session_id} + + if $reason == "" then {} else {reason:$reason} end' >"$dest" } handoff_result_ok() { @@ -270,6 +274,42 @@ handoff_opencode_session_id() { printf '%s' "$id" } +# usage-limit when LOG is a plan limit, else nothing. Prints the reason; does not +# rerun. ChatGPT is the session error type provider.chatgpt-limit, or the code +# subscription_sharing_usage_limit_exceeded in that error's body. Claude is a +# sentence Claude Code 2.1.316 composes ("You've hit your 5-hour limit", the +# weekly, Opus, Sonnet, and Fable lines, "You're out of usage credits") or the +# recorded extra-usage billing sentence. That sentence is read from the error +# event, or from stderr when the process exits before one. Other JSON is not a +# limit. A provider.rate-limit, including a bare HTTP 429, is not one: xAI's +# weekly-limit body is unverified. +handoff_usage_limit_reason() { + local log="$1" events + local phrase="You've hit your (5-hour|weekly|Opus|Sonnet|Fable) limit|You've hit your usage limit|You've hit your limit|You've reached your Fable limit|You're out of usage credits|Third-party apps now draw from your extra usage, not your plan limits" + [[ -f "$log" ]] || return 0 + events="$(jq -c -R 'fromjson? | select(type == "object" and .type == "error" and (.error | type == "object")) | .error' "$log" 2>/dev/null || true)" + # A JSON error event decides. Stderr counts only when the process exits first. + if [[ -n "$events" ]]; then + if jq -e -s --arg phrase "$phrase" ' + def hit: test($phrase); + any(.[]; + .type == "provider.chatgpt-limit" + or ((.response.body // "") | if type == "string" then contains("subscription_sharing_usage_limit_exceeded") else false end) + or ((.message // "") | hit) + or ((.response.body // "") | if type == "string" then hit else false end) + ) + ' <<<"$events" >/dev/null 2>&1; then + printf 'usage-limit' + fi + return 0 + fi + # grep reads to the end (no -q): with pipefail, an early exit would SIGPIPE jq + # on a long log and hide the match. + if jq -R -r 'select(try (fromjson | type == "object") catch false | not)' "$log" | grep -E "$phrase" >/dev/null; then + printf 'usage-limit' + fi +} + handoff_is_opencode_session() { [[ "$1" =~ ^ses_[A-Za-z0-9]+$ ]] } @@ -459,7 +499,7 @@ handoff_headless_comment_text() { handoff_run_headless() { local stage="$1" worktree="$2" issue="$3" resume="$4" dry="$5" binary_override="${6:-}" local mode binary native model_spec model session="" resumed=0 prompt bd beads_dir="" - local result log bin cmd_json out rc=0 tid present=false bd_present=false + local result log bin cmd_json out rc=0 tid present=false bd_present=false reason="" command -v jq >/dev/null 2>&1 || handoff_die "jq is required" mode="$(handoff_stage_field "$stage" mode)" || handoff_die "unknown stage: $stage" @@ -593,12 +633,17 @@ handoff_run_headless() { session="$tid" out="$(jq --arg session_id "$session" '.session_id = $session_id' <<<"$out")" fi + # The exhausted plan cannot be trusted to name its own limit. Classify the CLI + # log and stamp reason; do not start another command. + if [[ "$rc" -ne 0 ]]; then + reason="$(handoff_usage_limit_reason "$log")" + fi # bercail knows the session id; the agent only has to write status and summary. # Keep the agent's own uuid if the CLI log had none. Never invent one: a # made-up id would look resumable. Say so in the summary instead. if [[ -f "$result" ]] && jq -e 'type == "object"' "$result" >/dev/null 2>&1; then local filled - filled="$(jq --arg session_id "$session" --arg none "$HANDOFF_NO_SESSION" ' + filled="$(jq --arg session_id "$session" --arg none "$HANDOFF_NO_SESSION" --arg reason "$reason" ' (.session_id | type == "string" and test("^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$")) as $own | .status = (if (.status == "ready" or .status == "blocked" or .status == "failed") then .status else "failed" end) | .summary = (if (.summary | type == "string" and length > 0) then .summary else "headless stage finished without a usable summary" end) @@ -606,13 +651,14 @@ handoff_run_headless() { elif $own then . else .session_id = $none | .summary += " (bercail found no session id in the CLI log, so this run cannot be resumed.)" end + | if $reason != "" then .reason = $reason else del(.reason) end ' "$result")" printf '%s\n' "$filled" >"$result" fi if [[ ! -f "$result" ]] || ! handoff_result_ok "$result"; then [[ "$rc" -eq 0 ]] && rc=1 [[ -n "$session" ]] || session="$HANDOFF_NO_SESSION" - handoff_write_result "$result" failed "headless stage exited ${rc} without a result file" "$session" + handoff_write_result "$result" failed "headless stage exited ${rc} without a result file" "$session" "$reason" fi if [[ "$rc" -ne 0 ]]; then printf 'handoff: %s exited %s; result is %s\n' "$bin" "$rc" "$result" >&2 diff --git a/tests/handoff_spawn_test.sh b/tests/handoff_spawn_test.sh index e5cf5b8..2a5ceac 100755 --- a/tests/handoff_spawn_test.sh +++ b/tests/handoff_spawn_test.sh @@ -133,13 +133,11 @@ test_info_json_reports_graphite_and_dirty() { git_init "$repo" printf 'dirty\n' >>"$repo/file.txt" printf 'loose\n' >"$repo/untracked.txt" - mkdir -p "$(dirname "$(git -C "$repo" rev-parse --git-path .graphite_repo_config)")" + # git -C prints the path relative to $repo, not to the cwd. printed="$(git -C "$repo" rev-parse --git-path .graphite_repo_config)" - if [[ "$printed" == /* ]]; then - printf '{}\n' >"$printed" - else - printf '{}\n' >"$repo/$printed" - fi + [[ "$printed" == /* ]] || printed="$repo/$printed" + mkdir -p "$(dirname "$printed")" + printf '{}\n' >"$printed" printf '#!/bin/sh\nexit 1\n' >"$TMP_DIR/no-herdr" chmod +x "$TMP_DIR/no-herdr" unset HC_TERMINAL_ID HANDOFF_SPACE diff --git a/tests/usage_limit_reason_test.sh b/tests/usage_limit_reason_test.sh new file mode 100644 index 0000000..5a8607c --- /dev/null +++ b/tests/usage_limit_reason_test.sh @@ -0,0 +1,169 @@ +#!/usr/bin/env bash +# shellcheck shell=bash +# A stub opencode replays recorded plan-limit errors. No live provider calls. +set -euo pipefail + +unset BASH_ENV +export __MISE_BASH_ENV_LOADED=1 + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +TMP_DIR="$(mktemp -d)" +trap 'rm -rf "$TMP_DIR"' EXIT + +fail() { printf 'FAIL: %s\n' "$*" >&2; exit 1; } + +# shellcheck disable=SC1090 +source "$ROOT/skills/handoff/scripts/handoff-spawn" + +ISSUE=bercail-0s1 + +mkdir -p "$TMP_DIR/bin" +cat >"$TMP_DIR/bin/bd" <<'STUB' +#!/usr/bin/env bash +set -euo pipefail +case "$1" in + show) + [[ "$2" == "${BD_KNOWN:?}" ]] || { echo "no issue $2" >&2; exit 1; } + printf '[{"id":"%s"}]\n' "$2" + ;; + comment) + [[ "${3:-}" == --stdin ]] || exit 3 + cat >/dev/null + ;; + context) + printf '{"beads_dir":"%s"}\n' "$PWD/.beads" + ;; + *) exit 2 ;; +esac +STUB +chmod +x "$TMP_DIR/bin/bd" +export HANDOFF_BD_BIN="$TMP_DIR/bin/bd" +export BD_KNOWN="$ISSUE" + +# Prints $HANDOFF_STUB_PAYLOAD and exits 1. Counts invocations so a limit cannot +# start a second command. +cat >"$TMP_DIR/bin/opencode" <<'STUB' +#!/usr/bin/env bash +set -euo pipefail +printf '%s\n' "$*" >>"${HANDOFF_ARGV_LOG:?}" +printf '1\n' >>"${HANDOFF_RUNS:?}" +cat "${HANDOFF_STUB_PAYLOAD:?}" +exit 1 +STUB +chmod +x "$TMP_DIR/bin/opencode" +export HANDOFF_OPENCODE_BIN="$TMP_DIR/bin/opencode" +export HANDOFF_ARGV_LOG="$TMP_DIR/argv.log" +export HANDOFF_RUNS="$TMP_DIR/runs" +export HANDOFF_STUB_PAYLOAD="$TMP_DIR/payload" + +run_stage() { + local stage="$1" name="$2" rc=0 + local wt="$TMP_DIR/$name" + mkdir -p "$wt" + : >"$HANDOFF_ARGV_LOG" + : >"$HANDOFF_RUNS" + (cd "$wt" && "$ROOT/skills/handoff/scripts/handoff-spawn" \ + --stage "$stage" --issue "$ISSUE" --worktree "$wt" >/dev/null 2>&1) || rc=$? + [[ "$rc" -ne 0 ]] || fail "$name should exit nonzero" + [[ "$(wc -l <"$HANDOFF_RUNS")" -eq 1 ]] || fail "$name started more than once: $(cat "$HANDOFF_RUNS")" + printf '%s' "$wt" +} + +assert_limit() { + local wt="$1" label="$2" + jq -e '.status == "failed" + and .reason == "usage-limit" + and (.summary | type == "string" and length > 0) + and (.session_id | type == "string" and length > 0)' \ + "$wt/.bercail/handoff-result.json" >/dev/null \ + || fail "$label: $(cat "$wt/.bercail/handoff-result.json")" + handoff_result_ok "$wt/.bercail/handoff-result.json" \ + || fail "$label result should still pass handoff_result_ok" +} + +test_chatgpt_limit_sets_reason() { + # Recorded ChatGPT session error: type provider.chatgpt-limit, code in the body. + # Live capture was HTTP 200, not a 429. + jq -nc \ + --arg body '{"error":{"code":"subscription_sharing_usage_limit_exceeded","message":"Limit"}}' \ + '{type:"error", sessionID:"ses_gptlimit", error:{ + type:"provider.chatgpt-limit", + message:"ChatGPT usage limit reached. Check ChatGPT Settings → Usage for details.", + status:200, + response:{body:$body} + }}' >"$HANDOFF_STUB_PAYLOAD" + local wt + wt="$(run_stage bstack chatgpt)" + assert_limit "$wt" "chatgpt" + jq -e '.session_id == "ses_gptlimit"' "$wt/.bercail/handoff-result.json" >/dev/null \ + || fail "chatgpt session: $(cat "$wt/.bercail/handoff-result.json")" + wt="$(run_stage autopilot chatgpt-autopilot)" + assert_limit "$wt" "autopilot" + printf 'PASS: ChatGPT usage-limit error yields failed with reason usage-limit\n' +} + +test_claude_text_sets_reason() { + # The sentence Claude Code 2.1.316 composes, as opencode's plain error, and as + # stderr when the process exits before a JSON error event. + jq -nc '{type:"error", sessionID:"ses_claude", error:{ + type:"provider.unknown", + message:"You'\''ve hit your 5-hour limit · resets 6:10pm" + }}' >"$HANDOFF_STUB_PAYLOAD" + local wt + wt="$(run_stage bstack claude-json)" + assert_limit "$wt" "claude json" + printf '%s\n' "You've hit your 5-hour limit" >"$HANDOFF_STUB_PAYLOAD" + wt="$(run_stage bstack claude-text)" + assert_limit "$wt" "claude text" + printf 'PASS: Claude 5-hour limit text yields failed with reason usage-limit\n' +} + +test_rate_limit_has_no_reason() { + # A transient 429. Status alone is not a plan limit (xAI's weekly body is unverified). + jq -nc \ + --arg body '{"error":{"type":"rate_limit_error","message":"Too many requests"}}' \ + '{type:"error", sessionID:"ses_rate", error:{ + type:"provider.rate-limit", + message:"Too many requests", + status:429, + response:{body:$body} + }}' >"$HANDOFF_STUB_PAYLOAD" + local wt + wt="$(run_stage bstack rate)" + jq -e '.status == "failed" + and (.summary | type == "string" and length > 0) + and .session_id == "ses_rate" + and (has("reason") | not)' \ + "$wt/.bercail/handoff-result.json" >/dev/null \ + || fail "rate limit must not set reason: $(cat "$wt/.bercail/handoff-result.json")" + handoff_result_ok "$wt/.bercail/handoff-result.json" \ + || fail "rate-limit result should still pass handoff_result_ok" + + # A mention in other JSON, or a bare 429 with no code, is not a limit. + printf '%s\n' '{"type":"text","text":"subscription_sharing_usage_limit_exceeded"}' >"$TMP_DIR/mention.log" + [[ -z "$(handoff_usage_limit_reason "$TMP_DIR/mention.log")" ]] \ + || fail "a mention of the code is not a usage limit" + printf '%s\n' '{"type":"text","text":"You'"'"'ve hit your 5-hour limit"}' >"$TMP_DIR/quote.log" + [[ -z "$(handoff_usage_limit_reason "$TMP_DIR/quote.log")" ]] \ + || fail "a quoted phrase in other JSON is not a usage limit" + printf '%s\n' ' {"type":"text","text":"You'"'"'ve hit your 5-hour limit"}' >"$TMP_DIR/indented.log" + [[ -z "$(handoff_usage_limit_reason "$TMP_DIR/indented.log")" ]] \ + || fail "indented JSON is not stderr" + jq -nc '{type:"error", error:{type:"provider.unknown", message:"You'\''re out of usage credits"}}' >"$TMP_DIR/credits.log" + [[ "$(handoff_usage_limit_reason "$TMP_DIR/credits.log")" == usage-limit ]] \ + || fail "out of usage credits should be a plan limit" + jq -nc '{type:"error", error:{type:"provider.unknown", message:"HTTP 429", status:429}}' >"$TMP_DIR/bare429.log" + [[ -z "$(handoff_usage_limit_reason "$TMP_DIR/bare429.log")" ]] \ + || fail "status 429 alone is not a usage limit" + # The error event decides. A phrase on another line does not override a 429. + printf '%s\n' \ + '{"type":"error","error":{"type":"provider.rate-limit","message":"Too many requests","status":429}}' \ + "You've hit your 5-hour limit" >"$TMP_DIR/mixed.log" + [[ -z "$(handoff_usage_limit_reason "$TMP_DIR/mixed.log")" ]] \ + || fail "a rate-limit event must not pick up a phrase elsewhere" + printf 'PASS: a plain 429 yields no usage-limit reason\n' +} + +test_chatgpt_limit_sets_reason +test_claude_text_sets_reason +test_rate_limit_has_no_reason