diff --git a/CHANGELOG.md b/CHANGELOG.md index 84d9bdd8..8c717439 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -202,6 +202,16 @@ Changes are grouped by software version. Entries under Unreleased describe work - The AHE Adapter Evolver took a `change_manifest.json` an earlier step left for the current step's: a manifest naming the next iteration was refused at its own step, then archived as the next step's when that step's session wrote none. The file (or a directory a session made in its place) is now removed before each Evolve session. +- Nine example launchers left their wire meters running after the run (#82), so a later run on the same port + reused one and wrote its records into the earlier run's ledger. The meters a launch starts now stop within a second + or two of the runs that use them ending: its own trainings (and, for ServeLearnBench, the tests after them) and + those of any launch that reuses one, which each list their processes in the port's users file + (`/tmp/hermit-wire-meter--.users`); the harness2 restart loop is stopped with its meter. A + launch reuses a running meter only if it is still up three seconds on, so it never takes one that is being + stopped. A meter started on a ledger an earlier one left now carries its totals on (`LEDGER_RESUME=1` in + `hermit/runtime/wire_meter.py`, which these launchers set): the next run's meter on the port (`launch.sh run && + launch.sh run eval`, a ServeLearnBench `test` run again, a CL-Bench tag refused because it exists) wrote an empty + ledger over the earlier run's bill. - On macOS a CLI session stopped at its timeout or on cancel could leave processes behind or fail: the processes below it were found through `/proc`, which macOS lacks, so one in a session of its own (as a sandbox starts its program) was never stopped; and signalling a process group left with only exited, unreaped processes raises diff --git a/examples/clbench_database/mwm/launch.sh b/examples/clbench_database/mwm/launch.sh index 7741cfac..067133f3 100755 --- a/examples/clbench_database/mwm/launch.sh +++ b/examples/clbench_database/mwm/launch.sh @@ -78,14 +78,45 @@ fi command -v codex >/dev/null || { echo "REFUSING: codex CLI not on PATH"; exit 1; } [ -f "$TASKDIR/tasks.jsonl" ] || { echo "REFUSING: run $TASKDIR/prepare.py first"; exit 1; } +# The meters this launch starts stop within a second or two of the processes that use them ending (#82): its own and +# those of any launch that reuses one. Each launch on a port lists its processes in the port's users file, emptied +# when the meter stops; a launch reuses a running meter only if it is still up on a second look (one being stopped is +# gone by then). +METERS=() OWNED=() PORTS=() +meter_users() { echo "/tmp/hermit-wire-meter-$(id -u)-$1.users"; } # port: /tmp, which every launch on it shares +meter_started() { # port pid: a meter this launch started, this launch one of its users + { echo $$ >> "$(meter_users "$1")" || true; } 2>/dev/null + METERS+=("$2"); OWNED+=("$1") +} +stop_meters_after() { # pid...: what this launch leaves running; the meters it started stop after every user + local port files=() + [ $# -eq 0 ] || for port in ${PORTS[@]+"${PORTS[@]}"}; do + printf '%s\n' "$@" >> "$(meter_users "$port")" 2>/dev/null || true; done + [ ${#METERS[@]} -gt 0 ] || return 0 + for port in "${OWNED[@]}"; do files+=("$(meter_users "$port")"); done + setsid nohup bash -c 'while :; do up= + for p in $(cat $1 2>/dev/null); do kill -0 "$p" 2>/dev/null && { up=1; break; }; done + [ -n "$up" ] || break; sleep 1; done + for f in $1; do : > "$f"; done 2>/dev/null # a launch listed after this look finds the meter gone and starts one + for m in $2; do kill -- "-$m" 2>/dev/null || kill "$m" 2>/dev/null || true; done' _ "${files[*]}" "${METERS[*]}" \ + > /dev/null 2>&1 < /dev/null & +} +meter_health() { # port: the /health of a meter still up three seconds on, this launch listed as one of its users first + curl -s -m 2 "http://127.0.0.1:$1/health" > /dev/null 2>&1 \ + && { echo $$ >> "$(meter_users "$1")" || true; } 2>/dev/null \ + && sleep 3 && curl -s -m 2 "http://127.0.0.1:$1/health" 2>/dev/null +} +trap 'stop_meters_after ${PIDS[@]+"${PIDS[@]}"}' EXIT # after the trainings started below, if any start_meter() { # port upstream ledger log [extra env...]: a meter already on the port must face the same upstream local port=$1 upstream=$2 ledger=$3 log=$4 health; shift 4 - if health=$(curl -s -m 2 "http://127.0.0.1:$port/health" 2>/dev/null); then + PORTS+=("$port") # started here or reused: this launch's processes use it + if health=$(meter_health $port); then case "$health" in *"\"upstream\": \"$upstream\""*) echo "wire meter already on port $port -> $upstream";; *) echo "REFUSING: port $port has a wire meter facing another or an unknown upstream ($health); choose another WM_PORT"; exit 1;; esac else - env "$@" PORT=$port LEDGER="$ledger" UPSTREAM="$upstream" \ + env "$@" PORT=$port LEDGER="$ledger" LEDGER_RESUME=1 UPSTREAM="$upstream" \ setsid nohup "$PY" "$METER" > "$log" 2>&1 < /dev/null & + meter_started $port $! echo "wire meter pid $! port $port -> $upstream"; sleep 2 fi } diff --git a/examples/clbench_database/prompt_only/launch.sh b/examples/clbench_database/prompt_only/launch.sh index cc2316b1..99cc5904 100755 --- a/examples/clbench_database/prompt_only/launch.sh +++ b/examples/clbench_database/prompt_only/launch.sh @@ -82,14 +82,45 @@ fi command -v codex >/dev/null || { echo "REFUSING: codex CLI not on PATH"; exit 1; } [ -f "$TASKDIR/tasks.jsonl" ] || { echo "REFUSING: run $TASKDIR/prepare.py first"; exit 1; } +# The meters this launch starts stop within a second or two of the processes that use them ending (#82): its own and +# those of any launch that reuses one. Each launch on a port lists its processes in the port's users file, emptied +# when the meter stops; a launch reuses a running meter only if it is still up on a second look (one being stopped is +# gone by then). +METERS=() OWNED=() PORTS=() +meter_users() { echo "/tmp/hermit-wire-meter-$(id -u)-$1.users"; } # port: /tmp, which every launch on it shares +meter_started() { # port pid: a meter this launch started, this launch one of its users + { echo $$ >> "$(meter_users "$1")" || true; } 2>/dev/null + METERS+=("$2"); OWNED+=("$1") +} +stop_meters_after() { # pid...: what this launch leaves running; the meters it started stop after every user + local port files=() + [ $# -eq 0 ] || for port in ${PORTS[@]+"${PORTS[@]}"}; do + printf '%s\n' "$@" >> "$(meter_users "$port")" 2>/dev/null || true; done + [ ${#METERS[@]} -gt 0 ] || return 0 + for port in "${OWNED[@]}"; do files+=("$(meter_users "$port")"); done + setsid nohup bash -c 'while :; do up= + for p in $(cat $1 2>/dev/null); do kill -0 "$p" 2>/dev/null && { up=1; break; }; done + [ -n "$up" ] || break; sleep 1; done + for f in $1; do : > "$f"; done 2>/dev/null # a launch listed after this look finds the meter gone and starts one + for m in $2; do kill -- "-$m" 2>/dev/null || kill "$m" 2>/dev/null || true; done' _ "${files[*]}" "${METERS[*]}" \ + > /dev/null 2>&1 < /dev/null & +} +meter_health() { # port: the /health of a meter still up three seconds on, this launch listed as one of its users first + curl -s -m 2 "http://127.0.0.1:$1/health" > /dev/null 2>&1 \ + && { echo $$ >> "$(meter_users "$1")" || true; } 2>/dev/null \ + && sleep 3 && curl -s -m 2 "http://127.0.0.1:$1/health" 2>/dev/null +} +trap 'stop_meters_after ${PIDS[@]+"${PIDS[@]}"}' EXIT # after the trainings started below, if any start_meter() { # port upstream ledger log [extra env...]: a meter already on the port must face the same upstream local port=$1 upstream=$2 ledger=$3 log=$4 health; shift 4 - if health=$(curl -s -m 2 "http://127.0.0.1:$port/health" 2>/dev/null); then + PORTS+=("$port") # started here or reused: this launch's processes use it + if health=$(meter_health $port); then case "$health" in *"\"upstream\": \"$upstream\""*) echo "wire meter already on port $port -> $upstream";; *) echo "REFUSING: port $port has a wire meter facing another or an unknown upstream ($health); choose another WM_PORT"; exit 1;; esac else - env "$@" PORT=$port LEDGER="$ledger" UPSTREAM="$upstream" \ + env "$@" PORT=$port LEDGER="$ledger" LEDGER_RESUME=1 UPSTREAM="$upstream" \ setsid nohup "$PY" "$METER" > "$log" 2>&1 < /dev/null & + meter_started $port $! echo "wire meter pid $! port $port -> $upstream"; sleep 2 fi } diff --git a/examples/clbench_database/reharness/launch.sh b/examples/clbench_database/reharness/launch.sh index 8fb32079..2b9bee18 100755 --- a/examples/clbench_database/reharness/launch.sh +++ b/examples/clbench_database/reharness/launch.sh @@ -90,14 +90,45 @@ fi [ -f "$TASKDIR/tasks.jsonl" ] || { echo "REFUSING: run $TASKDIR/prepare.py first"; exit 1; } +# The meters this launch starts stop within a second or two of the processes that use them ending (#82): its own and +# those of any launch that reuses one. Each launch on a port lists its processes in the port's users file, emptied +# when the meter stops; a launch reuses a running meter only if it is still up on a second look (one being stopped is +# gone by then). +METERS=() OWNED=() PORTS=() +meter_users() { echo "/tmp/hermit-wire-meter-$(id -u)-$1.users"; } # port: /tmp, which every launch on it shares +meter_started() { # port pid: a meter this launch started, this launch one of its users + { echo $$ >> "$(meter_users "$1")" || true; } 2>/dev/null + METERS+=("$2"); OWNED+=("$1") +} +stop_meters_after() { # pid...: what this launch leaves running; the meters it started stop after every user + local port files=() + [ $# -eq 0 ] || for port in ${PORTS[@]+"${PORTS[@]}"}; do + printf '%s\n' "$@" >> "$(meter_users "$port")" 2>/dev/null || true; done + [ ${#METERS[@]} -gt 0 ] || return 0 + for port in "${OWNED[@]}"; do files+=("$(meter_users "$port")"); done + setsid nohup bash -c 'while :; do up= + for p in $(cat $1 2>/dev/null); do kill -0 "$p" 2>/dev/null && { up=1; break; }; done + [ -n "$up" ] || break; sleep 1; done + for f in $1; do : > "$f"; done 2>/dev/null # a launch listed after this look finds the meter gone and starts one + for m in $2; do kill -- "-$m" 2>/dev/null || kill "$m" 2>/dev/null || true; done' _ "${files[*]}" "${METERS[*]}" \ + > /dev/null 2>&1 < /dev/null & +} +meter_health() { # port: the /health of a meter still up three seconds on, this launch listed as one of its users first + curl -s -m 2 "http://127.0.0.1:$1/health" > /dev/null 2>&1 \ + && { echo $$ >> "$(meter_users "$1")" || true; } 2>/dev/null \ + && sleep 3 && curl -s -m 2 "http://127.0.0.1:$1/health" 2>/dev/null +} +trap 'stop_meters_after ${PIDS[@]+"${PIDS[@]}"}' EXIT # after the trainings started below, if any start_meter() { # port upstream ledger log [extra env...]: a meter already on the port must face the same upstream local port=$1 upstream=$2 ledger=$3 log=$4 health; shift 4 - if health=$(curl -s -m 2 "http://127.0.0.1:$port/health" 2>/dev/null); then + PORTS+=("$port") # started here or reused: this launch's processes use it + if health=$(meter_health $port); then case "$health" in *"\"upstream\": \"$upstream\""*) echo "wire meter already on port $port -> $upstream";; *) echo "REFUSING: port $port has a wire meter facing another or an unknown upstream ($health); choose another WM_PORT"; exit 1;; esac else - env "$@" PORT=$port LEDGER="$ledger" UPSTREAM="$upstream" \ + env "$@" PORT=$port LEDGER="$ledger" LEDGER_RESUME=1 UPSTREAM="$upstream" \ setsid nohup "$PY" "$METER" > "$log" 2>&1 < /dev/null & + meter_started $port $! echo "wire meter pid $! port $port -> $upstream"; sleep 2 curl -s -m 2 "http://127.0.0.1:$port/health" > /dev/null \ || { echo "REFUSING: the wire meter on port $port did not come up (see $log)"; exit 1; } @@ -108,7 +139,7 @@ start_meter $WM_PORT "$HOST_ORIGIN" "$RUNS/$TAG/wire_ledger.json" "$RUNS/$TAG/wi start_meter $META_PORT "$EVOLVER_ORIGIN" "$RUNS/$TAG/meta_wire_ledger.json" "$RUNS/$TAG/meta_wire_meter.log" # The embedding relay signs the reader's keyless requests with the embedding backend's key (none for a keyless server); # a relay already on the port is refused unless its /health says it signs (`"signs": true`). -if [ -n "$EMBED_API_KEY_ENV" ] && health=$(curl -s -m 2 "http://127.0.0.1:$EMBED_PORT/health" 2>/dev/null); then +if [ -n "$EMBED_API_KEY_ENV" ] && health=$(meter_health $EMBED_PORT); then case "$health" in *'"signs": true'*) ;; *) echo "REFUSING: port $EMBED_PORT has a relay that does not sign requests (started without UPSTREAM_KEY_ENV); choose another WM_PORT"; exit 1;; esac fi start_meter $EMBED_PORT "$EMBED_ORIGIN" "$RUNS/$TAG/embed_wire_ledger.json" "$RUNS/$TAG/embed_wire_meter.log" \ diff --git a/examples/clbench_poker/mwm/launch.sh b/examples/clbench_poker/mwm/launch.sh index 7be6f2df..c083e428 100755 --- a/examples/clbench_poker/mwm/launch.sh +++ b/examples/clbench_poker/mwm/launch.sh @@ -73,14 +73,45 @@ export MWM_CLBENCH=$CLBENCH MWM_METAWORLD=$METAWORLD command -v codex >/dev/null || { echo "REFUSING: codex CLI not on PATH"; exit 1; } [ -f "$TASKDIR/tasks.jsonl" ] || { echo "REFUSING: run $TASKDIR/prepare.py first"; exit 1; } +# The meters this launch starts stop within a second or two of the processes that use them ending (#82): its own and +# those of any launch that reuses one. Each launch on a port lists its processes in the port's users file, emptied +# when the meter stops; a launch reuses a running meter only if it is still up on a second look (one being stopped is +# gone by then). +METERS=() OWNED=() PORTS=() +meter_users() { echo "/tmp/hermit-wire-meter-$(id -u)-$1.users"; } # port: /tmp, which every launch on it shares +meter_started() { # port pid: a meter this launch started, this launch one of its users + { echo $$ >> "$(meter_users "$1")" || true; } 2>/dev/null + METERS+=("$2"); OWNED+=("$1") +} +stop_meters_after() { # pid...: what this launch leaves running; the meters it started stop after every user + local port files=() + [ $# -eq 0 ] || for port in ${PORTS[@]+"${PORTS[@]}"}; do + printf '%s\n' "$@" >> "$(meter_users "$port")" 2>/dev/null || true; done + [ ${#METERS[@]} -gt 0 ] || return 0 + for port in "${OWNED[@]}"; do files+=("$(meter_users "$port")"); done + setsid nohup bash -c 'while :; do up= + for p in $(cat $1 2>/dev/null); do kill -0 "$p" 2>/dev/null && { up=1; break; }; done + [ -n "$up" ] || break; sleep 1; done + for f in $1; do : > "$f"; done 2>/dev/null # a launch listed after this look finds the meter gone and starts one + for m in $2; do kill -- "-$m" 2>/dev/null || kill "$m" 2>/dev/null || true; done' _ "${files[*]}" "${METERS[*]}" \ + > /dev/null 2>&1 < /dev/null & +} +meter_health() { # port: the /health of a meter still up three seconds on, this launch listed as one of its users first + curl -s -m 2 "http://127.0.0.1:$1/health" > /dev/null 2>&1 \ + && { echo $$ >> "$(meter_users "$1")" || true; } 2>/dev/null \ + && sleep 3 && curl -s -m 2 "http://127.0.0.1:$1/health" 2>/dev/null +} +trap 'stop_meters_after ${PIDS[@]+"${PIDS[@]}"}' EXIT # after the trainings started below, if any start_meter() { # port upstream ledger log [extra env...]: a meter already on the port must face the same upstream local port=$1 upstream=$2 ledger=$3 log=$4 health; shift 4 - if health=$(curl -s -m 2 "http://127.0.0.1:$port/health" 2>/dev/null); then + PORTS+=("$port") # started here or reused: this launch's processes use it + if health=$(meter_health $port); then case "$health" in *"\"upstream\": \"$upstream\""*) echo "wire meter already on port $port -> $upstream";; *) echo "REFUSING: port $port has a wire meter facing another or an unknown upstream ($health); choose another WM_PORT"; exit 1;; esac else - env "$@" PORT=$port LEDGER="$ledger" UPSTREAM="$upstream" \ + env "$@" PORT=$port LEDGER="$ledger" LEDGER_RESUME=1 UPSTREAM="$upstream" \ setsid nohup "$PY" "$METER" > "$log" 2>&1 < /dev/null & + meter_started $port $! echo "wire meter pid $! port $port -> $upstream"; sleep 2 fi } diff --git a/examples/clbench_poker/prompt_only/launch.sh b/examples/clbench_poker/prompt_only/launch.sh index 551de82b..f6485ad0 100755 --- a/examples/clbench_poker/prompt_only/launch.sh +++ b/examples/clbench_poker/prompt_only/launch.sh @@ -77,14 +77,45 @@ export MWM_CLBENCH=$CLBENCH MWM_METAWORLD=$METAWORLD command -v codex >/dev/null || { echo "REFUSING: codex CLI not on PATH"; exit 1; } [ -f "$TASKDIR/tasks.jsonl" ] || { echo "REFUSING: run $TASKDIR/prepare.py first"; exit 1; } +# The meters this launch starts stop within a second or two of the processes that use them ending (#82): its own and +# those of any launch that reuses one. Each launch on a port lists its processes in the port's users file, emptied +# when the meter stops; a launch reuses a running meter only if it is still up on a second look (one being stopped is +# gone by then). +METERS=() OWNED=() PORTS=() +meter_users() { echo "/tmp/hermit-wire-meter-$(id -u)-$1.users"; } # port: /tmp, which every launch on it shares +meter_started() { # port pid: a meter this launch started, this launch one of its users + { echo $$ >> "$(meter_users "$1")" || true; } 2>/dev/null + METERS+=("$2"); OWNED+=("$1") +} +stop_meters_after() { # pid...: what this launch leaves running; the meters it started stop after every user + local port files=() + [ $# -eq 0 ] || for port in ${PORTS[@]+"${PORTS[@]}"}; do + printf '%s\n' "$@" >> "$(meter_users "$port")" 2>/dev/null || true; done + [ ${#METERS[@]} -gt 0 ] || return 0 + for port in "${OWNED[@]}"; do files+=("$(meter_users "$port")"); done + setsid nohup bash -c 'while :; do up= + for p in $(cat $1 2>/dev/null); do kill -0 "$p" 2>/dev/null && { up=1; break; }; done + [ -n "$up" ] || break; sleep 1; done + for f in $1; do : > "$f"; done 2>/dev/null # a launch listed after this look finds the meter gone and starts one + for m in $2; do kill -- "-$m" 2>/dev/null || kill "$m" 2>/dev/null || true; done' _ "${files[*]}" "${METERS[*]}" \ + > /dev/null 2>&1 < /dev/null & +} +meter_health() { # port: the /health of a meter still up three seconds on, this launch listed as one of its users first + curl -s -m 2 "http://127.0.0.1:$1/health" > /dev/null 2>&1 \ + && { echo $$ >> "$(meter_users "$1")" || true; } 2>/dev/null \ + && sleep 3 && curl -s -m 2 "http://127.0.0.1:$1/health" 2>/dev/null +} +trap 'stop_meters_after ${PIDS[@]+"${PIDS[@]}"}' EXIT # after the trainings started below, if any start_meter() { # port upstream ledger log [extra env...]: a meter already on the port must face the same upstream local port=$1 upstream=$2 ledger=$3 log=$4 health; shift 4 - if health=$(curl -s -m 2 "http://127.0.0.1:$port/health" 2>/dev/null); then + PORTS+=("$port") # started here or reused: this launch's processes use it + if health=$(meter_health $port); then case "$health" in *"\"upstream\": \"$upstream\""*) echo "wire meter already on port $port -> $upstream";; *) echo "REFUSING: port $port has a wire meter facing another or an unknown upstream ($health); choose another WM_PORT"; exit 1;; esac else - env "$@" PORT=$port LEDGER="$ledger" UPSTREAM="$upstream" \ + env "$@" PORT=$port LEDGER="$ledger" LEDGER_RESUME=1 UPSTREAM="$upstream" \ setsid nohup "$PY" "$METER" > "$log" 2>&1 < /dev/null & + meter_started $port $! echo "wire meter pid $! port $port -> $upstream"; sleep 2 fi } diff --git a/examples/clbench_poker/reharness/launch.sh b/examples/clbench_poker/reharness/launch.sh index 9c879ebf..d2e26eec 100755 --- a/examples/clbench_poker/reharness/launch.sh +++ b/examples/clbench_poker/reharness/launch.sh @@ -85,14 +85,45 @@ export MWM_CLBENCH=$CLBENCH MWM_METAWORLD=$METAWORLD [ -f "$TASKDIR/tasks.jsonl" ] || { echo "REFUSING: run $TASKDIR/prepare.py first"; exit 1; } +# The meters this launch starts stop within a second or two of the processes that use them ending (#82): its own and +# those of any launch that reuses one. Each launch on a port lists its processes in the port's users file, emptied +# when the meter stops; a launch reuses a running meter only if it is still up on a second look (one being stopped is +# gone by then). +METERS=() OWNED=() PORTS=() +meter_users() { echo "/tmp/hermit-wire-meter-$(id -u)-$1.users"; } # port: /tmp, which every launch on it shares +meter_started() { # port pid: a meter this launch started, this launch one of its users + { echo $$ >> "$(meter_users "$1")" || true; } 2>/dev/null + METERS+=("$2"); OWNED+=("$1") +} +stop_meters_after() { # pid...: what this launch leaves running; the meters it started stop after every user + local port files=() + [ $# -eq 0 ] || for port in ${PORTS[@]+"${PORTS[@]}"}; do + printf '%s\n' "$@" >> "$(meter_users "$port")" 2>/dev/null || true; done + [ ${#METERS[@]} -gt 0 ] || return 0 + for port in "${OWNED[@]}"; do files+=("$(meter_users "$port")"); done + setsid nohup bash -c 'while :; do up= + for p in $(cat $1 2>/dev/null); do kill -0 "$p" 2>/dev/null && { up=1; break; }; done + [ -n "$up" ] || break; sleep 1; done + for f in $1; do : > "$f"; done 2>/dev/null # a launch listed after this look finds the meter gone and starts one + for m in $2; do kill -- "-$m" 2>/dev/null || kill "$m" 2>/dev/null || true; done' _ "${files[*]}" "${METERS[*]}" \ + > /dev/null 2>&1 < /dev/null & +} +meter_health() { # port: the /health of a meter still up three seconds on, this launch listed as one of its users first + curl -s -m 2 "http://127.0.0.1:$1/health" > /dev/null 2>&1 \ + && { echo $$ >> "$(meter_users "$1")" || true; } 2>/dev/null \ + && sleep 3 && curl -s -m 2 "http://127.0.0.1:$1/health" 2>/dev/null +} +trap 'stop_meters_after ${PIDS[@]+"${PIDS[@]}"}' EXIT # after the trainings started below, if any start_meter() { # port upstream ledger log [extra env...]: a meter already on the port must face the same upstream local port=$1 upstream=$2 ledger=$3 log=$4 health; shift 4 - if health=$(curl -s -m 2 "http://127.0.0.1:$port/health" 2>/dev/null); then + PORTS+=("$port") # started here or reused: this launch's processes use it + if health=$(meter_health $port); then case "$health" in *"\"upstream\": \"$upstream\""*) echo "wire meter already on port $port -> $upstream";; *) echo "REFUSING: port $port has a wire meter facing another or an unknown upstream ($health); choose another WM_PORT"; exit 1;; esac else - env "$@" PORT=$port LEDGER="$ledger" UPSTREAM="$upstream" \ + env "$@" PORT=$port LEDGER="$ledger" LEDGER_RESUME=1 UPSTREAM="$upstream" \ setsid nohup "$PY" "$METER" > "$log" 2>&1 < /dev/null & + meter_started $port $! echo "wire meter pid $! port $port -> $upstream"; sleep 2 curl -s -m 2 "http://127.0.0.1:$port/health" > /dev/null \ || { echo "REFUSING: the wire meter on port $port did not come up (see $log)"; exit 1; } @@ -103,7 +134,7 @@ start_meter $WM_PORT "$HOST_ORIGIN" "$RUNS/$TAG/wire_ledger.json" "$RUNS/$TAG/wi start_meter $META_PORT "$EVOLVER_ORIGIN" "$RUNS/$TAG/meta_wire_ledger.json" "$RUNS/$TAG/meta_wire_meter.log" # The embedding relay signs the reader's keyless requests with the embedding backend's key (none for a keyless server); # a relay already on the port is refused unless its /health says it signs (`"signs": true`). -if [ -n "$EMBED_API_KEY_ENV" ] && health=$(curl -s -m 2 "http://127.0.0.1:$EMBED_PORT/health" 2>/dev/null); then +if [ -n "$EMBED_API_KEY_ENV" ] && health=$(meter_health $EMBED_PORT); then case "$health" in *'"signs": true'*) ;; *) echo "REFUSING: port $EMBED_PORT has a relay that does not sign requests (started without UPSTREAM_KEY_ENV); choose another WM_PORT"; exit 1;; esac fi start_meter $EMBED_PORT "$EMBED_ORIGIN" "$RUNS/$TAG/embed_wire_ledger.json" "$RUNS/$TAG/embed_wire_meter.log" \ diff --git a/examples/officeqa/harness2_stream/launch.sh b/examples/officeqa/harness2_stream/launch.sh index de90f070..51c9ecfe 100755 --- a/examples/officeqa/harness2_stream/launch.sh +++ b/examples/officeqa/harness2_stream/launch.sh @@ -98,12 +98,41 @@ print(((c.get('evolver') or {}).get('params') or {}).get('claude_bin') or '')" " fi "$PY" make_seed_codex.py "$PY" stream.py --seed "$SEED" --split $SPLITS --out "$STREAM" ${MAX_TASKS:+--max "$MAX_TASKS"} ${SKIP_TASKS:+--skip "$SKIP_TASKS"} +# The meters this launch starts stop within a second or two of the processes that use them ending (#82): its own and +# those of any launch that reuses one. Each launch on a port lists its processes in the port's users file, emptied +# when the meter stops; a launch reuses a running meter only if it is still up on a second look (one being stopped is +# gone by then). +METERS=() OWNED=() PORTS=() +meter_users() { echo "/tmp/hermit-wire-meter-$(id -u)-$1.users"; } # port: /tmp, which every launch on it shares +meter_started() { # port pid: a meter this launch started, this launch one of its users + { echo $$ >> "$(meter_users "$1")" || true; } 2>/dev/null + METERS+=("$2"); OWNED+=("$1") +} +stop_meters_after() { # pid...: what this launch leaves running; the meters it started stop after every user + local port files=() + [ $# -eq 0 ] || for port in ${PORTS[@]+"${PORTS[@]}"}; do + printf '%s\n' "$@" >> "$(meter_users "$port")" 2>/dev/null || true; done + [ ${#METERS[@]} -gt 0 ] || return 0 + for port in "${OWNED[@]}"; do files+=("$(meter_users "$port")"); done + setsid nohup bash -c 'while :; do up= + for p in $(cat $1 2>/dev/null); do kill -0 "$p" 2>/dev/null && { up=1; break; }; done + [ -n "$up" ] || break; sleep 1; done + for f in $1; do : > "$f"; done 2>/dev/null # a launch listed after this look finds the meter gone and starts one + for m in $2; do kill -- "-$m" 2>/dev/null || kill "$m" 2>/dev/null || true; done' _ "${files[*]}" "${METERS[*]}" \ + > /dev/null 2>&1 < /dev/null & +} +meter_health() { # port: the /health of a meter still up three seconds on, this launch listed as one of its users first + curl -s -m 2 "http://127.0.0.1:$1/health" > /dev/null 2>&1 \ + && { echo $$ >> "$(meter_users "$1")" || true; } 2>/dev/null \ + && sleep 3 && curl -s -m 2 "http://127.0.0.1:$1/health" 2>/dev/null +} if [ "$EDITORS" = claude ]; then # The meter restarts if it dies; each start keeps its own ledger (sum runs/$TAG.wire_ledger-*.json for the bill). - if [ "$METER" = yes ] && ! curl -s -m 2 "http://127.0.0.1:$WM_PORT/health" > /dev/null; then + if [ "$METER" = yes ] && ! meter_health $WM_PORT > /dev/null; then setsid nohup bash -c "while true; do PORT=$WM_PORT UPSTREAM=https://api.fireworks.ai \ LEDGER=$HERE/runs/$TAG.wire_ledger-\$(date -u +%Y%m%dT%H%M%SZ).json '$PY' -m hermit.runtime.wire_meter; \ sleep 5; done" >> "runs/$TAG.wire_meter.log" 2>&1 < /dev/null & + meter_started $WM_PORT $!; stop_meters_after $$ # the restart loop with its meter, after the run exec'd below sleep 3 fi # Claude Code updates itself: the run uses a copy of today's binary (or of CLAUDE_BIN, e.g. the stream it continues diff --git a/examples/officeqa/reharness/launch.sh b/examples/officeqa/reharness/launch.sh index c77792ad..f762c58b 100755 --- a/examples/officeqa/reharness/launch.sh +++ b/examples/officeqa/reharness/launch.sh @@ -75,8 +75,36 @@ if [ "${EMBED_KEY_REQUIRED:-1}" = 1 ] && [ -n "$KEY_ENV" ] && [ -z "${!KEY_ENV:- fi # The meter: one per port, started here if none answers; a meter facing another upstream is refused. mkdir -p runs +# The meters this launch starts stop within a second or two of the processes that use them ending (#82): its own and +# those of any launch that reuses one. Each launch on a port lists its processes in the port's users file, emptied +# when the meter stops; a launch reuses a running meter only if it is still up on a second look (one being stopped is +# gone by then). +METERS=() OWNED=() PORTS=() +meter_users() { echo "/tmp/hermit-wire-meter-$(id -u)-$1.users"; } # port: /tmp, which every launch on it shares +meter_started() { # port pid: a meter this launch started, this launch one of its users + { echo $$ >> "$(meter_users "$1")" || true; } 2>/dev/null + METERS+=("$2"); OWNED+=("$1") +} +stop_meters_after() { # pid...: what this launch leaves running; the meters it started stop after every user + local port files=() + [ $# -eq 0 ] || for port in ${PORTS[@]+"${PORTS[@]}"}; do + printf '%s\n' "$@" >> "$(meter_users "$port")" 2>/dev/null || true; done + [ ${#METERS[@]} -gt 0 ] || return 0 + for port in "${OWNED[@]}"; do files+=("$(meter_users "$port")"); done + setsid nohup bash -c 'while :; do up= + for p in $(cat $1 2>/dev/null); do kill -0 "$p" 2>/dev/null && { up=1; break; }; done + [ -n "$up" ] || break; sleep 1; done + for f in $1; do : > "$f"; done 2>/dev/null # a launch listed after this look finds the meter gone and starts one + for m in $2; do kill -- "-$m" 2>/dev/null || kill "$m" 2>/dev/null || true; done' _ "${files[*]}" "${METERS[*]}" \ + > /dev/null 2>&1 < /dev/null & +} +meter_health() { # port: the /health of a meter still up three seconds on, this launch listed as one of its users first + curl -s -m 2 "http://127.0.0.1:$1/health" > /dev/null 2>&1 \ + && { echo $$ >> "$(meter_users "$1")" || true; } 2>/dev/null \ + && sleep 3 && curl -s -m 2 "http://127.0.0.1:$1/health" 2>/dev/null +} LEDGER=$HERE/runs/wire_ledger-$WM_PORT.json -if health=$(curl -s -m 2 "http://127.0.0.1:$WM_PORT/health" 2>/dev/null); then +if health=$(meter_health $WM_PORT); then case "$health" in *"\"upstream\": \"$EMBED_ORIGIN\""*'"signs": true'*|*'"signs": true'*"\"upstream\": \"$EMBED_ORIGIN\""*) echo "wire meter already on port $WM_PORT -> $EMBED_ORIGIN (signs)";; @@ -86,9 +114,11 @@ if health=$(curl -s -m 2 "http://127.0.0.1:$WM_PORT/health" 2>/dev/null); then *) echo "REFUSING: port $WM_PORT has a wire meter facing another or an unknown upstream ($health); choose another WM_PORT"; exit 1;; esac else - env PORT=$WM_PORT UPSTREAM=$EMBED_ORIGIN ${KEY_ENV:+UPSTREAM_KEY_ENV=$KEY_ENV} LEDGER=$LEDGER \ + env PORT=$WM_PORT UPSTREAM=$EMBED_ORIGIN ${KEY_ENV:+UPSTREAM_KEY_ENV=$KEY_ENV} LEDGER=$LEDGER LEDGER_RESUME=1 \ setsid nohup "$PY" -m hermit.runtime.wire_meter >> "runs/wire_meter-$WM_PORT.log" 2>&1 < /dev/null & - echo "wire meter pid $! port $WM_PORT -> $EMBED_ORIGIN (ledger $LEDGER)"; sleep 2 + meter_started $WM_PORT $! + echo "wire meter pid $! port $WM_PORT -> $EMBED_ORIGIN (ledger $LEDGER)" + stop_meters_after $$; sleep 2 # $$: this launcher, then the hermit command it execs curl -s -m 2 "http://127.0.0.1:$WM_PORT/health" > /dev/null \ || { echo "REFUSING: the wire meter did not come up (runs/wire_meter-$WM_PORT.log)"; exit 1; } fi diff --git a/examples/servelearnbench/mwm/launch.sh b/examples/servelearnbench/mwm/launch.sh index 24ac59ba..5d48f497 100755 --- a/examples/servelearnbench/mwm/launch.sh +++ b/examples/servelearnbench/mwm/launch.sh @@ -81,14 +81,45 @@ HOST_KEY=""; [ -n "$HOST_API_KEY_ENV" ] && HOST_KEY=${!HOST_API_KEY_ENV:-} export SLB_MODEL=$HOST_MODEL_ID SLB_API_KEY=${HOST_KEY:-local} export SLB_REASONING_EFFORT=$EFFORT SLB_METHOD=$METHOD SLB_WORKDIR=$WD +# The meters this launch starts stop within a second or two of the processes that use them ending (#82): its own and +# those of any launch that reuses one. Each launch on a port lists its processes in the port's users file, emptied +# when the meter stops; a launch reuses a running meter only if it is still up on a second look (one being stopped is +# gone by then). +METERS=() OWNED=() PORTS=() +meter_users() { echo "/tmp/hermit-wire-meter-$(id -u)-$1.users"; } # port: /tmp, which every launch on it shares +meter_started() { # port pid: a meter this launch started, this launch one of its users + { echo $$ >> "$(meter_users "$1")" || true; } 2>/dev/null + METERS+=("$2"); OWNED+=("$1") +} +stop_meters_after() { # pid...: what this launch leaves running; the meters it started stop after every user + local port files=() + [ $# -eq 0 ] || for port in ${PORTS[@]+"${PORTS[@]}"}; do + printf '%s\n' "$@" >> "$(meter_users "$port")" 2>/dev/null || true; done + [ ${#METERS[@]} -gt 0 ] || return 0 + for port in "${OWNED[@]}"; do files+=("$(meter_users "$port")"); done + setsid nohup bash -c 'while :; do up= + for p in $(cat $1 2>/dev/null); do kill -0 "$p" 2>/dev/null && { up=1; break; }; done + [ -n "$up" ] || break; sleep 1; done + for f in $1; do : > "$f"; done 2>/dev/null # a launch listed after this look finds the meter gone and starts one + for m in $2; do kill -- "-$m" 2>/dev/null || kill "$m" 2>/dev/null || true; done' _ "${files[*]}" "${METERS[*]}" \ + > /dev/null 2>&1 < /dev/null & +} +meter_health() { # port: the /health of a meter still up three seconds on, this launch listed as one of its users first + curl -s -m 2 "http://127.0.0.1:$1/health" > /dev/null 2>&1 \ + && { echo $$ >> "$(meter_users "$1")" || true; } 2>/dev/null \ + && sleep 3 && curl -s -m 2 "http://127.0.0.1:$1/health" 2>/dev/null +} +trap 'stop_meters_after ${TRAIN_PID:-} ${TEST_PID:-}' EXIT # train mode: after the training and the tests after it start_meter() { # port upstream ledger log [extra env...]: a meter already on the port must face the same upstream local port=$1 upstream=$2 ledger=$3 log=$4 health; shift 4 - if health=$(curl -s -m 2 "http://127.0.0.1:$port/health" 2>/dev/null); then + PORTS+=("$port") # started here or reused: this launch's processes use it + if health=$(meter_health $port); then case "$health" in *"\"upstream\": \"$upstream\""*) echo "wire meter already on port $port -> $upstream";; *) echo "REFUSING: port $port has a wire meter facing another or an unknown upstream ($health); choose another WM_PORT"; exit 1;; esac else - env "$@" PORT=$port LEDGER="$ledger" UPSTREAM="$upstream" \ + env "$@" PORT=$port LEDGER="$ledger" LEDGER_RESUME=1 UPSTREAM="$upstream" \ setsid nohup "$PY" "$METER" > "$log" 2>&1 < /dev/null & + meter_started $port $! echo "wire meter pid $! port $port -> $upstream"; sleep 2 fi } diff --git a/hermit/runtime/wire_meter.py b/hermit/runtime/wire_meter.py index 64860ac6..007c1d5b 100644 --- a/hermit/runtime/wire_meter.py +++ b/hermit/runtime/wire_meter.py @@ -36,6 +36,9 @@ the relay adds a metering hop, not an egress path. - the ledger is persisted to LEDGER as JSON every FLUSH_S seconds and on SIGTERM; GET /ledger serves it live, GET /health the error counters. + LEDGER_RESUME=1: a meter started on a LEDGER an earlier one left carries + its totals on (the example launchers: they stop their meters when their + runs end, and the next run on the port starts another on the same file). Dialects: OpenAI chat completions (prompt_tokens includes cached) and Anthropic messages (input_tokens excludes cache components — added back), @@ -43,7 +46,8 @@ Env: PORT (default 3129), BIND (default 127.0.0.1; 0.0.0.0 for a sidecar that other containers call), UPSTREAM (default https://api.fireworks.ai), -PROXY (e.g. http://mw-egress-meta:3128), LEDGER (json path), FLUSH_S. +PROXY (e.g. http://mw-egress-meta:3128), LEDGER (json path), LEDGER_RESUME, +FLUSH_S. UPSTREAM_KEY_ENV names the variable holding the upstream's key: a request that arrives without an Authorization header gets `Bearer ` (2026-10-25). @@ -84,6 +88,7 @@ UPSTREAM = os.environ.get("UPSTREAM", "https://api.fireworks.ai") PROXY = os.environ.get("PROXY", "") LEDGER_PATH = os.environ.get("LEDGER", "") +LEDGER_RESUME = os.environ.get("LEDGER_RESUME", "") not in ("", "0") FLUSH_S = float(os.environ.get("FLUSH_S", "15")) REWRITE_DEV = os.environ.get("REWRITE_DEVELOPER_ROLE", "") not in ("", "0") # PRIME_EFFORT (e.g. "high"): on the `prime` bucket, replace Prime's own @@ -408,6 +413,22 @@ def _flush(path: str = LEDGER_PATH): pass +def _resume(path: str = LEDGER_PATH): + """The totals an earlier meter left in LEDGER, added to this one's.""" + try: + with open(path) as f: + old = json.load(f).get("ledger") or {} + with _lock: + for bucket, totals in old.items(): + b = _ledger.setdefault( + bucket, {"input": 0, "cacheRead": 0, "output": 0, + "calls": 0}) + for k in b: + b[k] += int(totals.get(k, 0)) + except (OSError, ValueError, AttributeError, TypeError): + pass + + class Handler(BaseHTTPRequestHandler): protocol_version = "HTTP/1.1" @@ -652,6 +673,8 @@ def main(): sys.exit(f"wire meter: a relay that signs requests (UPSTREAM_KEY_ENV) listens on the loopback only; refusing " f"BIND={BIND}") srv = Server((BIND, PORT), Handler) + if LEDGER_PATH and LEDGER_RESUME: + _resume() def _bye(sig, frame): _flush() diff --git a/tests/core/test_backends.py b/tests/core/test_backends.py index 6876ae2d..ba5cae70 100644 --- a/tests/core/test_backends.py +++ b/tests/core/test_backends.py @@ -3,9 +3,15 @@ dialect to a fake server.""" from http.server import BaseHTTPRequestHandler, HTTPServer import json +import os from pathlib import Path +import signal +import socket +import subprocess import sys import threading +import time +import urllib.request import pytest import yaml @@ -403,6 +409,44 @@ def test_cli_dispatch_and_meter_buckets(capsys): assert split_bucket("/health") == ("unattributed", "/health") +def test_a_meter_carries_on_the_ledger_an_earlier_one_left(tmp_path): + """A launcher stops its meter when its run ends and the next run on the port starts another on the same ledger + (#82): with LEDGER_RESUME=1 (the launchers set it) the new meter keeps the earlier totals instead of writing an + empty ledger over them; without it a meter starts empty, as lean_swarm's rerun of an attempt expects.""" + ledger = tmp_path / "wire_ledger.json" + earlier = {"run_o0": {"input": 1000, "cacheRead": 200, "output": 50, "calls": 3}} + ledger.write_text(json.dumps({"ledger": earlier, "health": {}, "ts": 1})) + with socket.socket() as probe: + probe.bind(("127.0.0.1", 0)) + port = probe.getsockname()[1] + env = {**os.environ, "PYTHONPATH": str(Path(__file__).resolve().parents[2]), "UPSTREAM": "http://127.0.0.1:9", + "PORT": str(port), "LEDGER": str(ledger), "FLUSH_S": "3600"} + env.pop("UPSTREAM_KEY_ENV", None) + + def meter(**extra) -> dict: + """The ledger a meter serves once up, and the one it writes when stopped as a launcher stops it (SIGTERM).""" + proc = subprocess.Popen([sys.executable, "-m", "hermit.runtime.wire_meter"], env={**env, **extra}, + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + try: + for _ in range(50): + try: + with urllib.request.urlopen(f"http://127.0.0.1:{port}/ledger", timeout=2) as response: + live = json.loads(response.read())["ledger"] + break + except OSError: + time.sleep(0.2) + else: + raise AssertionError("the meter did not come up") + finally: + proc.send_signal(signal.SIGTERM) + proc.wait(timeout=10) + assert json.loads(ledger.read_text())["ledger"] == live + return live + + assert meter(LEDGER_RESUME="1") == earlier + assert meter() == {} + + def test_a_task_agent_on_a_cli_base_is_bound_and_checked_like_the_cli(): """A task's own agent on agent.base: codex with a model reference is bound as Codex's model (not a dictionary for -m), and the run's preflight checks that binding for the Responses API."""