From 8489508cf3158d0310c0dff125c9a74b6a90969e Mon Sep 17 00:00:00 2001 From: SmartBea_Linlin_r <80329948+GaoRuilin1@users.noreply.github.com> Date: Sat, 7 Nov 2026 22:36:30 -0500 Subject: [PATCH 1/3] Stop the wire meters a launcher starts once its runs end (#82) Each launcher records the meters it starts and, through a detached waiter, stops them once its trainings (ServeLearnBench: the tests that follow) have ended; the harness2 restart loop is stopped with its meter as one process group. A meter found already running is left to the launch that started it. --- CHANGELOG.md | 4 ++++ examples/clbench_database/mwm/launch.sh | 11 +++++++++++ examples/clbench_database/prompt_only/launch.sh | 11 +++++++++++ examples/clbench_database/reharness/launch.sh | 11 +++++++++++ examples/clbench_poker/mwm/launch.sh | 11 +++++++++++ examples/clbench_poker/prompt_only/launch.sh | 11 +++++++++++ examples/clbench_poker/reharness/launch.sh | 11 +++++++++++ examples/officeqa/harness2_stream/launch.sh | 10 ++++++++++ examples/officeqa/reharness/launch.sh | 10 ++++++++++ examples/servelearnbench/mwm/launch.sh | 11 +++++++++++ 10 files changed, 101 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 021cc8c6..4e962f70 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -72,6 +72,10 @@ Changes are grouped by software version. Entries under Unreleased describe work - The AHE Adapter Evolver took a `change_manifest.json` an earlier step left for the current step's: a manifest naming the next iteration was refused at its own step, then archived as the next step's when that step's session wrote none. The file (or a directory a session made in its place) is now removed before each Evolve session. +- Nine example launchers left their wire meters running after the run (#82), so a later run on the same port + reused one and wrote its records into the earlier run's ledger. The meters a launch starts now stop once its + trainings (and, for ServeLearnBench, the tests after them) have ended, the harness2 restart loop included; a meter + found already running is left to the launch that started it. ## 0.2.0 - 2026-11-04 diff --git a/examples/clbench_database/mwm/launch.sh b/examples/clbench_database/mwm/launch.sh index 7741cfac..150b95d9 100755 --- a/examples/clbench_database/mwm/launch.sh +++ b/examples/clbench_database/mwm/launch.sh @@ -78,6 +78,16 @@ fi command -v codex >/dev/null || { echo "REFUSING: codex CLI not on PATH"; exit 1; } [ -f "$TASKDIR/tasks.jsonl" ] || { echo "REFUSING: run $TASKDIR/prepare.py first"; exit 1; } +# The meters this launch starts stop once the processes it names have ended (#82); a meter found already running is +# left to the launch that started it. +METERS=() +stop_meters_after() { # pid... + [ ${#METERS[@]} -gt 0 ] || return 0 + setsid nohup bash -c 'for p in $1; do while kill -0 "$p" 2>/dev/null; do sleep 30; done; done + for m in $2; do kill -- "-$m" 2>/dev/null || kill "$m" 2>/dev/null || true; done' _ "$*" "${METERS[*]}" \ + > /dev/null 2>&1 < /dev/null & +} +trap 'stop_meters_after ${PIDS[@]+"${PIDS[@]}"}' EXIT # after the trainings started below, if any start_meter() { # port upstream ledger log [extra env...]: a meter already on the port must face the same upstream local port=$1 upstream=$2 ledger=$3 log=$4 health; shift 4 if health=$(curl -s -m 2 "http://127.0.0.1:$port/health" 2>/dev/null); then @@ -86,6 +96,7 @@ start_meter() { # port upstream ledger log [extra env...]: a meter already on t else env "$@" PORT=$port LEDGER="$ledger" UPSTREAM="$upstream" \ setsid nohup "$PY" "$METER" > "$log" 2>&1 < /dev/null & + METERS+=("$!") echo "wire meter pid $! port $port -> $upstream"; sleep 2 fi } diff --git a/examples/clbench_database/prompt_only/launch.sh b/examples/clbench_database/prompt_only/launch.sh index cc2316b1..ba54de25 100755 --- a/examples/clbench_database/prompt_only/launch.sh +++ b/examples/clbench_database/prompt_only/launch.sh @@ -82,6 +82,16 @@ fi command -v codex >/dev/null || { echo "REFUSING: codex CLI not on PATH"; exit 1; } [ -f "$TASKDIR/tasks.jsonl" ] || { echo "REFUSING: run $TASKDIR/prepare.py first"; exit 1; } +# The meters this launch starts stop once the processes it names have ended (#82); a meter found already running is +# left to the launch that started it. +METERS=() +stop_meters_after() { # pid... + [ ${#METERS[@]} -gt 0 ] || return 0 + setsid nohup bash -c 'for p in $1; do while kill -0 "$p" 2>/dev/null; do sleep 30; done; done + for m in $2; do kill -- "-$m" 2>/dev/null || kill "$m" 2>/dev/null || true; done' _ "$*" "${METERS[*]}" \ + > /dev/null 2>&1 < /dev/null & +} +trap 'stop_meters_after ${PIDS[@]+"${PIDS[@]}"}' EXIT # after the trainings started below, if any start_meter() { # port upstream ledger log [extra env...]: a meter already on the port must face the same upstream local port=$1 upstream=$2 ledger=$3 log=$4 health; shift 4 if health=$(curl -s -m 2 "http://127.0.0.1:$port/health" 2>/dev/null); then @@ -90,6 +100,7 @@ start_meter() { # port upstream ledger log [extra env...]: a meter already on t else env "$@" PORT=$port LEDGER="$ledger" UPSTREAM="$upstream" \ setsid nohup "$PY" "$METER" > "$log" 2>&1 < /dev/null & + METERS+=("$!") echo "wire meter pid $! port $port -> $upstream"; sleep 2 fi } diff --git a/examples/clbench_database/reharness/launch.sh b/examples/clbench_database/reharness/launch.sh index 8fb32079..c0d6dbf6 100755 --- a/examples/clbench_database/reharness/launch.sh +++ b/examples/clbench_database/reharness/launch.sh @@ -90,6 +90,16 @@ fi [ -f "$TASKDIR/tasks.jsonl" ] || { echo "REFUSING: run $TASKDIR/prepare.py first"; exit 1; } +# The meters this launch starts stop once the processes it names have ended (#82); a meter found already running is +# left to the launch that started it. +METERS=() +stop_meters_after() { # pid... + [ ${#METERS[@]} -gt 0 ] || return 0 + setsid nohup bash -c 'for p in $1; do while kill -0 "$p" 2>/dev/null; do sleep 30; done; done + for m in $2; do kill -- "-$m" 2>/dev/null || kill "$m" 2>/dev/null || true; done' _ "$*" "${METERS[*]}" \ + > /dev/null 2>&1 < /dev/null & +} +trap 'stop_meters_after ${PIDS[@]+"${PIDS[@]}"}' EXIT # after the trainings started below, if any start_meter() { # port upstream ledger log [extra env...]: a meter already on the port must face the same upstream local port=$1 upstream=$2 ledger=$3 log=$4 health; shift 4 if health=$(curl -s -m 2 "http://127.0.0.1:$port/health" 2>/dev/null); then @@ -98,6 +108,7 @@ start_meter() { # port upstream ledger log [extra env...]: a meter already on t else env "$@" PORT=$port LEDGER="$ledger" UPSTREAM="$upstream" \ setsid nohup "$PY" "$METER" > "$log" 2>&1 < /dev/null & + METERS+=("$!") echo "wire meter pid $! port $port -> $upstream"; sleep 2 curl -s -m 2 "http://127.0.0.1:$port/health" > /dev/null \ || { echo "REFUSING: the wire meter on port $port did not come up (see $log)"; exit 1; } diff --git a/examples/clbench_poker/mwm/launch.sh b/examples/clbench_poker/mwm/launch.sh index 7be6f2df..bab79338 100755 --- a/examples/clbench_poker/mwm/launch.sh +++ b/examples/clbench_poker/mwm/launch.sh @@ -73,6 +73,16 @@ export MWM_CLBENCH=$CLBENCH MWM_METAWORLD=$METAWORLD command -v codex >/dev/null || { echo "REFUSING: codex CLI not on PATH"; exit 1; } [ -f "$TASKDIR/tasks.jsonl" ] || { echo "REFUSING: run $TASKDIR/prepare.py first"; exit 1; } +# The meters this launch starts stop once the processes it names have ended (#82); a meter found already running is +# left to the launch that started it. +METERS=() +stop_meters_after() { # pid... + [ ${#METERS[@]} -gt 0 ] || return 0 + setsid nohup bash -c 'for p in $1; do while kill -0 "$p" 2>/dev/null; do sleep 30; done; done + for m in $2; do kill -- "-$m" 2>/dev/null || kill "$m" 2>/dev/null || true; done' _ "$*" "${METERS[*]}" \ + > /dev/null 2>&1 < /dev/null & +} +trap 'stop_meters_after ${PIDS[@]+"${PIDS[@]}"}' EXIT # after the trainings started below, if any start_meter() { # port upstream ledger log [extra env...]: a meter already on the port must face the same upstream local port=$1 upstream=$2 ledger=$3 log=$4 health; shift 4 if health=$(curl -s -m 2 "http://127.0.0.1:$port/health" 2>/dev/null); then @@ -81,6 +91,7 @@ start_meter() { # port upstream ledger log [extra env...]: a meter already on t else env "$@" PORT=$port LEDGER="$ledger" UPSTREAM="$upstream" \ setsid nohup "$PY" "$METER" > "$log" 2>&1 < /dev/null & + METERS+=("$!") echo "wire meter pid $! port $port -> $upstream"; sleep 2 fi } diff --git a/examples/clbench_poker/prompt_only/launch.sh b/examples/clbench_poker/prompt_only/launch.sh index 551de82b..4446b701 100755 --- a/examples/clbench_poker/prompt_only/launch.sh +++ b/examples/clbench_poker/prompt_only/launch.sh @@ -77,6 +77,16 @@ export MWM_CLBENCH=$CLBENCH MWM_METAWORLD=$METAWORLD command -v codex >/dev/null || { echo "REFUSING: codex CLI not on PATH"; exit 1; } [ -f "$TASKDIR/tasks.jsonl" ] || { echo "REFUSING: run $TASKDIR/prepare.py first"; exit 1; } +# The meters this launch starts stop once the processes it names have ended (#82); a meter found already running is +# left to the launch that started it. +METERS=() +stop_meters_after() { # pid... + [ ${#METERS[@]} -gt 0 ] || return 0 + setsid nohup bash -c 'for p in $1; do while kill -0 "$p" 2>/dev/null; do sleep 30; done; done + for m in $2; do kill -- "-$m" 2>/dev/null || kill "$m" 2>/dev/null || true; done' _ "$*" "${METERS[*]}" \ + > /dev/null 2>&1 < /dev/null & +} +trap 'stop_meters_after ${PIDS[@]+"${PIDS[@]}"}' EXIT # after the trainings started below, if any start_meter() { # port upstream ledger log [extra env...]: a meter already on the port must face the same upstream local port=$1 upstream=$2 ledger=$3 log=$4 health; shift 4 if health=$(curl -s -m 2 "http://127.0.0.1:$port/health" 2>/dev/null); then @@ -85,6 +95,7 @@ start_meter() { # port upstream ledger log [extra env...]: a meter already on t else env "$@" PORT=$port LEDGER="$ledger" UPSTREAM="$upstream" \ setsid nohup "$PY" "$METER" > "$log" 2>&1 < /dev/null & + METERS+=("$!") echo "wire meter pid $! port $port -> $upstream"; sleep 2 fi } diff --git a/examples/clbench_poker/reharness/launch.sh b/examples/clbench_poker/reharness/launch.sh index 9c879ebf..f34b461a 100755 --- a/examples/clbench_poker/reharness/launch.sh +++ b/examples/clbench_poker/reharness/launch.sh @@ -85,6 +85,16 @@ export MWM_CLBENCH=$CLBENCH MWM_METAWORLD=$METAWORLD [ -f "$TASKDIR/tasks.jsonl" ] || { echo "REFUSING: run $TASKDIR/prepare.py first"; exit 1; } +# The meters this launch starts stop once the processes it names have ended (#82); a meter found already running is +# left to the launch that started it. +METERS=() +stop_meters_after() { # pid... + [ ${#METERS[@]} -gt 0 ] || return 0 + setsid nohup bash -c 'for p in $1; do while kill -0 "$p" 2>/dev/null; do sleep 30; done; done + for m in $2; do kill -- "-$m" 2>/dev/null || kill "$m" 2>/dev/null || true; done' _ "$*" "${METERS[*]}" \ + > /dev/null 2>&1 < /dev/null & +} +trap 'stop_meters_after ${PIDS[@]+"${PIDS[@]}"}' EXIT # after the trainings started below, if any start_meter() { # port upstream ledger log [extra env...]: a meter already on the port must face the same upstream local port=$1 upstream=$2 ledger=$3 log=$4 health; shift 4 if health=$(curl -s -m 2 "http://127.0.0.1:$port/health" 2>/dev/null); then @@ -93,6 +103,7 @@ start_meter() { # port upstream ledger log [extra env...]: a meter already on t else env "$@" PORT=$port LEDGER="$ledger" UPSTREAM="$upstream" \ setsid nohup "$PY" "$METER" > "$log" 2>&1 < /dev/null & + METERS+=("$!") echo "wire meter pid $! port $port -> $upstream"; sleep 2 curl -s -m 2 "http://127.0.0.1:$port/health" > /dev/null \ || { echo "REFUSING: the wire meter on port $port did not come up (see $log)"; exit 1; } diff --git a/examples/officeqa/harness2_stream/launch.sh b/examples/officeqa/harness2_stream/launch.sh index e226a5c8..aa16a53e 100755 --- a/examples/officeqa/harness2_stream/launch.sh +++ b/examples/officeqa/harness2_stream/launch.sh @@ -65,12 +65,22 @@ fi mkdir -p runs "$PY" make_seed_codex.py "$PY" stream.py --seed "$SEED" --split $SPLITS --out "$STREAM" ${MAX_TASKS:+--max "$MAX_TASKS"} ${SKIP_TASKS:+--skip "$SKIP_TASKS"} +# The meters this launch starts stop once the processes it names have ended (#82); a meter found already running is +# left to the launch that started it. +METERS=() +stop_meters_after() { # pid... + [ ${#METERS[@]} -gt 0 ] || return 0 + setsid nohup bash -c 'for p in $1; do while kill -0 "$p" 2>/dev/null; do sleep 30; done; done + for m in $2; do kill -- "-$m" 2>/dev/null || kill "$m" 2>/dev/null || true; done' _ "$*" "${METERS[*]}" \ + > /dev/null 2>&1 < /dev/null & +} if [ "$EDITORS" = claude ]; then # The meter restarts if it dies; each start keeps its own ledger (sum runs/$TAG.wire_ledger-*.json for the bill). if ! curl -s -m 2 "http://127.0.0.1:$WM_PORT/health" > /dev/null; then setsid nohup bash -c "while true; do PORT=$WM_PORT UPSTREAM=https://api.fireworks.ai \ LEDGER=$HERE/runs/$TAG.wire_ledger-\$(date -u +%Y%m%dT%H%M%SZ).json '$PY' -m hermit.runtime.wire_meter; \ sleep 5; done" >> "runs/$TAG.wire_meter.log" 2>&1 < /dev/null & + METERS+=("$!"); stop_meters_after $$ # the restart loop with its meter, when the run exec'd below ends sleep 3 fi # Claude Code updates itself: the run uses a copy of today's binary (or of CLAUDE_BIN, e.g. the stream it continues diff --git a/examples/officeqa/reharness/launch.sh b/examples/officeqa/reharness/launch.sh index c77792ad..73798b87 100755 --- a/examples/officeqa/reharness/launch.sh +++ b/examples/officeqa/reharness/launch.sh @@ -75,6 +75,15 @@ if [ "${EMBED_KEY_REQUIRED:-1}" = 1 ] && [ -n "$KEY_ENV" ] && [ -z "${!KEY_ENV:- fi # The meter: one per port, started here if none answers; a meter facing another upstream is refused. mkdir -p runs +# The meters this launch starts stop once the processes it names have ended (#82); a meter found already running is +# left to the launch that started it. +METERS=() +stop_meters_after() { # pid... + [ ${#METERS[@]} -gt 0 ] || return 0 + setsid nohup bash -c 'for p in $1; do while kill -0 "$p" 2>/dev/null; do sleep 30; done; done + for m in $2; do kill -- "-$m" 2>/dev/null || kill "$m" 2>/dev/null || true; done' _ "$*" "${METERS[*]}" \ + > /dev/null 2>&1 < /dev/null & +} LEDGER=$HERE/runs/wire_ledger-$WM_PORT.json if health=$(curl -s -m 2 "http://127.0.0.1:$WM_PORT/health" 2>/dev/null); then case "$health" in @@ -88,6 +97,7 @@ if health=$(curl -s -m 2 "http://127.0.0.1:$WM_PORT/health" 2>/dev/null); then else env PORT=$WM_PORT UPSTREAM=$EMBED_ORIGIN ${KEY_ENV:+UPSTREAM_KEY_ENV=$KEY_ENV} LEDGER=$LEDGER \ setsid nohup "$PY" -m hermit.runtime.wire_meter >> "runs/wire_meter-$WM_PORT.log" 2>&1 < /dev/null & + METERS+=("$!"); stop_meters_after $$ # $$: this launcher, then the hermit command it execs echo "wire meter pid $! port $WM_PORT -> $EMBED_ORIGIN (ledger $LEDGER)"; sleep 2 curl -s -m 2 "http://127.0.0.1:$WM_PORT/health" > /dev/null \ || { echo "REFUSING: the wire meter did not come up (runs/wire_meter-$WM_PORT.log)"; exit 1; } diff --git a/examples/servelearnbench/mwm/launch.sh b/examples/servelearnbench/mwm/launch.sh index 24ac59ba..28bd9dff 100755 --- a/examples/servelearnbench/mwm/launch.sh +++ b/examples/servelearnbench/mwm/launch.sh @@ -81,6 +81,16 @@ HOST_KEY=""; [ -n "$HOST_API_KEY_ENV" ] && HOST_KEY=${!HOST_API_KEY_ENV:-} export SLB_MODEL=$HOST_MODEL_ID SLB_API_KEY=${HOST_KEY:-local} export SLB_REASONING_EFFORT=$EFFORT SLB_METHOD=$METHOD SLB_WORKDIR=$WD +# The meters this launch starts stop once the processes it names have ended (#82); a meter found already running is +# left to the launch that started it. +METERS=() +stop_meters_after() { # pid... + [ ${#METERS[@]} -gt 0 ] || return 0 + setsid nohup bash -c 'for p in $1; do while kill -0 "$p" 2>/dev/null; do sleep 30; done; done + for m in $2; do kill -- "-$m" 2>/dev/null || kill "$m" 2>/dev/null || true; done' _ "$*" "${METERS[*]}" \ + > /dev/null 2>&1 < /dev/null & +} +trap 'stop_meters_after ${TEST_PID:-}' EXIT # train mode: after the tests that follow the training start_meter() { # port upstream ledger log [extra env...]: a meter already on the port must face the same upstream local port=$1 upstream=$2 ledger=$3 log=$4 health; shift 4 if health=$(curl -s -m 2 "http://127.0.0.1:$port/health" 2>/dev/null); then @@ -89,6 +99,7 @@ start_meter() { # port upstream ledger log [extra env...]: a meter already on t else env "$@" PORT=$port LEDGER="$ledger" UPSTREAM="$upstream" \ setsid nohup "$PY" "$METER" > "$log" 2>&1 < /dev/null & + METERS+=("$!") echo "wire meter pid $! port $port -> $upstream"; sleep 2 fi } From 6de34f0010fc41225d50fc311ae2f195baf4d89e Mon Sep 17 00:00:00 2001 From: SmartBea_Linlin_r <80329948+GaoRuilin1@users.noreply.github.com> Date: Sat, 7 Nov 2026 22:43:11 -0500 Subject: [PATCH 2/3] Wire meters: stop within a second; reuse only a meter still up on a second look A launch started right after another could reuse that launch's meter while its waiter was about to stop it. ServeLearnBench's waiter also waits for the training itself. --- CHANGELOG.md | 7 ++++--- examples/clbench_database/mwm/launch.sh | 12 ++++++++---- examples/clbench_database/prompt_only/launch.sh | 12 ++++++++---- examples/clbench_database/reharness/launch.sh | 14 +++++++++----- examples/clbench_poker/mwm/launch.sh | 12 ++++++++---- examples/clbench_poker/prompt_only/launch.sh | 12 ++++++++---- examples/clbench_poker/reharness/launch.sh | 14 +++++++++----- examples/officeqa/harness2_stream/launch.sh | 12 ++++++++---- examples/officeqa/reharness/launch.sh | 12 ++++++++---- examples/servelearnbench/mwm/launch.sh | 14 +++++++++----- 10 files changed, 79 insertions(+), 42 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4e962f70..70c51141 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -73,9 +73,10 @@ Changes are grouped by software version. Entries under Unreleased describe work naming the next iteration was refused at its own step, then archived as the next step's when that step's session wrote none. The file (or a directory a session made in its place) is now removed before each Evolve session. - Nine example launchers left their wire meters running after the run (#82), so a later run on the same port - reused one and wrote its records into the earlier run's ledger. The meters a launch starts now stop once its - trainings (and, for ServeLearnBench, the tests after them) have ended, the harness2 restart loop included; a meter - found already running is left to the launch that started it. + reused one and wrote its records into the earlier run's ledger. The meters a launch starts now stop within a second + or two of its trainings (and, for ServeLearnBench, the tests after them) ending, the harness2 restart loop + included. A launch reuses a running meter only if it is still up three seconds on, so it never takes one that is + being stopped; a meter shared by two runs at once still stops when the run that started it ends. ## 0.2.0 - 2026-11-04 diff --git a/examples/clbench_database/mwm/launch.sh b/examples/clbench_database/mwm/launch.sh index 150b95d9..8a7f83e4 100755 --- a/examples/clbench_database/mwm/launch.sh +++ b/examples/clbench_database/mwm/launch.sh @@ -78,19 +78,23 @@ fi command -v codex >/dev/null || { echo "REFUSING: codex CLI not on PATH"; exit 1; } [ -f "$TASKDIR/tasks.jsonl" ] || { echo "REFUSING: run $TASKDIR/prepare.py first"; exit 1; } -# The meters this launch starts stop once the processes it names have ended (#82); a meter found already running is -# left to the launch that started it. +# The meters this launch starts stop within a second or two of the processes it names ending (#82). A meter found +# running belongs to the launch that started it: a launch reuses one only if it is still up on a second look. METERS=() stop_meters_after() { # pid... [ ${#METERS[@]} -gt 0 ] || return 0 - setsid nohup bash -c 'for p in $1; do while kill -0 "$p" 2>/dev/null; do sleep 30; done; done + setsid nohup bash -c 'for p in $1; do while kill -0 "$p" 2>/dev/null; do sleep 1; done; done for m in $2; do kill -- "-$m" 2>/dev/null || kill "$m" 2>/dev/null || true; done' _ "$*" "${METERS[*]}" \ > /dev/null 2>&1 < /dev/null & } +meter_health() { # port: the /health of a meter still up three seconds on (one being stopped is gone by then) + curl -s -m 2 "http://127.0.0.1:$1/health" > /dev/null 2>&1 && sleep 3 \ + && curl -s -m 2 "http://127.0.0.1:$1/health" 2>/dev/null +} trap 'stop_meters_after ${PIDS[@]+"${PIDS[@]}"}' EXIT # after the trainings started below, if any start_meter() { # port upstream ledger log [extra env...]: a meter already on the port must face the same upstream local port=$1 upstream=$2 ledger=$3 log=$4 health; shift 4 - if health=$(curl -s -m 2 "http://127.0.0.1:$port/health" 2>/dev/null); then + if health=$(meter_health $port); then case "$health" in *"\"upstream\": \"$upstream\""*) echo "wire meter already on port $port -> $upstream";; *) echo "REFUSING: port $port has a wire meter facing another or an unknown upstream ($health); choose another WM_PORT"; exit 1;; esac else diff --git a/examples/clbench_database/prompt_only/launch.sh b/examples/clbench_database/prompt_only/launch.sh index ba54de25..2f186b1f 100755 --- a/examples/clbench_database/prompt_only/launch.sh +++ b/examples/clbench_database/prompt_only/launch.sh @@ -82,19 +82,23 @@ fi command -v codex >/dev/null || { echo "REFUSING: codex CLI not on PATH"; exit 1; } [ -f "$TASKDIR/tasks.jsonl" ] || { echo "REFUSING: run $TASKDIR/prepare.py first"; exit 1; } -# The meters this launch starts stop once the processes it names have ended (#82); a meter found already running is -# left to the launch that started it. +# The meters this launch starts stop within a second or two of the processes it names ending (#82). A meter found +# running belongs to the launch that started it: a launch reuses one only if it is still up on a second look. METERS=() stop_meters_after() { # pid... [ ${#METERS[@]} -gt 0 ] || return 0 - setsid nohup bash -c 'for p in $1; do while kill -0 "$p" 2>/dev/null; do sleep 30; done; done + setsid nohup bash -c 'for p in $1; do while kill -0 "$p" 2>/dev/null; do sleep 1; done; done for m in $2; do kill -- "-$m" 2>/dev/null || kill "$m" 2>/dev/null || true; done' _ "$*" "${METERS[*]}" \ > /dev/null 2>&1 < /dev/null & } +meter_health() { # port: the /health of a meter still up three seconds on (one being stopped is gone by then) + curl -s -m 2 "http://127.0.0.1:$1/health" > /dev/null 2>&1 && sleep 3 \ + && curl -s -m 2 "http://127.0.0.1:$1/health" 2>/dev/null +} trap 'stop_meters_after ${PIDS[@]+"${PIDS[@]}"}' EXIT # after the trainings started below, if any start_meter() { # port upstream ledger log [extra env...]: a meter already on the port must face the same upstream local port=$1 upstream=$2 ledger=$3 log=$4 health; shift 4 - if health=$(curl -s -m 2 "http://127.0.0.1:$port/health" 2>/dev/null); then + if health=$(meter_health $port); then case "$health" in *"\"upstream\": \"$upstream\""*) echo "wire meter already on port $port -> $upstream";; *) echo "REFUSING: port $port has a wire meter facing another or an unknown upstream ($health); choose another WM_PORT"; exit 1;; esac else diff --git a/examples/clbench_database/reharness/launch.sh b/examples/clbench_database/reharness/launch.sh index c0d6dbf6..49550ab4 100755 --- a/examples/clbench_database/reharness/launch.sh +++ b/examples/clbench_database/reharness/launch.sh @@ -90,19 +90,23 @@ fi [ -f "$TASKDIR/tasks.jsonl" ] || { echo "REFUSING: run $TASKDIR/prepare.py first"; exit 1; } -# The meters this launch starts stop once the processes it names have ended (#82); a meter found already running is -# left to the launch that started it. +# The meters this launch starts stop within a second or two of the processes it names ending (#82). A meter found +# running belongs to the launch that started it: a launch reuses one only if it is still up on a second look. METERS=() stop_meters_after() { # pid... [ ${#METERS[@]} -gt 0 ] || return 0 - setsid nohup bash -c 'for p in $1; do while kill -0 "$p" 2>/dev/null; do sleep 30; done; done + setsid nohup bash -c 'for p in $1; do while kill -0 "$p" 2>/dev/null; do sleep 1; done; done for m in $2; do kill -- "-$m" 2>/dev/null || kill "$m" 2>/dev/null || true; done' _ "$*" "${METERS[*]}" \ > /dev/null 2>&1 < /dev/null & } +meter_health() { # port: the /health of a meter still up three seconds on (one being stopped is gone by then) + curl -s -m 2 "http://127.0.0.1:$1/health" > /dev/null 2>&1 && sleep 3 \ + && curl -s -m 2 "http://127.0.0.1:$1/health" 2>/dev/null +} trap 'stop_meters_after ${PIDS[@]+"${PIDS[@]}"}' EXIT # after the trainings started below, if any start_meter() { # port upstream ledger log [extra env...]: a meter already on the port must face the same upstream local port=$1 upstream=$2 ledger=$3 log=$4 health; shift 4 - if health=$(curl -s -m 2 "http://127.0.0.1:$port/health" 2>/dev/null); then + if health=$(meter_health $port); then case "$health" in *"\"upstream\": \"$upstream\""*) echo "wire meter already on port $port -> $upstream";; *) echo "REFUSING: port $port has a wire meter facing another or an unknown upstream ($health); choose another WM_PORT"; exit 1;; esac else @@ -119,7 +123,7 @@ start_meter $WM_PORT "$HOST_ORIGIN" "$RUNS/$TAG/wire_ledger.json" "$RUNS/$TAG/wi start_meter $META_PORT "$EVOLVER_ORIGIN" "$RUNS/$TAG/meta_wire_ledger.json" "$RUNS/$TAG/meta_wire_meter.log" # The embedding relay signs the reader's keyless requests with the embedding backend's key (none for a keyless server); # a relay already on the port is refused unless its /health says it signs (`"signs": true`). -if [ -n "$EMBED_API_KEY_ENV" ] && health=$(curl -s -m 2 "http://127.0.0.1:$EMBED_PORT/health" 2>/dev/null); then +if [ -n "$EMBED_API_KEY_ENV" ] && health=$(meter_health $EMBED_PORT); then case "$health" in *'"signs": true'*) ;; *) echo "REFUSING: port $EMBED_PORT has a relay that does not sign requests (started without UPSTREAM_KEY_ENV); choose another WM_PORT"; exit 1;; esac fi start_meter $EMBED_PORT "$EMBED_ORIGIN" "$RUNS/$TAG/embed_wire_ledger.json" "$RUNS/$TAG/embed_wire_meter.log" \ diff --git a/examples/clbench_poker/mwm/launch.sh b/examples/clbench_poker/mwm/launch.sh index bab79338..ac8de031 100755 --- a/examples/clbench_poker/mwm/launch.sh +++ b/examples/clbench_poker/mwm/launch.sh @@ -73,19 +73,23 @@ export MWM_CLBENCH=$CLBENCH MWM_METAWORLD=$METAWORLD command -v codex >/dev/null || { echo "REFUSING: codex CLI not on PATH"; exit 1; } [ -f "$TASKDIR/tasks.jsonl" ] || { echo "REFUSING: run $TASKDIR/prepare.py first"; exit 1; } -# The meters this launch starts stop once the processes it names have ended (#82); a meter found already running is -# left to the launch that started it. +# The meters this launch starts stop within a second or two of the processes it names ending (#82). A meter found +# running belongs to the launch that started it: a launch reuses one only if it is still up on a second look. METERS=() stop_meters_after() { # pid... [ ${#METERS[@]} -gt 0 ] || return 0 - setsid nohup bash -c 'for p in $1; do while kill -0 "$p" 2>/dev/null; do sleep 30; done; done + setsid nohup bash -c 'for p in $1; do while kill -0 "$p" 2>/dev/null; do sleep 1; done; done for m in $2; do kill -- "-$m" 2>/dev/null || kill "$m" 2>/dev/null || true; done' _ "$*" "${METERS[*]}" \ > /dev/null 2>&1 < /dev/null & } +meter_health() { # port: the /health of a meter still up three seconds on (one being stopped is gone by then) + curl -s -m 2 "http://127.0.0.1:$1/health" > /dev/null 2>&1 && sleep 3 \ + && curl -s -m 2 "http://127.0.0.1:$1/health" 2>/dev/null +} trap 'stop_meters_after ${PIDS[@]+"${PIDS[@]}"}' EXIT # after the trainings started below, if any start_meter() { # port upstream ledger log [extra env...]: a meter already on the port must face the same upstream local port=$1 upstream=$2 ledger=$3 log=$4 health; shift 4 - if health=$(curl -s -m 2 "http://127.0.0.1:$port/health" 2>/dev/null); then + if health=$(meter_health $port); then case "$health" in *"\"upstream\": \"$upstream\""*) echo "wire meter already on port $port -> $upstream";; *) echo "REFUSING: port $port has a wire meter facing another or an unknown upstream ($health); choose another WM_PORT"; exit 1;; esac else diff --git a/examples/clbench_poker/prompt_only/launch.sh b/examples/clbench_poker/prompt_only/launch.sh index 4446b701..c298154b 100755 --- a/examples/clbench_poker/prompt_only/launch.sh +++ b/examples/clbench_poker/prompt_only/launch.sh @@ -77,19 +77,23 @@ export MWM_CLBENCH=$CLBENCH MWM_METAWORLD=$METAWORLD command -v codex >/dev/null || { echo "REFUSING: codex CLI not on PATH"; exit 1; } [ -f "$TASKDIR/tasks.jsonl" ] || { echo "REFUSING: run $TASKDIR/prepare.py first"; exit 1; } -# The meters this launch starts stop once the processes it names have ended (#82); a meter found already running is -# left to the launch that started it. +# The meters this launch starts stop within a second or two of the processes it names ending (#82). A meter found +# running belongs to the launch that started it: a launch reuses one only if it is still up on a second look. METERS=() stop_meters_after() { # pid... [ ${#METERS[@]} -gt 0 ] || return 0 - setsid nohup bash -c 'for p in $1; do while kill -0 "$p" 2>/dev/null; do sleep 30; done; done + setsid nohup bash -c 'for p in $1; do while kill -0 "$p" 2>/dev/null; do sleep 1; done; done for m in $2; do kill -- "-$m" 2>/dev/null || kill "$m" 2>/dev/null || true; done' _ "$*" "${METERS[*]}" \ > /dev/null 2>&1 < /dev/null & } +meter_health() { # port: the /health of a meter still up three seconds on (one being stopped is gone by then) + curl -s -m 2 "http://127.0.0.1:$1/health" > /dev/null 2>&1 && sleep 3 \ + && curl -s -m 2 "http://127.0.0.1:$1/health" 2>/dev/null +} trap 'stop_meters_after ${PIDS[@]+"${PIDS[@]}"}' EXIT # after the trainings started below, if any start_meter() { # port upstream ledger log [extra env...]: a meter already on the port must face the same upstream local port=$1 upstream=$2 ledger=$3 log=$4 health; shift 4 - if health=$(curl -s -m 2 "http://127.0.0.1:$port/health" 2>/dev/null); then + if health=$(meter_health $port); then case "$health" in *"\"upstream\": \"$upstream\""*) echo "wire meter already on port $port -> $upstream";; *) echo "REFUSING: port $port has a wire meter facing another or an unknown upstream ($health); choose another WM_PORT"; exit 1;; esac else diff --git a/examples/clbench_poker/reharness/launch.sh b/examples/clbench_poker/reharness/launch.sh index f34b461a..8be9e4d4 100755 --- a/examples/clbench_poker/reharness/launch.sh +++ b/examples/clbench_poker/reharness/launch.sh @@ -85,19 +85,23 @@ export MWM_CLBENCH=$CLBENCH MWM_METAWORLD=$METAWORLD [ -f "$TASKDIR/tasks.jsonl" ] || { echo "REFUSING: run $TASKDIR/prepare.py first"; exit 1; } -# The meters this launch starts stop once the processes it names have ended (#82); a meter found already running is -# left to the launch that started it. +# The meters this launch starts stop within a second or two of the processes it names ending (#82). A meter found +# running belongs to the launch that started it: a launch reuses one only if it is still up on a second look. METERS=() stop_meters_after() { # pid... [ ${#METERS[@]} -gt 0 ] || return 0 - setsid nohup bash -c 'for p in $1; do while kill -0 "$p" 2>/dev/null; do sleep 30; done; done + setsid nohup bash -c 'for p in $1; do while kill -0 "$p" 2>/dev/null; do sleep 1; done; done for m in $2; do kill -- "-$m" 2>/dev/null || kill "$m" 2>/dev/null || true; done' _ "$*" "${METERS[*]}" \ > /dev/null 2>&1 < /dev/null & } +meter_health() { # port: the /health of a meter still up three seconds on (one being stopped is gone by then) + curl -s -m 2 "http://127.0.0.1:$1/health" > /dev/null 2>&1 && sleep 3 \ + && curl -s -m 2 "http://127.0.0.1:$1/health" 2>/dev/null +} trap 'stop_meters_after ${PIDS[@]+"${PIDS[@]}"}' EXIT # after the trainings started below, if any start_meter() { # port upstream ledger log [extra env...]: a meter already on the port must face the same upstream local port=$1 upstream=$2 ledger=$3 log=$4 health; shift 4 - if health=$(curl -s -m 2 "http://127.0.0.1:$port/health" 2>/dev/null); then + if health=$(meter_health $port); then case "$health" in *"\"upstream\": \"$upstream\""*) echo "wire meter already on port $port -> $upstream";; *) echo "REFUSING: port $port has a wire meter facing another or an unknown upstream ($health); choose another WM_PORT"; exit 1;; esac else @@ -114,7 +118,7 @@ start_meter $WM_PORT "$HOST_ORIGIN" "$RUNS/$TAG/wire_ledger.json" "$RUNS/$TAG/wi start_meter $META_PORT "$EVOLVER_ORIGIN" "$RUNS/$TAG/meta_wire_ledger.json" "$RUNS/$TAG/meta_wire_meter.log" # The embedding relay signs the reader's keyless requests with the embedding backend's key (none for a keyless server); # a relay already on the port is refused unless its /health says it signs (`"signs": true`). -if [ -n "$EMBED_API_KEY_ENV" ] && health=$(curl -s -m 2 "http://127.0.0.1:$EMBED_PORT/health" 2>/dev/null); then +if [ -n "$EMBED_API_KEY_ENV" ] && health=$(meter_health $EMBED_PORT); then case "$health" in *'"signs": true'*) ;; *) echo "REFUSING: port $EMBED_PORT has a relay that does not sign requests (started without UPSTREAM_KEY_ENV); choose another WM_PORT"; exit 1;; esac fi start_meter $EMBED_PORT "$EMBED_ORIGIN" "$RUNS/$TAG/embed_wire_ledger.json" "$RUNS/$TAG/embed_wire_meter.log" \ diff --git a/examples/officeqa/harness2_stream/launch.sh b/examples/officeqa/harness2_stream/launch.sh index aa16a53e..a999cdc5 100755 --- a/examples/officeqa/harness2_stream/launch.sh +++ b/examples/officeqa/harness2_stream/launch.sh @@ -65,18 +65,22 @@ fi mkdir -p runs "$PY" make_seed_codex.py "$PY" stream.py --seed "$SEED" --split $SPLITS --out "$STREAM" ${MAX_TASKS:+--max "$MAX_TASKS"} ${SKIP_TASKS:+--skip "$SKIP_TASKS"} -# The meters this launch starts stop once the processes it names have ended (#82); a meter found already running is -# left to the launch that started it. +# The meters this launch starts stop within a second or two of the processes it names ending (#82). A meter found +# running belongs to the launch that started it: a launch reuses one only if it is still up on a second look. METERS=() stop_meters_after() { # pid... [ ${#METERS[@]} -gt 0 ] || return 0 - setsid nohup bash -c 'for p in $1; do while kill -0 "$p" 2>/dev/null; do sleep 30; done; done + setsid nohup bash -c 'for p in $1; do while kill -0 "$p" 2>/dev/null; do sleep 1; done; done for m in $2; do kill -- "-$m" 2>/dev/null || kill "$m" 2>/dev/null || true; done' _ "$*" "${METERS[*]}" \ > /dev/null 2>&1 < /dev/null & } +meter_health() { # port: the /health of a meter still up three seconds on (one being stopped is gone by then) + curl -s -m 2 "http://127.0.0.1:$1/health" > /dev/null 2>&1 && sleep 3 \ + && curl -s -m 2 "http://127.0.0.1:$1/health" 2>/dev/null +} if [ "$EDITORS" = claude ]; then # The meter restarts if it dies; each start keeps its own ledger (sum runs/$TAG.wire_ledger-*.json for the bill). - if ! curl -s -m 2 "http://127.0.0.1:$WM_PORT/health" > /dev/null; then + if ! meter_health $WM_PORT > /dev/null; then setsid nohup bash -c "while true; do PORT=$WM_PORT UPSTREAM=https://api.fireworks.ai \ LEDGER=$HERE/runs/$TAG.wire_ledger-\$(date -u +%Y%m%dT%H%M%SZ).json '$PY' -m hermit.runtime.wire_meter; \ sleep 5; done" >> "runs/$TAG.wire_meter.log" 2>&1 < /dev/null & diff --git a/examples/officeqa/reharness/launch.sh b/examples/officeqa/reharness/launch.sh index 73798b87..33dd157f 100755 --- a/examples/officeqa/reharness/launch.sh +++ b/examples/officeqa/reharness/launch.sh @@ -75,17 +75,21 @@ if [ "${EMBED_KEY_REQUIRED:-1}" = 1 ] && [ -n "$KEY_ENV" ] && [ -z "${!KEY_ENV:- fi # The meter: one per port, started here if none answers; a meter facing another upstream is refused. mkdir -p runs -# The meters this launch starts stop once the processes it names have ended (#82); a meter found already running is -# left to the launch that started it. +# The meters this launch starts stop within a second or two of the processes it names ending (#82). A meter found +# running belongs to the launch that started it: a launch reuses one only if it is still up on a second look. METERS=() stop_meters_after() { # pid... [ ${#METERS[@]} -gt 0 ] || return 0 - setsid nohup bash -c 'for p in $1; do while kill -0 "$p" 2>/dev/null; do sleep 30; done; done + setsid nohup bash -c 'for p in $1; do while kill -0 "$p" 2>/dev/null; do sleep 1; done; done for m in $2; do kill -- "-$m" 2>/dev/null || kill "$m" 2>/dev/null || true; done' _ "$*" "${METERS[*]}" \ > /dev/null 2>&1 < /dev/null & } +meter_health() { # port: the /health of a meter still up three seconds on (one being stopped is gone by then) + curl -s -m 2 "http://127.0.0.1:$1/health" > /dev/null 2>&1 && sleep 3 \ + && curl -s -m 2 "http://127.0.0.1:$1/health" 2>/dev/null +} LEDGER=$HERE/runs/wire_ledger-$WM_PORT.json -if health=$(curl -s -m 2 "http://127.0.0.1:$WM_PORT/health" 2>/dev/null); then +if health=$(meter_health $WM_PORT); then case "$health" in *"\"upstream\": \"$EMBED_ORIGIN\""*'"signs": true'*|*'"signs": true'*"\"upstream\": \"$EMBED_ORIGIN\""*) echo "wire meter already on port $WM_PORT -> $EMBED_ORIGIN (signs)";; diff --git a/examples/servelearnbench/mwm/launch.sh b/examples/servelearnbench/mwm/launch.sh index 28bd9dff..540888ed 100755 --- a/examples/servelearnbench/mwm/launch.sh +++ b/examples/servelearnbench/mwm/launch.sh @@ -81,19 +81,23 @@ HOST_KEY=""; [ -n "$HOST_API_KEY_ENV" ] && HOST_KEY=${!HOST_API_KEY_ENV:-} export SLB_MODEL=$HOST_MODEL_ID SLB_API_KEY=${HOST_KEY:-local} export SLB_REASONING_EFFORT=$EFFORT SLB_METHOD=$METHOD SLB_WORKDIR=$WD -# The meters this launch starts stop once the processes it names have ended (#82); a meter found already running is -# left to the launch that started it. +# The meters this launch starts stop within a second or two of the processes it names ending (#82). A meter found +# running belongs to the launch that started it: a launch reuses one only if it is still up on a second look. METERS=() stop_meters_after() { # pid... [ ${#METERS[@]} -gt 0 ] || return 0 - setsid nohup bash -c 'for p in $1; do while kill -0 "$p" 2>/dev/null; do sleep 30; done; done + setsid nohup bash -c 'for p in $1; do while kill -0 "$p" 2>/dev/null; do sleep 1; done; done for m in $2; do kill -- "-$m" 2>/dev/null || kill "$m" 2>/dev/null || true; done' _ "$*" "${METERS[*]}" \ > /dev/null 2>&1 < /dev/null & } -trap 'stop_meters_after ${TEST_PID:-}' EXIT # train mode: after the tests that follow the training +meter_health() { # port: the /health of a meter still up three seconds on (one being stopped is gone by then) + curl -s -m 2 "http://127.0.0.1:$1/health" > /dev/null 2>&1 && sleep 3 \ + && curl -s -m 2 "http://127.0.0.1:$1/health" 2>/dev/null +} +trap 'stop_meters_after ${TRAIN_PID:-} ${TEST_PID:-}' EXIT # train mode: after the training and the tests after it start_meter() { # port upstream ledger log [extra env...]: a meter already on the port must face the same upstream local port=$1 upstream=$2 ledger=$3 log=$4 health; shift 4 - if health=$(curl -s -m 2 "http://127.0.0.1:$port/health" 2>/dev/null); then + if health=$(meter_health $port); then case "$health" in *"\"upstream\": \"$upstream\""*) echo "wire meter already on port $port -> $upstream";; *) echo "REFUSING: port $port has a wire meter facing another or an unknown upstream ($health); choose another WM_PORT"; exit 1;; esac else From 66fd8a613eddf8f490509c9ded9c5986115d622a Mon Sep 17 00:00:00 2001 From: Haizhong Zheng Date: Mon, 9 Nov 2026 15:18:50 -0500 Subject: [PATCH 3/3] Wire meters: a shared meter stops after every launch using it; a restarted one keeps its ledger Review fixes for #82: - A meter started on a ledger an earlier one left carries its totals on (LEDGER_RESUME=1 in hermit/runtime/wire_meter.py, set by the eight launchers whose ledger outlives a meter). Once the launchers stop their meters, the next meter on the port (`run && run eval`, a ServeLearnBench test run again, a CL-Bench tag refused because it exists) wrote an empty ledger over the earlier bill. Off by default: lean_swarm's rerun of an attempt starts empty as before. - Every launch on a port lists its processes in /tmp/hermit-wire-meter--.users, and the launch that started the meter stops it only once none of them is alive, so a launch that reuses a running meter no longer loses it when the starter's run ends. - OfficeQA ReHarness prints the meter's pid, not the watcher's. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 11 +++-- examples/clbench_database/mwm/launch.sh | 38 +++++++++++----- .../clbench_database/prompt_only/launch.sh | 38 +++++++++++----- examples/clbench_database/reharness/launch.sh | 38 +++++++++++----- examples/clbench_poker/mwm/launch.sh | 38 +++++++++++----- examples/clbench_poker/prompt_only/launch.sh | 38 +++++++++++----- examples/clbench_poker/reharness/launch.sh | 38 +++++++++++----- examples/officeqa/harness2_stream/launch.sh | 35 ++++++++++----- examples/officeqa/reharness/launch.sh | 40 ++++++++++++----- examples/servelearnbench/mwm/launch.sh | 38 +++++++++++----- hermit/runtime/wire_meter.py | 25 ++++++++++- tests/core/test_backends.py | 44 +++++++++++++++++++ 12 files changed, 318 insertions(+), 103 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ed3c7ce6..bbaf2a3a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -204,9 +204,14 @@ Changes are grouped by software version. Entries under Unreleased describe work wrote none. The file (or a directory a session made in its place) is now removed before each Evolve session. - Nine example launchers left their wire meters running after the run (#82), so a later run on the same port reused one and wrote its records into the earlier run's ledger. The meters a launch starts now stop within a second - or two of its trainings (and, for ServeLearnBench, the tests after them) ending, the harness2 restart loop - included. A launch reuses a running meter only if it is still up three seconds on, so it never takes one that is - being stopped; a meter shared by two runs at once still stops when the run that started it ends. + or two of the runs that use them ending: its own trainings (and, for ServeLearnBench, the tests after them) and + those of any launch that reuses one, which each list their processes in the port's users file + (`/tmp/hermit-wire-meter--.users`); the harness2 restart loop is stopped with its meter. A + launch reuses a running meter only if it is still up three seconds on, so it never takes one that is being + stopped. A meter started on a ledger an earlier one left now carries its totals on (`LEDGER_RESUME=1` in + `hermit/runtime/wire_meter.py`, which these launchers set): the next run's meter on the port (`launch.sh run && + launch.sh run eval`, a ServeLearnBench `test` run again, a CL-Bench tag refused because it exists) wrote an empty + ledger over the earlier run's bill. ## 0.2.0 - 2026-11-04 diff --git a/examples/clbench_database/mwm/launch.sh b/examples/clbench_database/mwm/launch.sh index 8a7f83e4..067133f3 100755 --- a/examples/clbench_database/mwm/launch.sh +++ b/examples/clbench_database/mwm/launch.sh @@ -78,29 +78,45 @@ fi command -v codex >/dev/null || { echo "REFUSING: codex CLI not on PATH"; exit 1; } [ -f "$TASKDIR/tasks.jsonl" ] || { echo "REFUSING: run $TASKDIR/prepare.py first"; exit 1; } -# The meters this launch starts stop within a second or two of the processes it names ending (#82). A meter found -# running belongs to the launch that started it: a launch reuses one only if it is still up on a second look. -METERS=() -stop_meters_after() { # pid... +# The meters this launch starts stop within a second or two of the processes that use them ending (#82): its own and +# those of any launch that reuses one. Each launch on a port lists its processes in the port's users file, emptied +# when the meter stops; a launch reuses a running meter only if it is still up on a second look (one being stopped is +# gone by then). +METERS=() OWNED=() PORTS=() +meter_users() { echo "/tmp/hermit-wire-meter-$(id -u)-$1.users"; } # port: /tmp, which every launch on it shares +meter_started() { # port pid: a meter this launch started, this launch one of its users + { echo $$ >> "$(meter_users "$1")" || true; } 2>/dev/null + METERS+=("$2"); OWNED+=("$1") +} +stop_meters_after() { # pid...: what this launch leaves running; the meters it started stop after every user + local port files=() + [ $# -eq 0 ] || for port in ${PORTS[@]+"${PORTS[@]}"}; do + printf '%s\n' "$@" >> "$(meter_users "$port")" 2>/dev/null || true; done [ ${#METERS[@]} -gt 0 ] || return 0 - setsid nohup bash -c 'for p in $1; do while kill -0 "$p" 2>/dev/null; do sleep 1; done; done - for m in $2; do kill -- "-$m" 2>/dev/null || kill "$m" 2>/dev/null || true; done' _ "$*" "${METERS[*]}" \ + for port in "${OWNED[@]}"; do files+=("$(meter_users "$port")"); done + setsid nohup bash -c 'while :; do up= + for p in $(cat $1 2>/dev/null); do kill -0 "$p" 2>/dev/null && { up=1; break; }; done + [ -n "$up" ] || break; sleep 1; done + for f in $1; do : > "$f"; done 2>/dev/null # a launch listed after this look finds the meter gone and starts one + for m in $2; do kill -- "-$m" 2>/dev/null || kill "$m" 2>/dev/null || true; done' _ "${files[*]}" "${METERS[*]}" \ > /dev/null 2>&1 < /dev/null & } -meter_health() { # port: the /health of a meter still up three seconds on (one being stopped is gone by then) - curl -s -m 2 "http://127.0.0.1:$1/health" > /dev/null 2>&1 && sleep 3 \ - && curl -s -m 2 "http://127.0.0.1:$1/health" 2>/dev/null +meter_health() { # port: the /health of a meter still up three seconds on, this launch listed as one of its users first + curl -s -m 2 "http://127.0.0.1:$1/health" > /dev/null 2>&1 \ + && { echo $$ >> "$(meter_users "$1")" || true; } 2>/dev/null \ + && sleep 3 && curl -s -m 2 "http://127.0.0.1:$1/health" 2>/dev/null } trap 'stop_meters_after ${PIDS[@]+"${PIDS[@]}"}' EXIT # after the trainings started below, if any start_meter() { # port upstream ledger log [extra env...]: a meter already on the port must face the same upstream local port=$1 upstream=$2 ledger=$3 log=$4 health; shift 4 + PORTS+=("$port") # started here or reused: this launch's processes use it if health=$(meter_health $port); then case "$health" in *"\"upstream\": \"$upstream\""*) echo "wire meter already on port $port -> $upstream";; *) echo "REFUSING: port $port has a wire meter facing another or an unknown upstream ($health); choose another WM_PORT"; exit 1;; esac else - env "$@" PORT=$port LEDGER="$ledger" UPSTREAM="$upstream" \ + env "$@" PORT=$port LEDGER="$ledger" LEDGER_RESUME=1 UPSTREAM="$upstream" \ setsid nohup "$PY" "$METER" > "$log" 2>&1 < /dev/null & - METERS+=("$!") + meter_started $port $! echo "wire meter pid $! port $port -> $upstream"; sleep 2 fi } diff --git a/examples/clbench_database/prompt_only/launch.sh b/examples/clbench_database/prompt_only/launch.sh index 2f186b1f..99cc5904 100755 --- a/examples/clbench_database/prompt_only/launch.sh +++ b/examples/clbench_database/prompt_only/launch.sh @@ -82,29 +82,45 @@ fi command -v codex >/dev/null || { echo "REFUSING: codex CLI not on PATH"; exit 1; } [ -f "$TASKDIR/tasks.jsonl" ] || { echo "REFUSING: run $TASKDIR/prepare.py first"; exit 1; } -# The meters this launch starts stop within a second or two of the processes it names ending (#82). A meter found -# running belongs to the launch that started it: a launch reuses one only if it is still up on a second look. -METERS=() -stop_meters_after() { # pid... +# The meters this launch starts stop within a second or two of the processes that use them ending (#82): its own and +# those of any launch that reuses one. Each launch on a port lists its processes in the port's users file, emptied +# when the meter stops; a launch reuses a running meter only if it is still up on a second look (one being stopped is +# gone by then). +METERS=() OWNED=() PORTS=() +meter_users() { echo "/tmp/hermit-wire-meter-$(id -u)-$1.users"; } # port: /tmp, which every launch on it shares +meter_started() { # port pid: a meter this launch started, this launch one of its users + { echo $$ >> "$(meter_users "$1")" || true; } 2>/dev/null + METERS+=("$2"); OWNED+=("$1") +} +stop_meters_after() { # pid...: what this launch leaves running; the meters it started stop after every user + local port files=() + [ $# -eq 0 ] || for port in ${PORTS[@]+"${PORTS[@]}"}; do + printf '%s\n' "$@" >> "$(meter_users "$port")" 2>/dev/null || true; done [ ${#METERS[@]} -gt 0 ] || return 0 - setsid nohup bash -c 'for p in $1; do while kill -0 "$p" 2>/dev/null; do sleep 1; done; done - for m in $2; do kill -- "-$m" 2>/dev/null || kill "$m" 2>/dev/null || true; done' _ "$*" "${METERS[*]}" \ + for port in "${OWNED[@]}"; do files+=("$(meter_users "$port")"); done + setsid nohup bash -c 'while :; do up= + for p in $(cat $1 2>/dev/null); do kill -0 "$p" 2>/dev/null && { up=1; break; }; done + [ -n "$up" ] || break; sleep 1; done + for f in $1; do : > "$f"; done 2>/dev/null # a launch listed after this look finds the meter gone and starts one + for m in $2; do kill -- "-$m" 2>/dev/null || kill "$m" 2>/dev/null || true; done' _ "${files[*]}" "${METERS[*]}" \ > /dev/null 2>&1 < /dev/null & } -meter_health() { # port: the /health of a meter still up three seconds on (one being stopped is gone by then) - curl -s -m 2 "http://127.0.0.1:$1/health" > /dev/null 2>&1 && sleep 3 \ - && curl -s -m 2 "http://127.0.0.1:$1/health" 2>/dev/null +meter_health() { # port: the /health of a meter still up three seconds on, this launch listed as one of its users first + curl -s -m 2 "http://127.0.0.1:$1/health" > /dev/null 2>&1 \ + && { echo $$ >> "$(meter_users "$1")" || true; } 2>/dev/null \ + && sleep 3 && curl -s -m 2 "http://127.0.0.1:$1/health" 2>/dev/null } trap 'stop_meters_after ${PIDS[@]+"${PIDS[@]}"}' EXIT # after the trainings started below, if any start_meter() { # port upstream ledger log [extra env...]: a meter already on the port must face the same upstream local port=$1 upstream=$2 ledger=$3 log=$4 health; shift 4 + PORTS+=("$port") # started here or reused: this launch's processes use it if health=$(meter_health $port); then case "$health" in *"\"upstream\": \"$upstream\""*) echo "wire meter already on port $port -> $upstream";; *) echo "REFUSING: port $port has a wire meter facing another or an unknown upstream ($health); choose another WM_PORT"; exit 1;; esac else - env "$@" PORT=$port LEDGER="$ledger" UPSTREAM="$upstream" \ + env "$@" PORT=$port LEDGER="$ledger" LEDGER_RESUME=1 UPSTREAM="$upstream" \ setsid nohup "$PY" "$METER" > "$log" 2>&1 < /dev/null & - METERS+=("$!") + meter_started $port $! echo "wire meter pid $! port $port -> $upstream"; sleep 2 fi } diff --git a/examples/clbench_database/reharness/launch.sh b/examples/clbench_database/reharness/launch.sh index 49550ab4..2b9bee18 100755 --- a/examples/clbench_database/reharness/launch.sh +++ b/examples/clbench_database/reharness/launch.sh @@ -90,29 +90,45 @@ fi [ -f "$TASKDIR/tasks.jsonl" ] || { echo "REFUSING: run $TASKDIR/prepare.py first"; exit 1; } -# The meters this launch starts stop within a second or two of the processes it names ending (#82). A meter found -# running belongs to the launch that started it: a launch reuses one only if it is still up on a second look. -METERS=() -stop_meters_after() { # pid... +# The meters this launch starts stop within a second or two of the processes that use them ending (#82): its own and +# those of any launch that reuses one. Each launch on a port lists its processes in the port's users file, emptied +# when the meter stops; a launch reuses a running meter only if it is still up on a second look (one being stopped is +# gone by then). +METERS=() OWNED=() PORTS=() +meter_users() { echo "/tmp/hermit-wire-meter-$(id -u)-$1.users"; } # port: /tmp, which every launch on it shares +meter_started() { # port pid: a meter this launch started, this launch one of its users + { echo $$ >> "$(meter_users "$1")" || true; } 2>/dev/null + METERS+=("$2"); OWNED+=("$1") +} +stop_meters_after() { # pid...: what this launch leaves running; the meters it started stop after every user + local port files=() + [ $# -eq 0 ] || for port in ${PORTS[@]+"${PORTS[@]}"}; do + printf '%s\n' "$@" >> "$(meter_users "$port")" 2>/dev/null || true; done [ ${#METERS[@]} -gt 0 ] || return 0 - setsid nohup bash -c 'for p in $1; do while kill -0 "$p" 2>/dev/null; do sleep 1; done; done - for m in $2; do kill -- "-$m" 2>/dev/null || kill "$m" 2>/dev/null || true; done' _ "$*" "${METERS[*]}" \ + for port in "${OWNED[@]}"; do files+=("$(meter_users "$port")"); done + setsid nohup bash -c 'while :; do up= + for p in $(cat $1 2>/dev/null); do kill -0 "$p" 2>/dev/null && { up=1; break; }; done + [ -n "$up" ] || break; sleep 1; done + for f in $1; do : > "$f"; done 2>/dev/null # a launch listed after this look finds the meter gone and starts one + for m in $2; do kill -- "-$m" 2>/dev/null || kill "$m" 2>/dev/null || true; done' _ "${files[*]}" "${METERS[*]}" \ > /dev/null 2>&1 < /dev/null & } -meter_health() { # port: the /health of a meter still up three seconds on (one being stopped is gone by then) - curl -s -m 2 "http://127.0.0.1:$1/health" > /dev/null 2>&1 && sleep 3 \ - && curl -s -m 2 "http://127.0.0.1:$1/health" 2>/dev/null +meter_health() { # port: the /health of a meter still up three seconds on, this launch listed as one of its users first + curl -s -m 2 "http://127.0.0.1:$1/health" > /dev/null 2>&1 \ + && { echo $$ >> "$(meter_users "$1")" || true; } 2>/dev/null \ + && sleep 3 && curl -s -m 2 "http://127.0.0.1:$1/health" 2>/dev/null } trap 'stop_meters_after ${PIDS[@]+"${PIDS[@]}"}' EXIT # after the trainings started below, if any start_meter() { # port upstream ledger log [extra env...]: a meter already on the port must face the same upstream local port=$1 upstream=$2 ledger=$3 log=$4 health; shift 4 + PORTS+=("$port") # started here or reused: this launch's processes use it if health=$(meter_health $port); then case "$health" in *"\"upstream\": \"$upstream\""*) echo "wire meter already on port $port -> $upstream";; *) echo "REFUSING: port $port has a wire meter facing another or an unknown upstream ($health); choose another WM_PORT"; exit 1;; esac else - env "$@" PORT=$port LEDGER="$ledger" UPSTREAM="$upstream" \ + env "$@" PORT=$port LEDGER="$ledger" LEDGER_RESUME=1 UPSTREAM="$upstream" \ setsid nohup "$PY" "$METER" > "$log" 2>&1 < /dev/null & - METERS+=("$!") + meter_started $port $! echo "wire meter pid $! port $port -> $upstream"; sleep 2 curl -s -m 2 "http://127.0.0.1:$port/health" > /dev/null \ || { echo "REFUSING: the wire meter on port $port did not come up (see $log)"; exit 1; } diff --git a/examples/clbench_poker/mwm/launch.sh b/examples/clbench_poker/mwm/launch.sh index ac8de031..c083e428 100755 --- a/examples/clbench_poker/mwm/launch.sh +++ b/examples/clbench_poker/mwm/launch.sh @@ -73,29 +73,45 @@ export MWM_CLBENCH=$CLBENCH MWM_METAWORLD=$METAWORLD command -v codex >/dev/null || { echo "REFUSING: codex CLI not on PATH"; exit 1; } [ -f "$TASKDIR/tasks.jsonl" ] || { echo "REFUSING: run $TASKDIR/prepare.py first"; exit 1; } -# The meters this launch starts stop within a second or two of the processes it names ending (#82). A meter found -# running belongs to the launch that started it: a launch reuses one only if it is still up on a second look. -METERS=() -stop_meters_after() { # pid... +# The meters this launch starts stop within a second or two of the processes that use them ending (#82): its own and +# those of any launch that reuses one. Each launch on a port lists its processes in the port's users file, emptied +# when the meter stops; a launch reuses a running meter only if it is still up on a second look (one being stopped is +# gone by then). +METERS=() OWNED=() PORTS=() +meter_users() { echo "/tmp/hermit-wire-meter-$(id -u)-$1.users"; } # port: /tmp, which every launch on it shares +meter_started() { # port pid: a meter this launch started, this launch one of its users + { echo $$ >> "$(meter_users "$1")" || true; } 2>/dev/null + METERS+=("$2"); OWNED+=("$1") +} +stop_meters_after() { # pid...: what this launch leaves running; the meters it started stop after every user + local port files=() + [ $# -eq 0 ] || for port in ${PORTS[@]+"${PORTS[@]}"}; do + printf '%s\n' "$@" >> "$(meter_users "$port")" 2>/dev/null || true; done [ ${#METERS[@]} -gt 0 ] || return 0 - setsid nohup bash -c 'for p in $1; do while kill -0 "$p" 2>/dev/null; do sleep 1; done; done - for m in $2; do kill -- "-$m" 2>/dev/null || kill "$m" 2>/dev/null || true; done' _ "$*" "${METERS[*]}" \ + for port in "${OWNED[@]}"; do files+=("$(meter_users "$port")"); done + setsid nohup bash -c 'while :; do up= + for p in $(cat $1 2>/dev/null); do kill -0 "$p" 2>/dev/null && { up=1; break; }; done + [ -n "$up" ] || break; sleep 1; done + for f in $1; do : > "$f"; done 2>/dev/null # a launch listed after this look finds the meter gone and starts one + for m in $2; do kill -- "-$m" 2>/dev/null || kill "$m" 2>/dev/null || true; done' _ "${files[*]}" "${METERS[*]}" \ > /dev/null 2>&1 < /dev/null & } -meter_health() { # port: the /health of a meter still up three seconds on (one being stopped is gone by then) - curl -s -m 2 "http://127.0.0.1:$1/health" > /dev/null 2>&1 && sleep 3 \ - && curl -s -m 2 "http://127.0.0.1:$1/health" 2>/dev/null +meter_health() { # port: the /health of a meter still up three seconds on, this launch listed as one of its users first + curl -s -m 2 "http://127.0.0.1:$1/health" > /dev/null 2>&1 \ + && { echo $$ >> "$(meter_users "$1")" || true; } 2>/dev/null \ + && sleep 3 && curl -s -m 2 "http://127.0.0.1:$1/health" 2>/dev/null } trap 'stop_meters_after ${PIDS[@]+"${PIDS[@]}"}' EXIT # after the trainings started below, if any start_meter() { # port upstream ledger log [extra env...]: a meter already on the port must face the same upstream local port=$1 upstream=$2 ledger=$3 log=$4 health; shift 4 + PORTS+=("$port") # started here or reused: this launch's processes use it if health=$(meter_health $port); then case "$health" in *"\"upstream\": \"$upstream\""*) echo "wire meter already on port $port -> $upstream";; *) echo "REFUSING: port $port has a wire meter facing another or an unknown upstream ($health); choose another WM_PORT"; exit 1;; esac else - env "$@" PORT=$port LEDGER="$ledger" UPSTREAM="$upstream" \ + env "$@" PORT=$port LEDGER="$ledger" LEDGER_RESUME=1 UPSTREAM="$upstream" \ setsid nohup "$PY" "$METER" > "$log" 2>&1 < /dev/null & - METERS+=("$!") + meter_started $port $! echo "wire meter pid $! port $port -> $upstream"; sleep 2 fi } diff --git a/examples/clbench_poker/prompt_only/launch.sh b/examples/clbench_poker/prompt_only/launch.sh index c298154b..f6485ad0 100755 --- a/examples/clbench_poker/prompt_only/launch.sh +++ b/examples/clbench_poker/prompt_only/launch.sh @@ -77,29 +77,45 @@ export MWM_CLBENCH=$CLBENCH MWM_METAWORLD=$METAWORLD command -v codex >/dev/null || { echo "REFUSING: codex CLI not on PATH"; exit 1; } [ -f "$TASKDIR/tasks.jsonl" ] || { echo "REFUSING: run $TASKDIR/prepare.py first"; exit 1; } -# The meters this launch starts stop within a second or two of the processes it names ending (#82). A meter found -# running belongs to the launch that started it: a launch reuses one only if it is still up on a second look. -METERS=() -stop_meters_after() { # pid... +# The meters this launch starts stop within a second or two of the processes that use them ending (#82): its own and +# those of any launch that reuses one. Each launch on a port lists its processes in the port's users file, emptied +# when the meter stops; a launch reuses a running meter only if it is still up on a second look (one being stopped is +# gone by then). +METERS=() OWNED=() PORTS=() +meter_users() { echo "/tmp/hermit-wire-meter-$(id -u)-$1.users"; } # port: /tmp, which every launch on it shares +meter_started() { # port pid: a meter this launch started, this launch one of its users + { echo $$ >> "$(meter_users "$1")" || true; } 2>/dev/null + METERS+=("$2"); OWNED+=("$1") +} +stop_meters_after() { # pid...: what this launch leaves running; the meters it started stop after every user + local port files=() + [ $# -eq 0 ] || for port in ${PORTS[@]+"${PORTS[@]}"}; do + printf '%s\n' "$@" >> "$(meter_users "$port")" 2>/dev/null || true; done [ ${#METERS[@]} -gt 0 ] || return 0 - setsid nohup bash -c 'for p in $1; do while kill -0 "$p" 2>/dev/null; do sleep 1; done; done - for m in $2; do kill -- "-$m" 2>/dev/null || kill "$m" 2>/dev/null || true; done' _ "$*" "${METERS[*]}" \ + for port in "${OWNED[@]}"; do files+=("$(meter_users "$port")"); done + setsid nohup bash -c 'while :; do up= + for p in $(cat $1 2>/dev/null); do kill -0 "$p" 2>/dev/null && { up=1; break; }; done + [ -n "$up" ] || break; sleep 1; done + for f in $1; do : > "$f"; done 2>/dev/null # a launch listed after this look finds the meter gone and starts one + for m in $2; do kill -- "-$m" 2>/dev/null || kill "$m" 2>/dev/null || true; done' _ "${files[*]}" "${METERS[*]}" \ > /dev/null 2>&1 < /dev/null & } -meter_health() { # port: the /health of a meter still up three seconds on (one being stopped is gone by then) - curl -s -m 2 "http://127.0.0.1:$1/health" > /dev/null 2>&1 && sleep 3 \ - && curl -s -m 2 "http://127.0.0.1:$1/health" 2>/dev/null +meter_health() { # port: the /health of a meter still up three seconds on, this launch listed as one of its users first + curl -s -m 2 "http://127.0.0.1:$1/health" > /dev/null 2>&1 \ + && { echo $$ >> "$(meter_users "$1")" || true; } 2>/dev/null \ + && sleep 3 && curl -s -m 2 "http://127.0.0.1:$1/health" 2>/dev/null } trap 'stop_meters_after ${PIDS[@]+"${PIDS[@]}"}' EXIT # after the trainings started below, if any start_meter() { # port upstream ledger log [extra env...]: a meter already on the port must face the same upstream local port=$1 upstream=$2 ledger=$3 log=$4 health; shift 4 + PORTS+=("$port") # started here or reused: this launch's processes use it if health=$(meter_health $port); then case "$health" in *"\"upstream\": \"$upstream\""*) echo "wire meter already on port $port -> $upstream";; *) echo "REFUSING: port $port has a wire meter facing another or an unknown upstream ($health); choose another WM_PORT"; exit 1;; esac else - env "$@" PORT=$port LEDGER="$ledger" UPSTREAM="$upstream" \ + env "$@" PORT=$port LEDGER="$ledger" LEDGER_RESUME=1 UPSTREAM="$upstream" \ setsid nohup "$PY" "$METER" > "$log" 2>&1 < /dev/null & - METERS+=("$!") + meter_started $port $! echo "wire meter pid $! port $port -> $upstream"; sleep 2 fi } diff --git a/examples/clbench_poker/reharness/launch.sh b/examples/clbench_poker/reharness/launch.sh index 8be9e4d4..d2e26eec 100755 --- a/examples/clbench_poker/reharness/launch.sh +++ b/examples/clbench_poker/reharness/launch.sh @@ -85,29 +85,45 @@ export MWM_CLBENCH=$CLBENCH MWM_METAWORLD=$METAWORLD [ -f "$TASKDIR/tasks.jsonl" ] || { echo "REFUSING: run $TASKDIR/prepare.py first"; exit 1; } -# The meters this launch starts stop within a second or two of the processes it names ending (#82). A meter found -# running belongs to the launch that started it: a launch reuses one only if it is still up on a second look. -METERS=() -stop_meters_after() { # pid... +# The meters this launch starts stop within a second or two of the processes that use them ending (#82): its own and +# those of any launch that reuses one. Each launch on a port lists its processes in the port's users file, emptied +# when the meter stops; a launch reuses a running meter only if it is still up on a second look (one being stopped is +# gone by then). +METERS=() OWNED=() PORTS=() +meter_users() { echo "/tmp/hermit-wire-meter-$(id -u)-$1.users"; } # port: /tmp, which every launch on it shares +meter_started() { # port pid: a meter this launch started, this launch one of its users + { echo $$ >> "$(meter_users "$1")" || true; } 2>/dev/null + METERS+=("$2"); OWNED+=("$1") +} +stop_meters_after() { # pid...: what this launch leaves running; the meters it started stop after every user + local port files=() + [ $# -eq 0 ] || for port in ${PORTS[@]+"${PORTS[@]}"}; do + printf '%s\n' "$@" >> "$(meter_users "$port")" 2>/dev/null || true; done [ ${#METERS[@]} -gt 0 ] || return 0 - setsid nohup bash -c 'for p in $1; do while kill -0 "$p" 2>/dev/null; do sleep 1; done; done - for m in $2; do kill -- "-$m" 2>/dev/null || kill "$m" 2>/dev/null || true; done' _ "$*" "${METERS[*]}" \ + for port in "${OWNED[@]}"; do files+=("$(meter_users "$port")"); done + setsid nohup bash -c 'while :; do up= + for p in $(cat $1 2>/dev/null); do kill -0 "$p" 2>/dev/null && { up=1; break; }; done + [ -n "$up" ] || break; sleep 1; done + for f in $1; do : > "$f"; done 2>/dev/null # a launch listed after this look finds the meter gone and starts one + for m in $2; do kill -- "-$m" 2>/dev/null || kill "$m" 2>/dev/null || true; done' _ "${files[*]}" "${METERS[*]}" \ > /dev/null 2>&1 < /dev/null & } -meter_health() { # port: the /health of a meter still up three seconds on (one being stopped is gone by then) - curl -s -m 2 "http://127.0.0.1:$1/health" > /dev/null 2>&1 && sleep 3 \ - && curl -s -m 2 "http://127.0.0.1:$1/health" 2>/dev/null +meter_health() { # port: the /health of a meter still up three seconds on, this launch listed as one of its users first + curl -s -m 2 "http://127.0.0.1:$1/health" > /dev/null 2>&1 \ + && { echo $$ >> "$(meter_users "$1")" || true; } 2>/dev/null \ + && sleep 3 && curl -s -m 2 "http://127.0.0.1:$1/health" 2>/dev/null } trap 'stop_meters_after ${PIDS[@]+"${PIDS[@]}"}' EXIT # after the trainings started below, if any start_meter() { # port upstream ledger log [extra env...]: a meter already on the port must face the same upstream local port=$1 upstream=$2 ledger=$3 log=$4 health; shift 4 + PORTS+=("$port") # started here or reused: this launch's processes use it if health=$(meter_health $port); then case "$health" in *"\"upstream\": \"$upstream\""*) echo "wire meter already on port $port -> $upstream";; *) echo "REFUSING: port $port has a wire meter facing another or an unknown upstream ($health); choose another WM_PORT"; exit 1;; esac else - env "$@" PORT=$port LEDGER="$ledger" UPSTREAM="$upstream" \ + env "$@" PORT=$port LEDGER="$ledger" LEDGER_RESUME=1 UPSTREAM="$upstream" \ setsid nohup "$PY" "$METER" > "$log" 2>&1 < /dev/null & - METERS+=("$!") + meter_started $port $! echo "wire meter pid $! port $port -> $upstream"; sleep 2 curl -s -m 2 "http://127.0.0.1:$port/health" > /dev/null \ || { echo "REFUSING: the wire meter on port $port did not come up (see $log)"; exit 1; } diff --git a/examples/officeqa/harness2_stream/launch.sh b/examples/officeqa/harness2_stream/launch.sh index e6f653a2..51c9ecfe 100755 --- a/examples/officeqa/harness2_stream/launch.sh +++ b/examples/officeqa/harness2_stream/launch.sh @@ -98,18 +98,33 @@ print(((c.get('evolver') or {}).get('params') or {}).get('claude_bin') or '')" " fi "$PY" make_seed_codex.py "$PY" stream.py --seed "$SEED" --split $SPLITS --out "$STREAM" ${MAX_TASKS:+--max "$MAX_TASKS"} ${SKIP_TASKS:+--skip "$SKIP_TASKS"} -# The meters this launch starts stop within a second or two of the processes it names ending (#82). A meter found -# running belongs to the launch that started it: a launch reuses one only if it is still up on a second look. -METERS=() -stop_meters_after() { # pid... +# The meters this launch starts stop within a second or two of the processes that use them ending (#82): its own and +# those of any launch that reuses one. Each launch on a port lists its processes in the port's users file, emptied +# when the meter stops; a launch reuses a running meter only if it is still up on a second look (one being stopped is +# gone by then). +METERS=() OWNED=() PORTS=() +meter_users() { echo "/tmp/hermit-wire-meter-$(id -u)-$1.users"; } # port: /tmp, which every launch on it shares +meter_started() { # port pid: a meter this launch started, this launch one of its users + { echo $$ >> "$(meter_users "$1")" || true; } 2>/dev/null + METERS+=("$2"); OWNED+=("$1") +} +stop_meters_after() { # pid...: what this launch leaves running; the meters it started stop after every user + local port files=() + [ $# -eq 0 ] || for port in ${PORTS[@]+"${PORTS[@]}"}; do + printf '%s\n' "$@" >> "$(meter_users "$port")" 2>/dev/null || true; done [ ${#METERS[@]} -gt 0 ] || return 0 - setsid nohup bash -c 'for p in $1; do while kill -0 "$p" 2>/dev/null; do sleep 1; done; done - for m in $2; do kill -- "-$m" 2>/dev/null || kill "$m" 2>/dev/null || true; done' _ "$*" "${METERS[*]}" \ + for port in "${OWNED[@]}"; do files+=("$(meter_users "$port")"); done + setsid nohup bash -c 'while :; do up= + for p in $(cat $1 2>/dev/null); do kill -0 "$p" 2>/dev/null && { up=1; break; }; done + [ -n "$up" ] || break; sleep 1; done + for f in $1; do : > "$f"; done 2>/dev/null # a launch listed after this look finds the meter gone and starts one + for m in $2; do kill -- "-$m" 2>/dev/null || kill "$m" 2>/dev/null || true; done' _ "${files[*]}" "${METERS[*]}" \ > /dev/null 2>&1 < /dev/null & } -meter_health() { # port: the /health of a meter still up three seconds on (one being stopped is gone by then) - curl -s -m 2 "http://127.0.0.1:$1/health" > /dev/null 2>&1 && sleep 3 \ - && curl -s -m 2 "http://127.0.0.1:$1/health" 2>/dev/null +meter_health() { # port: the /health of a meter still up three seconds on, this launch listed as one of its users first + curl -s -m 2 "http://127.0.0.1:$1/health" > /dev/null 2>&1 \ + && { echo $$ >> "$(meter_users "$1")" || true; } 2>/dev/null \ + && sleep 3 && curl -s -m 2 "http://127.0.0.1:$1/health" 2>/dev/null } if [ "$EDITORS" = claude ]; then # The meter restarts if it dies; each start keeps its own ledger (sum runs/$TAG.wire_ledger-*.json for the bill). @@ -117,7 +132,7 @@ if [ "$EDITORS" = claude ]; then setsid nohup bash -c "while true; do PORT=$WM_PORT UPSTREAM=https://api.fireworks.ai \ LEDGER=$HERE/runs/$TAG.wire_ledger-\$(date -u +%Y%m%dT%H%M%SZ).json '$PY' -m hermit.runtime.wire_meter; \ sleep 5; done" >> "runs/$TAG.wire_meter.log" 2>&1 < /dev/null & - METERS+=("$!"); stop_meters_after $$ # the restart loop with its meter, when the run exec'd below ends + meter_started $WM_PORT $!; stop_meters_after $$ # the restart loop with its meter, after the run exec'd below sleep 3 fi # Claude Code updates itself: the run uses a copy of today's binary (or of CLAUDE_BIN, e.g. the stream it continues diff --git a/examples/officeqa/reharness/launch.sh b/examples/officeqa/reharness/launch.sh index 33dd157f..f762c58b 100755 --- a/examples/officeqa/reharness/launch.sh +++ b/examples/officeqa/reharness/launch.sh @@ -75,18 +75,33 @@ if [ "${EMBED_KEY_REQUIRED:-1}" = 1 ] && [ -n "$KEY_ENV" ] && [ -z "${!KEY_ENV:- fi # The meter: one per port, started here if none answers; a meter facing another upstream is refused. mkdir -p runs -# The meters this launch starts stop within a second or two of the processes it names ending (#82). A meter found -# running belongs to the launch that started it: a launch reuses one only if it is still up on a second look. -METERS=() -stop_meters_after() { # pid... +# The meters this launch starts stop within a second or two of the processes that use them ending (#82): its own and +# those of any launch that reuses one. Each launch on a port lists its processes in the port's users file, emptied +# when the meter stops; a launch reuses a running meter only if it is still up on a second look (one being stopped is +# gone by then). +METERS=() OWNED=() PORTS=() +meter_users() { echo "/tmp/hermit-wire-meter-$(id -u)-$1.users"; } # port: /tmp, which every launch on it shares +meter_started() { # port pid: a meter this launch started, this launch one of its users + { echo $$ >> "$(meter_users "$1")" || true; } 2>/dev/null + METERS+=("$2"); OWNED+=("$1") +} +stop_meters_after() { # pid...: what this launch leaves running; the meters it started stop after every user + local port files=() + [ $# -eq 0 ] || for port in ${PORTS[@]+"${PORTS[@]}"}; do + printf '%s\n' "$@" >> "$(meter_users "$port")" 2>/dev/null || true; done [ ${#METERS[@]} -gt 0 ] || return 0 - setsid nohup bash -c 'for p in $1; do while kill -0 "$p" 2>/dev/null; do sleep 1; done; done - for m in $2; do kill -- "-$m" 2>/dev/null || kill "$m" 2>/dev/null || true; done' _ "$*" "${METERS[*]}" \ + for port in "${OWNED[@]}"; do files+=("$(meter_users "$port")"); done + setsid nohup bash -c 'while :; do up= + for p in $(cat $1 2>/dev/null); do kill -0 "$p" 2>/dev/null && { up=1; break; }; done + [ -n "$up" ] || break; sleep 1; done + for f in $1; do : > "$f"; done 2>/dev/null # a launch listed after this look finds the meter gone and starts one + for m in $2; do kill -- "-$m" 2>/dev/null || kill "$m" 2>/dev/null || true; done' _ "${files[*]}" "${METERS[*]}" \ > /dev/null 2>&1 < /dev/null & } -meter_health() { # port: the /health of a meter still up three seconds on (one being stopped is gone by then) - curl -s -m 2 "http://127.0.0.1:$1/health" > /dev/null 2>&1 && sleep 3 \ - && curl -s -m 2 "http://127.0.0.1:$1/health" 2>/dev/null +meter_health() { # port: the /health of a meter still up three seconds on, this launch listed as one of its users first + curl -s -m 2 "http://127.0.0.1:$1/health" > /dev/null 2>&1 \ + && { echo $$ >> "$(meter_users "$1")" || true; } 2>/dev/null \ + && sleep 3 && curl -s -m 2 "http://127.0.0.1:$1/health" 2>/dev/null } LEDGER=$HERE/runs/wire_ledger-$WM_PORT.json if health=$(meter_health $WM_PORT); then @@ -99,10 +114,11 @@ if health=$(meter_health $WM_PORT); then *) echo "REFUSING: port $WM_PORT has a wire meter facing another or an unknown upstream ($health); choose another WM_PORT"; exit 1;; esac else - env PORT=$WM_PORT UPSTREAM=$EMBED_ORIGIN ${KEY_ENV:+UPSTREAM_KEY_ENV=$KEY_ENV} LEDGER=$LEDGER \ + env PORT=$WM_PORT UPSTREAM=$EMBED_ORIGIN ${KEY_ENV:+UPSTREAM_KEY_ENV=$KEY_ENV} LEDGER=$LEDGER LEDGER_RESUME=1 \ setsid nohup "$PY" -m hermit.runtime.wire_meter >> "runs/wire_meter-$WM_PORT.log" 2>&1 < /dev/null & - METERS+=("$!"); stop_meters_after $$ # $$: this launcher, then the hermit command it execs - echo "wire meter pid $! port $WM_PORT -> $EMBED_ORIGIN (ledger $LEDGER)"; sleep 2 + meter_started $WM_PORT $! + echo "wire meter pid $! port $WM_PORT -> $EMBED_ORIGIN (ledger $LEDGER)" + stop_meters_after $$; sleep 2 # $$: this launcher, then the hermit command it execs curl -s -m 2 "http://127.0.0.1:$WM_PORT/health" > /dev/null \ || { echo "REFUSING: the wire meter did not come up (runs/wire_meter-$WM_PORT.log)"; exit 1; } fi diff --git a/examples/servelearnbench/mwm/launch.sh b/examples/servelearnbench/mwm/launch.sh index 540888ed..5d48f497 100755 --- a/examples/servelearnbench/mwm/launch.sh +++ b/examples/servelearnbench/mwm/launch.sh @@ -81,29 +81,45 @@ HOST_KEY=""; [ -n "$HOST_API_KEY_ENV" ] && HOST_KEY=${!HOST_API_KEY_ENV:-} export SLB_MODEL=$HOST_MODEL_ID SLB_API_KEY=${HOST_KEY:-local} export SLB_REASONING_EFFORT=$EFFORT SLB_METHOD=$METHOD SLB_WORKDIR=$WD -# The meters this launch starts stop within a second or two of the processes it names ending (#82). A meter found -# running belongs to the launch that started it: a launch reuses one only if it is still up on a second look. -METERS=() -stop_meters_after() { # pid... +# The meters this launch starts stop within a second or two of the processes that use them ending (#82): its own and +# those of any launch that reuses one. Each launch on a port lists its processes in the port's users file, emptied +# when the meter stops; a launch reuses a running meter only if it is still up on a second look (one being stopped is +# gone by then). +METERS=() OWNED=() PORTS=() +meter_users() { echo "/tmp/hermit-wire-meter-$(id -u)-$1.users"; } # port: /tmp, which every launch on it shares +meter_started() { # port pid: a meter this launch started, this launch one of its users + { echo $$ >> "$(meter_users "$1")" || true; } 2>/dev/null + METERS+=("$2"); OWNED+=("$1") +} +stop_meters_after() { # pid...: what this launch leaves running; the meters it started stop after every user + local port files=() + [ $# -eq 0 ] || for port in ${PORTS[@]+"${PORTS[@]}"}; do + printf '%s\n' "$@" >> "$(meter_users "$port")" 2>/dev/null || true; done [ ${#METERS[@]} -gt 0 ] || return 0 - setsid nohup bash -c 'for p in $1; do while kill -0 "$p" 2>/dev/null; do sleep 1; done; done - for m in $2; do kill -- "-$m" 2>/dev/null || kill "$m" 2>/dev/null || true; done' _ "$*" "${METERS[*]}" \ + for port in "${OWNED[@]}"; do files+=("$(meter_users "$port")"); done + setsid nohup bash -c 'while :; do up= + for p in $(cat $1 2>/dev/null); do kill -0 "$p" 2>/dev/null && { up=1; break; }; done + [ -n "$up" ] || break; sleep 1; done + for f in $1; do : > "$f"; done 2>/dev/null # a launch listed after this look finds the meter gone and starts one + for m in $2; do kill -- "-$m" 2>/dev/null || kill "$m" 2>/dev/null || true; done' _ "${files[*]}" "${METERS[*]}" \ > /dev/null 2>&1 < /dev/null & } -meter_health() { # port: the /health of a meter still up three seconds on (one being stopped is gone by then) - curl -s -m 2 "http://127.0.0.1:$1/health" > /dev/null 2>&1 && sleep 3 \ - && curl -s -m 2 "http://127.0.0.1:$1/health" 2>/dev/null +meter_health() { # port: the /health of a meter still up three seconds on, this launch listed as one of its users first + curl -s -m 2 "http://127.0.0.1:$1/health" > /dev/null 2>&1 \ + && { echo $$ >> "$(meter_users "$1")" || true; } 2>/dev/null \ + && sleep 3 && curl -s -m 2 "http://127.0.0.1:$1/health" 2>/dev/null } trap 'stop_meters_after ${TRAIN_PID:-} ${TEST_PID:-}' EXIT # train mode: after the training and the tests after it start_meter() { # port upstream ledger log [extra env...]: a meter already on the port must face the same upstream local port=$1 upstream=$2 ledger=$3 log=$4 health; shift 4 + PORTS+=("$port") # started here or reused: this launch's processes use it if health=$(meter_health $port); then case "$health" in *"\"upstream\": \"$upstream\""*) echo "wire meter already on port $port -> $upstream";; *) echo "REFUSING: port $port has a wire meter facing another or an unknown upstream ($health); choose another WM_PORT"; exit 1;; esac else - env "$@" PORT=$port LEDGER="$ledger" UPSTREAM="$upstream" \ + env "$@" PORT=$port LEDGER="$ledger" LEDGER_RESUME=1 UPSTREAM="$upstream" \ setsid nohup "$PY" "$METER" > "$log" 2>&1 < /dev/null & - METERS+=("$!") + meter_started $port $! echo "wire meter pid $! port $port -> $upstream"; sleep 2 fi } diff --git a/hermit/runtime/wire_meter.py b/hermit/runtime/wire_meter.py index 64860ac6..007c1d5b 100644 --- a/hermit/runtime/wire_meter.py +++ b/hermit/runtime/wire_meter.py @@ -36,6 +36,9 @@ the relay adds a metering hop, not an egress path. - the ledger is persisted to LEDGER as JSON every FLUSH_S seconds and on SIGTERM; GET /ledger serves it live, GET /health the error counters. + LEDGER_RESUME=1: a meter started on a LEDGER an earlier one left carries + its totals on (the example launchers: they stop their meters when their + runs end, and the next run on the port starts another on the same file). Dialects: OpenAI chat completions (prompt_tokens includes cached) and Anthropic messages (input_tokens excludes cache components — added back), @@ -43,7 +46,8 @@ Env: PORT (default 3129), BIND (default 127.0.0.1; 0.0.0.0 for a sidecar that other containers call), UPSTREAM (default https://api.fireworks.ai), -PROXY (e.g. http://mw-egress-meta:3128), LEDGER (json path), FLUSH_S. +PROXY (e.g. http://mw-egress-meta:3128), LEDGER (json path), LEDGER_RESUME, +FLUSH_S. UPSTREAM_KEY_ENV names the variable holding the upstream's key: a request that arrives without an Authorization header gets `Bearer ` (2026-10-25). @@ -84,6 +88,7 @@ UPSTREAM = os.environ.get("UPSTREAM", "https://api.fireworks.ai") PROXY = os.environ.get("PROXY", "") LEDGER_PATH = os.environ.get("LEDGER", "") +LEDGER_RESUME = os.environ.get("LEDGER_RESUME", "") not in ("", "0") FLUSH_S = float(os.environ.get("FLUSH_S", "15")) REWRITE_DEV = os.environ.get("REWRITE_DEVELOPER_ROLE", "") not in ("", "0") # PRIME_EFFORT (e.g. "high"): on the `prime` bucket, replace Prime's own @@ -408,6 +413,22 @@ def _flush(path: str = LEDGER_PATH): pass +def _resume(path: str = LEDGER_PATH): + """The totals an earlier meter left in LEDGER, added to this one's.""" + try: + with open(path) as f: + old = json.load(f).get("ledger") or {} + with _lock: + for bucket, totals in old.items(): + b = _ledger.setdefault( + bucket, {"input": 0, "cacheRead": 0, "output": 0, + "calls": 0}) + for k in b: + b[k] += int(totals.get(k, 0)) + except (OSError, ValueError, AttributeError, TypeError): + pass + + class Handler(BaseHTTPRequestHandler): protocol_version = "HTTP/1.1" @@ -652,6 +673,8 @@ def main(): sys.exit(f"wire meter: a relay that signs requests (UPSTREAM_KEY_ENV) listens on the loopback only; refusing " f"BIND={BIND}") srv = Server((BIND, PORT), Handler) + if LEDGER_PATH and LEDGER_RESUME: + _resume() def _bye(sig, frame): _flush() diff --git a/tests/core/test_backends.py b/tests/core/test_backends.py index 6876ae2d..ba5cae70 100644 --- a/tests/core/test_backends.py +++ b/tests/core/test_backends.py @@ -3,9 +3,15 @@ dialect to a fake server.""" from http.server import BaseHTTPRequestHandler, HTTPServer import json +import os from pathlib import Path +import signal +import socket +import subprocess import sys import threading +import time +import urllib.request import pytest import yaml @@ -403,6 +409,44 @@ def test_cli_dispatch_and_meter_buckets(capsys): assert split_bucket("/health") == ("unattributed", "/health") +def test_a_meter_carries_on_the_ledger_an_earlier_one_left(tmp_path): + """A launcher stops its meter when its run ends and the next run on the port starts another on the same ledger + (#82): with LEDGER_RESUME=1 (the launchers set it) the new meter keeps the earlier totals instead of writing an + empty ledger over them; without it a meter starts empty, as lean_swarm's rerun of an attempt expects.""" + ledger = tmp_path / "wire_ledger.json" + earlier = {"run_o0": {"input": 1000, "cacheRead": 200, "output": 50, "calls": 3}} + ledger.write_text(json.dumps({"ledger": earlier, "health": {}, "ts": 1})) + with socket.socket() as probe: + probe.bind(("127.0.0.1", 0)) + port = probe.getsockname()[1] + env = {**os.environ, "PYTHONPATH": str(Path(__file__).resolve().parents[2]), "UPSTREAM": "http://127.0.0.1:9", + "PORT": str(port), "LEDGER": str(ledger), "FLUSH_S": "3600"} + env.pop("UPSTREAM_KEY_ENV", None) + + def meter(**extra) -> dict: + """The ledger a meter serves once up, and the one it writes when stopped as a launcher stops it (SIGTERM).""" + proc = subprocess.Popen([sys.executable, "-m", "hermit.runtime.wire_meter"], env={**env, **extra}, + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + try: + for _ in range(50): + try: + with urllib.request.urlopen(f"http://127.0.0.1:{port}/ledger", timeout=2) as response: + live = json.loads(response.read())["ledger"] + break + except OSError: + time.sleep(0.2) + else: + raise AssertionError("the meter did not come up") + finally: + proc.send_signal(signal.SIGTERM) + proc.wait(timeout=10) + assert json.loads(ledger.read_text())["ledger"] == live + return live + + assert meter(LEDGER_RESUME="1") == earlier + assert meter() == {} + + def test_a_task_agent_on_a_cli_base_is_bound_and_checked_like_the_cli(): """A task's own agent on agent.base: codex with a model reference is bound as Codex's model (not a dictionary for -m), and the run's preflight checks that binding for the Responses API."""