Stop the wire meters a launcher starts once its runs end

CHANGELOG.md

@@ -202,6 +202,16 @@ Changes are grouped by software version. Entries under Unreleased describe work
 - The AHE Adapter Evolver took a `change_manifest.json` an earlier step left for the current step's: a manifest
   naming the next iteration was refused at its own step, then archived as the next step's when that step's session
   wrote none. The file (or a directory a session made in its place) is now removed before each Evolve session.
+- Nine example launchers left their wire meters running after the run (#82), so a later run on the same port
+  reused one and wrote its records into the earlier run's ledger. The meters a launch starts now stop within a second
+  or two of the runs that use them ending: its own trainings (and, for ServeLearnBench, the tests after them) and
+  those of any launch that reuses one, which each list their processes in the port's users file
+  (`/tmp/hermit-wire-meter-<uid>-<port>.users`); the harness2 restart loop is stopped with its meter. A
+  launch reuses a running meter only if it is still up three seconds on, so it never takes one that is being
+  stopped. A meter started on a ledger an earlier one left now carries its totals on (`LEDGER_RESUME=1` in
+  `hermit/runtime/wire_meter.py`, which these launchers set): the next run's meter on the port (`launch.sh run &&
+  launch.sh run eval`, a ServeLearnBench `test` run again, a CL-Bench tag refused because it exists) wrote an empty
+  ledger over the earlier run's bill.
 - On macOS a CLI session stopped at its timeout or on cancel could leave processes behind or fail: the processes
   below it were found through `/proc`, which macOS lacks, so one in a session of its own (as a sandbox starts its
   program) was never stopped; and signalling a process group left with only exited, unreaped processes raises

examples/clbench_database/mwm/launch.sh

@@ -78,14 +78,45 @@ fi
 command -v codex >/dev/null || { echo "REFUSING: codex CLI not on PATH"; exit 1; }
 [ -f "$TASKDIR/tasks.jsonl" ] || { echo "REFUSING: run $TASKDIR/prepare.py first"; exit 1; }
 
+# The meters this launch starts stop within a second or two of the processes that use them ending (#82): its own and
+# those of any launch that reuses one. Each launch on a port lists its processes in the port's users file, emptied
+# when the meter stops; a launch reuses a running meter only if it is still up on a second look (one being stopped is
+# gone by then).
+METERS=() OWNED=() PORTS=()
+meter_users() { echo "/tmp/hermit-wire-meter-$(id -u)-$1.users"; }   # port: /tmp, which every launch on it shares
+meter_started() {  # port pid: a meter this launch started, this launch one of its users
+  { echo $$ >> "$(meter_users "$1")" || true; } 2>/dev/null
+  METERS+=("$2"); OWNED+=("$1")
+}
+stop_meters_after() {  # pid...: what this launch leaves running; the meters it started stop after every user
+  local port files=()
+  [ $# -eq 0 ] || for port in ${PORTS[@]+"${PORTS[@]}"}; do
+    printf '%s\n' "$@" >> "$(meter_users "$port")" 2>/dev/null || true; done
+  [ ${#METERS[@]} -gt 0 ] || return 0
+  for port in "${OWNED[@]}"; do files+=("$(meter_users "$port")"); done
+  setsid nohup bash -c 'while :; do up=
+      for p in $(cat $1 2>/dev/null); do kill -0 "$p" 2>/dev/null && { up=1; break; }; done
+      [ -n "$up" ] || break; sleep 1; done
+    for f in $1; do : > "$f"; done 2>/dev/null    # a launch listed after this look finds the meter gone and starts one
+    for m in $2; do kill -- "-$m" 2>/dev/null || kill "$m" 2>/dev/null || true; done' _ "${files[*]}" "${METERS[*]}" \
+    > /dev/null 2>&1 < /dev/null &
+}
+meter_health() {  # port: the /health of a meter still up three seconds on, this launch listed as one of its users first
+  curl -s -m 2 "http://127.0.0.1:$1/health" > /dev/null 2>&1 \
+    && { echo $$ >> "$(meter_users "$1")" || true; } 2>/dev/null \
+    && sleep 3 && curl -s -m 2 "http://127.0.0.1:$1/health" 2>/dev/null
+}
+trap 'stop_meters_after ${PIDS[@]+"${PIDS[@]}"}' EXIT   # after the trainings started below, if any
 start_meter() {  # port upstream ledger log [extra env...]: a meter already on the port must face the same upstream
   local port=$1 upstream=$2 ledger=$3 log=$4 health; shift 4
-  if health=$(curl -s -m 2 "http://127.0.0.1:$port/health" 2>/dev/null); then
+  PORTS+=("$port")                                  # started here or reused: this launch's processes use it
+  if health=$(meter_health $port); then
     case "$health" in *"\"upstream\": \"$upstream\""*) echo "wire meter already on port $port -> $upstream";;
       *) echo "REFUSING: port $port has a wire meter facing another or an unknown upstream ($health); choose another WM_PORT"; exit 1;; esac
   else
-    env "$@" PORT=$port LEDGER="$ledger" UPSTREAM="$upstream" \
+    env "$@" PORT=$port LEDGER="$ledger" LEDGER_RESUME=1 UPSTREAM="$upstream" \
       setsid nohup "$PY" "$METER" > "$log" 2>&1 < /dev/null &
+    meter_started $port $!
     echo "wire meter pid $! port $port -> $upstream"; sleep 2
   fi
 }

examples/clbench_database/prompt_only/launch.sh

@@ -82,14 +82,45 @@ fi
 command -v codex >/dev/null || { echo "REFUSING: codex CLI not on PATH"; exit 1; }
 [ -f "$TASKDIR/tasks.jsonl" ] || { echo "REFUSING: run $TASKDIR/prepare.py first"; exit 1; }
 
+# The meters this launch starts stop within a second or two of the processes that use them ending (#82): its own and
+# those of any launch that reuses one. Each launch on a port lists its processes in the port's users file, emptied
+# when the meter stops; a launch reuses a running meter only if it is still up on a second look (one being stopped is
+# gone by then).
+METERS=() OWNED=() PORTS=()
+meter_users() { echo "/tmp/hermit-wire-meter-$(id -u)-$1.users"; }   # port: /tmp, which every launch on it shares
+meter_started() {  # port pid: a meter this launch started, this launch one of its users
+  { echo $$ >> "$(meter_users "$1")" || true; } 2>/dev/null
+  METERS+=("$2"); OWNED+=("$1")
+}
+stop_meters_after() {  # pid...: what this launch leaves running; the meters it started stop after every user
+  local port files=()
+  [ $# -eq 0 ] || for port in ${PORTS[@]+"${PORTS[@]}"}; do
+    printf '%s\n' "$@" >> "$(meter_users "$port")" 2>/dev/null || true; done
+  [ ${#METERS[@]} -gt 0 ] || return 0
+  for port in "${OWNED[@]}"; do files+=("$(meter_users "$port")"); done
+  setsid nohup bash -c 'while :; do up=
+      for p in $(cat $1 2>/dev/null); do kill -0 "$p" 2>/dev/null && { up=1; break; }; done
+      [ -n "$up" ] || break; sleep 1; done
+    for f in $1; do : > "$f"; done 2>/dev/null    # a launch listed after this look finds the meter gone and starts one
+    for m in $2; do kill -- "-$m" 2>/dev/null || kill "$m" 2>/dev/null || true; done' _ "${files[*]}" "${METERS[*]}" \
+    > /dev/null 2>&1 < /dev/null &
+}
+meter_health() {  # port: the /health of a meter still up three seconds on, this launch listed as one of its users first
+  curl -s -m 2 "http://127.0.0.1:$1/health" > /dev/null 2>&1 \
+    && { echo $$ >> "$(meter_users "$1")" || true; } 2>/dev/null \
+    && sleep 3 && curl -s -m 2 "http://127.0.0.1:$1/health" 2>/dev/null
+}
+trap 'stop_meters_after ${PIDS[@]+"${PIDS[@]}"}' EXIT   # after the trainings started below, if any
 start_meter() {  # port upstream ledger log [extra env...]: a meter already on the port must face the same upstream
   local port=$1 upstream=$2 ledger=$3 log=$4 health; shift 4
-  if health=$(curl -s -m 2 "http://127.0.0.1:$port/health" 2>/dev/null); then
+  PORTS+=("$port")                                  # started here or reused: this launch's processes use it
+  if health=$(meter_health $port); then
     case "$health" in *"\"upstream\": \"$upstream\""*) echo "wire meter already on port $port -> $upstream";;
       *) echo "REFUSING: port $port has a wire meter facing another or an unknown upstream ($health); choose another WM_PORT"; exit 1;; esac
   else
-    env "$@" PORT=$port LEDGER="$ledger" UPSTREAM="$upstream" \
+    env "$@" PORT=$port LEDGER="$ledger" LEDGER_RESUME=1 UPSTREAM="$upstream" \
       setsid nohup "$PY" "$METER" > "$log" 2>&1 < /dev/null &
+    meter_started $port $!
     echo "wire meter pid $! port $port -> $upstream"; sleep 2
   fi
 }

examples/clbench_database/reharness/launch.sh

@@ -90,14 +90,45 @@ fi
 
 [ -f "$TASKDIR/tasks.jsonl" ] || { echo "REFUSING: run $TASKDIR/prepare.py first"; exit 1; }
 
+# The meters this launch starts stop within a second or two of the processes that use them ending (#82): its own and
+# those of any launch that reuses one. Each launch on a port lists its processes in the port's users file, emptied
+# when the meter stops; a launch reuses a running meter only if it is still up on a second look (one being stopped is
+# gone by then).
+METERS=() OWNED=() PORTS=()
+meter_users() { echo "/tmp/hermit-wire-meter-$(id -u)-$1.users"; }   # port: /tmp, which every launch on it shares
+meter_started() {  # port pid: a meter this launch started, this launch one of its users
+  { echo $$ >> "$(meter_users "$1")" || true; } 2>/dev/null
+  METERS+=("$2"); OWNED+=("$1")
+}
+stop_meters_after() {  # pid...: what this launch leaves running; the meters it started stop after every user
+  local port files=()
+  [ $# -eq 0 ] || for port in ${PORTS[@]+"${PORTS[@]}"}; do
+    printf '%s\n' "$@" >> "$(meter_users "$port")" 2>/dev/null || true; done
+  [ ${#METERS[@]} -gt 0 ] || return 0
+  for port in "${OWNED[@]}"; do files+=("$(meter_users "$port")"); done
+  setsid nohup bash -c 'while :; do up=
+      for p in $(cat $1 2>/dev/null); do kill -0 "$p" 2>/dev/null && { up=1; break; }; done
+      [ -n "$up" ] || break; sleep 1; done
+    for f in $1; do : > "$f"; done 2>/dev/null    # a launch listed after this look finds the meter gone and starts one
+    for m in $2; do kill -- "-$m" 2>/dev/null || kill "$m" 2>/dev/null || true; done' _ "${files[*]}" "${METERS[*]}" \
+    > /dev/null 2>&1 < /dev/null &
+}
+meter_health() {  # port: the /health of a meter still up three seconds on, this launch listed as one of its users first
+  curl -s -m 2 "http://127.0.0.1:$1/health" > /dev/null 2>&1 \
+    && { echo $$ >> "$(meter_users "$1")" || true; } 2>/dev/null \
+    && sleep 3 && curl -s -m 2 "http://127.0.0.1:$1/health" 2>/dev/null
+}
+trap 'stop_meters_after ${PIDS[@]+"${PIDS[@]}"}' EXIT   # after the trainings started below, if any
 start_meter() {  # port upstream ledger log [extra env...]: a meter already on the port must face the same upstream
   local port=$1 upstream=$2 ledger=$3 log=$4 health; shift 4
-  if health=$(curl -s -m 2 "http://127.0.0.1:$port/health" 2>/dev/null); then
+  PORTS+=("$port")                                  # started here or reused: this launch's processes use it
+  if health=$(meter_health $port); then
     case "$health" in *"\"upstream\": \"$upstream\""*) echo "wire meter already on port $port -> $upstream";;
       *) echo "REFUSING: port $port has a wire meter facing another or an unknown upstream ($health); choose another WM_PORT"; exit 1;; esac
   else
-    env "$@" PORT=$port LEDGER="$ledger" UPSTREAM="$upstream" \
+    env "$@" PORT=$port LEDGER="$ledger" LEDGER_RESUME=1 UPSTREAM="$upstream" \
       setsid nohup "$PY" "$METER" > "$log" 2>&1 < /dev/null &
+    meter_started $port $!
     echo "wire meter pid $! port $port -> $upstream"; sleep 2
     curl -s -m 2 "http://127.0.0.1:$port/health" > /dev/null \
       || { echo "REFUSING: the wire meter on port $port did not come up (see $log)"; exit 1; }
@@ -108,7 +139,7 @@ start_meter $WM_PORT "$HOST_ORIGIN" "$RUNS/$TAG/wire_ledger.json" "$RUNS/$TAG/wi
 start_meter $META_PORT "$EVOLVER_ORIGIN" "$RUNS/$TAG/meta_wire_ledger.json" "$RUNS/$TAG/meta_wire_meter.log"
 # The embedding relay signs the reader's keyless requests with the embedding backend's key (none for a keyless server);
 # a relay already on the port is refused unless its /health says it signs (`"signs": true`).
-if [ -n "$EMBED_API_KEY_ENV" ] && health=$(curl -s -m 2 "http://127.0.0.1:$EMBED_PORT/health" 2>/dev/null); then
+if [ -n "$EMBED_API_KEY_ENV" ] && health=$(meter_health $EMBED_PORT); then
   case "$health" in *'"signs": true'*) ;; *) echo "REFUSING: port $EMBED_PORT has a relay that does not sign requests (started without UPSTREAM_KEY_ENV); choose another WM_PORT"; exit 1;; esac
 fi
 start_meter $EMBED_PORT "$EMBED_ORIGIN" "$RUNS/$TAG/embed_wire_ledger.json" "$RUNS/$TAG/embed_wire_meter.log" \

examples/clbench_poker/mwm/launch.sh

@@ -73,14 +73,45 @@ export MWM_CLBENCH=$CLBENCH MWM_METAWORLD=$METAWORLD
 command -v codex >/dev/null || { echo "REFUSING: codex CLI not on PATH"; exit 1; }
 [ -f "$TASKDIR/tasks.jsonl" ] || { echo "REFUSING: run $TASKDIR/prepare.py first"; exit 1; }
 
+# The meters this launch starts stop within a second or two of the processes that use them ending (#82): its own and
+# those of any launch that reuses one. Each launch on a port lists its processes in the port's users file, emptied
+# when the meter stops; a launch reuses a running meter only if it is still up on a second look (one being stopped is
+# gone by then).
+METERS=() OWNED=() PORTS=()
+meter_users() { echo "/tmp/hermit-wire-meter-$(id -u)-$1.users"; }   # port: /tmp, which every launch on it shares
+meter_started() {  # port pid: a meter this launch started, this launch one of its users
+  { echo $$ >> "$(meter_users "$1")" || true; } 2>/dev/null
+  METERS+=("$2"); OWNED+=("$1")
+}
+stop_meters_after() {  # pid...: what this launch leaves running; the meters it started stop after every user
+  local port files=()
+  [ $# -eq 0 ] || for port in ${PORTS[@]+"${PORTS[@]}"}; do
+    printf '%s\n' "$@" >> "$(meter_users "$port")" 2>/dev/null || true; done
+  [ ${#METERS[@]} -gt 0 ] || return 0
+  for port in "${OWNED[@]}"; do files+=("$(meter_users "$port")"); done
+  setsid nohup bash -c 'while :; do up=
+      for p in $(cat $1 2>/dev/null); do kill -0 "$p" 2>/dev/null && { up=1; break; }; done
+      [ -n "$up" ] || break; sleep 1; done
+    for f in $1; do : > "$f"; done 2>/dev/null    # a launch listed after this look finds the meter gone and starts one
+    for m in $2; do kill -- "-$m" 2>/dev/null || kill "$m" 2>/dev/null || true; done' _ "${files[*]}" "${METERS[*]}" \
+    > /dev/null 2>&1 < /dev/null &
+}
+meter_health() {  # port: the /health of a meter still up three seconds on, this launch listed as one of its users first
+  curl -s -m 2 "http://127.0.0.1:$1/health" > /dev/null 2>&1 \
+    && { echo $$ >> "$(meter_users "$1")" || true; } 2>/dev/null \
+    && sleep 3 && curl -s -m 2 "http://127.0.0.1:$1/health" 2>/dev/null
+}
+trap 'stop_meters_after ${PIDS[@]+"${PIDS[@]}"}' EXIT   # after the trainings started below, if any
 start_meter() {  # port upstream ledger log [extra env...]: a meter already on the port must face the same upstream
   local port=$1 upstream=$2 ledger=$3 log=$4 health; shift 4
-  if health=$(curl -s -m 2 "http://127.0.0.1:$port/health" 2>/dev/null); then
+  PORTS+=("$port")                                  # started here or reused: this launch's processes use it
+  if health=$(meter_health $port); then
     case "$health" in *"\"upstream\": \"$upstream\""*) echo "wire meter already on port $port -> $upstream";;
       *) echo "REFUSING: port $port has a wire meter facing another or an unknown upstream ($health); choose another WM_PORT"; exit 1;; esac
   else
-    env "$@" PORT=$port LEDGER="$ledger" UPSTREAM="$upstream" \
+    env "$@" PORT=$port LEDGER="$ledger" LEDGER_RESUME=1 UPSTREAM="$upstream" \
       setsid nohup "$PY" "$METER" > "$log" 2>&1 < /dev/null &
+    meter_started $port $!
     echo "wire meter pid $! port $port -> $upstream"; sleep 2
   fi
 }

examples/clbench_poker/prompt_only/launch.sh

@@ -77,14 +77,45 @@ export MWM_CLBENCH=$CLBENCH MWM_METAWORLD=$METAWORLD
 command -v codex >/dev/null || { echo "REFUSING: codex CLI not on PATH"; exit 1; }
 [ -f "$TASKDIR/tasks.jsonl" ] || { echo "REFUSING: run $TASKDIR/prepare.py first"; exit 1; }
 
+# The meters this launch starts stop within a second or two of the processes that use them ending (#82): its own and
+# those of any launch that reuses one. Each launch on a port lists its processes in the port's users file, emptied
+# when the meter stops; a launch reuses a running meter only if it is still up on a second look (one being stopped is
+# gone by then).
+METERS=() OWNED=() PORTS=()
+meter_users() { echo "/tmp/hermit-wire-meter-$(id -u)-$1.users"; }   # port: /tmp, which every launch on it shares
+meter_started() {  # port pid: a meter this launch started, this launch one of its users
+  { echo $$ >> "$(meter_users "$1")" || true; } 2>/dev/null
+  METERS+=("$2"); OWNED+=("$1")
+}
+stop_meters_after() {  # pid...: what this launch leaves running; the meters it started stop after every user
+  local port files=()
+  [ $# -eq 0 ] || for port in ${PORTS[@]+"${PORTS[@]}"}; do
+    printf '%s\n' "$@" >> "$(meter_users "$port")" 2>/dev/null || true; done
+  [ ${#METERS[@]} -gt 0 ] || return 0
+  for port in "${OWNED[@]}"; do files+=("$(meter_users "$port")"); done
+  setsid nohup bash -c 'while :; do up=
+      for p in $(cat $1 2>/dev/null); do kill -0 "$p" 2>/dev/null && { up=1; break; }; done
+      [ -n "$up" ] || break; sleep 1; done
+    for f in $1; do : > "$f"; done 2>/dev/null    # a launch listed after this look finds the meter gone and starts one
+    for m in $2; do kill -- "-$m" 2>/dev/null || kill "$m" 2>/dev/null || true; done' _ "${files[*]}" "${METERS[*]}" \
+    > /dev/null 2>&1 < /dev/null &
+}
+meter_health() {  # port: the /health of a meter still up three seconds on, this launch listed as one of its users first
+  curl -s -m 2 "http://127.0.0.1:$1/health" > /dev/null 2>&1 \
+    && { echo $$ >> "$(meter_users "$1")" || true; } 2>/dev/null \
+    && sleep 3 && curl -s -m 2 "http://127.0.0.1:$1/health" 2>/dev/null
+}
+trap 'stop_meters_after ${PIDS[@]+"${PIDS[@]}"}' EXIT   # after the trainings started below, if any
 start_meter() {  # port upstream ledger log [extra env...]: a meter already on the port must face the same upstream
   local port=$1 upstream=$2 ledger=$3 log=$4 health; shift 4
-  if health=$(curl -s -m 2 "http://127.0.0.1:$port/health" 2>/dev/null); then
+  PORTS+=("$port")                                  # started here or reused: this launch's processes use it
+  if health=$(meter_health $port); then
     case "$health" in *"\"upstream\": \"$upstream\""*) echo "wire meter already on port $port -> $upstream";;
       *) echo "REFUSING: port $port has a wire meter facing another or an unknown upstream ($health); choose another WM_PORT"; exit 1;; esac
   else
-    env "$@" PORT=$port LEDGER="$ledger" UPSTREAM="$upstream" \
+    env "$@" PORT=$port LEDGER="$ledger" LEDGER_RESUME=1 UPSTREAM="$upstream" \
       setsid nohup "$PY" "$METER" > "$log" 2>&1 < /dev/null &
+    meter_started $port $!
     echo "wire meter pid $! port $port -> $upstream"; sleep 2
   fi
 }