From a3c0a4043b27a6a8c29dd375971f78dcffffa6d5 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 21 Sep 2026 13:42:18 +0700 Subject: [PATCH] fix(queen): busy is not dead - the supervisor stops killing a working server The first version allowed four misses of a ten-second /health probe. Measured 2026-09-21, it killed the server itself - at fourteen lanes and at twenty, each time about two minutes after the lanes filled: [liveness] /health did not answer (1 of 4) ... [liveness] /health did not answer (4 of 4) [liveness] the server stopped answering; ending it so the platform restarts the container [liveness] the server exited (137) With every lane working the event loop is saturated and /health takes longer than ten seconds. The server was alive and making progress, and ending it destroyed every bee's turn in flight - the protection doing more damage than what it protects against. A probe now waits LIVENESS_TIMEOUT (30 s) and twelve misses in a row are needed: about twelve minutes of silence. Both halves proven before shipping: a server answering in 3 s against a 5 s probe still running after 30 s a server frozen with SIGSTOP ended, exit 143 Co-Authored-By: Claude Opus 5 --- trios/agent-server/docker-entrypoint.sh | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/trios/agent-server/docker-entrypoint.sh b/trios/agent-server/docker-entrypoint.sh index a16807ada..22e1e9822 100755 --- a/trios/agent-server/docker-entrypoint.sh +++ b/trios/agent-server/docker-entrypoint.sh @@ -41,6 +41,16 @@ set -e # declares it; curl is not installed here. # # One healthy answer resets the count: a slow minute is not a dead server. +# +# BUSY IS NOT DEAD. The first version allowed four misses of a ten-second probe, +# and measured 2026-09-21 it killed the server itself - twice, at fourteen lanes +# and at twenty, each time about two minutes after the lanes filled. With every +# lane working, the event loop is saturated and /health takes longer than ten +# seconds; the server was alive and making progress, and ending it destroyed +# every bee's turn in flight. So a probe now waits LIVENESS_TIMEOUT (30 s) and +# twelve misses in a row are needed: roughly twelve minutes of silence. A hang +# of the kind this exists for - nine hours of `Application failed to respond` +# - is caught all the same; a busy stretch is not. run_supervised() { "$@" & server=$! @@ -48,11 +58,12 @@ run_supervised() { ( port="${PORT:-8080}" interval="${LIVENESS_INTERVAL:-30}" - fails_allowed="${LIVENESS_FAILS:-4}" + fails_allowed="${LIVENESS_FAILS:-12}" + probe_timeout="${LIVENESS_TIMEOUT:-30}" sleep "${LIVENESS_GRACE:-240}" fails=0 while kill -0 "$server" 2>/dev/null; do - if python3 -c "import urllib.request; urllib.request.urlopen('http://127.0.0.1:$port/health', timeout=10)" >/dev/null 2>&1; then + if python3 -c "import urllib.request; urllib.request.urlopen('http://127.0.0.1:$port/health', timeout=$probe_timeout)" >/dev/null 2>&1; then fails=0 else fails=$((fails + 1))