From 207963b379482d5f738c6af5b6e2f296ff50290c Mon Sep 17 00:00:00 2001 From: sethforprivacy Date: Thu, 10 Sep 2026 07:12:09 -0700 Subject: [PATCH] Raise the managed mesh peer-outage grace from 4 s to 300 s PeerWatch.transport_error latched mesh_failure after 4 s without an authenticated peer response. Peer health travels over the management LAN, not the ring fabric, so any management-switch event longer than 4 s (a reboot, firmware update or STP reconvergence) SIGKILLed a healthy model on all four ranks while the RoCE ring was untouched; with Restart=no the lane stayed down until an operator intervened. A 101 s switch reboot on 2026-09-09 did exactly that for 5 h. Generation changes, negative readiness, authentication failures and local marker exits keep their no-grace semantics; only the transport-unavailable path waits 300 s. MANAGED_MESH.md and the unit test are updated. --- runtime/glm53-spark-mtp3-mesh/MANAGED_MESH.md | 4 ++-- runtime/glm53-spark-mtp3-mesh/managed_service.py | 2 +- runtime/glm53-spark-mtp3-mesh/test_managed_service.py | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/runtime/glm53-spark-mtp3-mesh/MANAGED_MESH.md b/runtime/glm53-spark-mtp3-mesh/MANAGED_MESH.md index f795f032..c2f258d1 100644 --- a/runtime/glm53-spark-mtp3-mesh/MANAGED_MESH.md +++ b/runtime/glm53-spark-mtp3-mesh/MANAGED_MESH.md @@ -361,7 +361,7 @@ The bounded `--run-seconds` mode is for isolated diagnostics only. | Check | Configured interval or bound | |---|---| | Child-process and peer checks | 1-second loop; peer HTTP timeout 2 seconds | -| Unavailable peer connection | 4-second grace after the first transport failure; degraded health blocks model startup | +| Unavailable peer connection | 300-second grace after the first transport failure (covers a management-switch reboot); degraded health blocks model startup | | Docker container status | One background query at a time, 3-second timeout; unknown status blocks model startup | | MAC/IP, Ethernet MTU, sysfs GID/netdev, routes, qdiscs, TC state | 5-second periodic check | | Full RDMA active-MTU probe | Startup and approximately every 60 seconds | @@ -369,7 +369,7 @@ The bounded `--run-seconds` mode is for isolated diagnostics only. | systemd watchdog | 15 seconds | A connection timeout or other peer transport error enters a degraded state. -Existing serving is retained during a four-second grace interval measured +Existing serving is retained during a 300-second grace interval measured from the first observed transport failure. Successful authenticated peer responses clear that interval. Degraded peer health blocks model startup. An authentication failure, explicit negative readiness, or changed process diff --git a/runtime/glm53-spark-mtp3-mesh/managed_service.py b/runtime/glm53-spark-mtp3-mesh/managed_service.py index 9f1076cb..e619a4c8 100644 --- a/runtime/glm53-spark-mtp3-mesh/managed_service.py +++ b/runtime/glm53-spark-mtp3-mesh/managed_service.py @@ -29,7 +29,7 @@ PROTOCOL = 'sparkring-managed-mesh/v1' POLL_SECONDS = 1.0 PEER_TIMEOUT = 2.0 -PEER_OUTAGE_GRACE = 4.0 +PEER_OUTAGE_GRACE = 300.0 NETWORK_POLL_SECONDS = 5.0 HEALTH_MAX_AGE = 10.0 diff --git a/runtime/glm53-spark-mtp3-mesh/test_managed_service.py b/runtime/glm53-spark-mtp3-mesh/test_managed_service.py index fcdbbac4..c4d47034 100644 --- a/runtime/glm53-spark-mtp3-mesh/test_managed_service.py +++ b/runtime/glm53-spark-mtp3-mesh/test_managed_service.py @@ -50,9 +50,9 @@ def test_sustained_peer_connection_loss_latches_failure(): watch = service.PeerWatch() watch.observe(rows()) watch.transport_error(10.0) - watch.transport_error(13.9) + watch.transport_error(309.9) with pytest.raises(RuntimeError, match='grace'): - watch.transport_error(14.0) + watch.transport_error(310.0) def test_generation_change_is_not_given_a_transport_grace():