From 68c5fe2f51342f253a628a56c6adc5f9b4531d6b Mon Sep 17 00:00:00 2001
From: Joseph Mearman <joseph@mearman.co.uk>
Date: Wed, 21 Oct 2026 07:47:45 +0100
Subject: [PATCH] test(mesh): check a server stepped down to etcd-only gives
 the Lease back

The stepped-down server has no API server, so the release goes through
another server's. The scenario now waits for the Lease to be free
rather than expired, and fails if the service logged that it could not
release it. The harness inventory also turns the runtime autostart
off, so a run against a Colima profile installs no login item.
---
 tests/mesh/run.sh | 17 ++++++++++++++++-
 1 file changed, 16 insertions(+), 1 deletion(-)

diff --git a/tests/mesh/run.sh b/tests/mesh/run.sh
index 52124b4..22cd2ab 100755
--- a/tests/mesh/run.sh
+++ b/tests/mesh/run.sh
@@ -173,6 +173,8 @@ EOF
     echo "        github_runner_cluster_etcd_heartbeat_interval: ${etcd_heartbeat_ms}"
     echo "        github_runner_cluster_etcd_election_timeout: ${etcd_election_ms}"
     echo "        github_runner_cluster_regrade: ${regrade}"
+    # The test cluster is throwaway: the role must not install a login item that starts the host's Colima profile.
+    echo "        github_runner_cluster_runtime_autostart: false"
     if [ "$pin_context" = 1 ]; then
       echo "        github_runner_cluster_docker_context: ${grtest_pinned_context}"
     fi
@@ -575,6 +577,15 @@ peer_refusal() {
   ! regrade_on 3 fleet-regrade down >"${work}/peer.log" 2>&1 && grep -F "etcd-voting nodes not ready: $(node_name 2)" "${work}/peer.log" >/dev/null
 }
 at_level() { [ "$(level_of "$1")" = "$2" ]; }
+# Whether the regrade Lease exists and nobody holds it, as a release leaves it, read through node 1's API. A Lease left to expire keeps its holder, so this tells a release from an expiry at any time.
+lease_free() { [ "$(kubectl_on 1 get lease fleet-regrade --namespace kube-system -o 'jsonpath={.spec.holderIdentity}' 2>/dev/null; echo "/$?")" = /0 ]; }
+# Whether the regrade service of node $1 has logged, in the file $2 inside it, that it reached the level $3.
+logged_level() { regrade_on "$1" cat "$2" | grep -Fx "now at $3" >/dev/null; }
+# A step down to etcd-only gives the Lease back from another server's API, since the stepped-down server has none: once the step has finished the Lease is free, and the service never logged that it could not release it.
+assert_released() {
+  wait_for "the Lease to be released after $(node_name "$1")'s step down to etcd-only" 120 lease_free
+  if regrade_on "$1" cat "$2" | grep -F 'lease could not be released' >/dev/null; then regrade_on "$1" cat "$2"; fail "$(node_name "$1") could not release the Lease"; fi
+}
 node_is() { [ "$(node_status "$1" "$2")" = "$3" ]; }
 
 # The numbers of the nodes whose regrade service reports a level below full. A read that gives no level, or unknown, is not a step: a test node's k3s container can restart by itself under the strain, and its status then reads unknown or fails for a moment, which says nothing about whether the server stepped.
@@ -636,6 +647,8 @@ EOF
     [ "$SECONDS" -lt "$deadline" ] || { regrade_on 2 cat /tmp/watch.log || true; fail "$(node_name 2) did not reach etcd-only"; }
     sleep 2
   done
+  wait_for "the watcher on $(node_name 2) to finish its step to etcd-only" 300 logged_level 2 /tmp/watch.log etcd-only
+  assert_released 2 /tmp/watch.log
   regrade_on 2 cat /tmp/watch.log
   regrade_on 2 cat /tmp/watch.log | grep -F 'step down full -> control-plane' >/dev/null || fail "the watcher did not step down through control-plane first"
 
@@ -699,8 +712,10 @@ EOF
   wait_for "$(node_name 3) to read control-plane after a step down by hand" 180 at_level 3 control-plane
   wait_for "$(node_name 3) to be cordoned" 60 node_is 1 3 Ready,SchedulingDisabled
   [ "$(level_label 3)" = control-plane ] || fail "the Node does not carry the level label at control-plane"
-  regrade_on 3 fleet-regrade down
+  regrade_on 3 sh -c 'fleet-regrade down >/tmp/down.log 2>&1' || { regrade_on 3 cat /tmp/down.log; fail "the second step down by hand on $(node_name 3) failed"; }
+  regrade_on 3 cat /tmp/down.log
   wait_for "$(node_name 3) to read etcd-only after a second step down by hand" 180 at_level 3 etcd-only
+  assert_released 3 /tmp/down.log
   # The level is a file on the data volume that the entrypoint reads at every start, so neither a restart by hand nor a recreate, which is what Compose does when the role changes the container's configuration, brings the server back at full.
   docker restart "$(container 3)" >/dev/null
   wait_for "$(node_name 3) to be at etcd-only after a restart by hand" 300 at_level 3 etcd-only