fix(chain): wait for the nightly build race; guard manual quarantine

.github/workflows/rustfs-validation-heal-sweep.yml

@@ -77,7 +77,10 @@ jobs:
     concurrency:
       group: rustfs-shared-functional-tests-v2
       cancel-in-progress: false
-    timeout-minutes: 600
+    # Observed full-catalog pace is 6-14 h (disk/versions/shards cases run
+    # 5-30 min each); the runner service is occasionally bounced by host
+    # maintenance, so budget generously for a Beijing-night window.
+    timeout-minutes: 990
     steps:
       - name: Checkout chain tooling
         uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
@@ -293,11 +296,53 @@ jobs:
         timeout-minutes: 5
         env:
           FUNCTIONAL_ENVIRONMENT_RECEIPT: ${{ runner.temp }}/environment-validation-heal.json
-        run: python3 scripts/functional_environment.py enter --suite heal
+          GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
+        run: |
+          set -uo pipefail
+          if ! python3 scripts/functional_environment.py enter --suite heal; then
+            # A cancelled run (e.g. the runner service bounced mid-case)
+            # leaves its quarantine markers behind and blocks every later
+            # admission. When the marker belongs to a run that is no longer
+            # in progress, recover it and retry once.
+            echo "admission failed; checking for stale quarantine"
+            STALE=0
+            for node in ${RUSTFS_NODES}; do
+              MARKER="$(ssh -n -o BatchMode=yes -o ConnectTimeout=10 "${RUSTFS_SSH_USER}@${node}" \
+                'cat ~/.rustfs-functional-environment/active.json 2>/dev/null' || true)"
+              [ -n "${MARKER}" ] || continue
+              # Only functional-fleet markers owned by a CI run are
+              # recoverable. Manual markers (e.g. an authorized performance
+              # baseline) have no run_id and must never be touched.
+              KIND="$(printf '%s' "${MARKER}" | jq -r '.resource_lock // empty')"
+              OWNER="$(printf '%s' "${MARKER}" | jq -r '.run_id // empty')"
+              if [ "${KIND}" != "rustfs-shared-functional-tests-v2" ] || ! [[ "${OWNER}" =~ ^[1-9][0-9]*$ ]]; then
+                echo "node ${node}: manual/foreign quarantine present; not touching"
+                STALE=1
+                continue
+              fi
+              STATE="$(gh api "repos/rustfs/auto-testing/actions/runs/${OWNER}" --jq '.status' 2>/dev/null || echo unknown)"
+              echo "node ${node}: marker owner run ${OWNER} is ${STATE}"
+              if [ "${STATE}" != "in_progress" ] && [ "${STATE}" != "queued" ]; then
+                ssh -n -o BatchMode=yes "${RUSTFS_SSH_USER}@${node}" '
+                  SUDO="sudo -n"
+                  ${SUDO} systemctl stop rustfs 2>/dev/null || true
+                  if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then ${SUDO} dpkg -P rustfs >/dev/null 2>&1; fi
+                  for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done
+                  ${SUDO} rm -f /etc/default/rustfs
+                  rm -f ~/.rustfs-functional-environment/active.json
+                ' && echo "node ${node}: recovered" || { echo "node ${node}: recovery failed"; STALE=1; }
+              else
+                echo "node ${node}: owner still active; not touching"
+                STALE=1
+              fi
+            done
+            [ "${STALE}" -eq 0 ] || { echo "live quarantine present; not recovering"; exit 1; }
+            python3 scripts/functional_environment.py enter --suite heal
+          fi
 
       - name: Run heal validation cases
         id: run_cases
-        timeout-minutes: 540
+        timeout-minutes: 900
         env:
           AUTO_TESTING_SHA: ${{ steps.candidate.outputs.testing_sha }}
           HEAL_ROOT: ${{ runner.temp }}/validation-heal

scripts/resolve_functional_candidate.py

@@ -125,7 +125,23 @@ def resolve(run_id, attempt, *, allow_validation=False):
     require(path == ".github/workflows/nightly-gnu.yml" and run.get("head_branch") == "main" or validation,
             "candidate must come from nightly-gnu on main or an explicitly allowed validation build")
     require((run.get("head_repository") or {}).get("full_name") == REPOSITORY, "candidate came from another repository")
-    require(run.get("event") in ("schedule", "workflow_dispatch") and run.get("status") == "completed" and run.get("conclusion") == "success", "candidate attempt has not completed successfully")
+    require(run.get("event") in ("schedule", "workflow_dispatch"), "candidate build has an unexpected trigger")
+    # The nightly build dispatches this chain from a late step, seconds
+    # before the run itself reaches completed/success — the chain can win
+    # that race (run 41872426981 lost it by 12 s). Poll briefly instead of
+    # failing instantly.
+    import time
+    for waited in range(0, 601, 15):
+        if run.get("status") == "completed":
+            break
+        if waited:
+            time.sleep(15)
+        try:
+            run = api(f"{endpoint}/attempts/{attempt}")
+        except Exception:
+            run = {}
+        print(f"waiting for build attempt to complete (waited {waited}s, status={run.get('status')})", flush=True)
+    require(run.get("status") == "completed" and run.get("conclusion") == "success", "candidate attempt has not completed successfully")
     if validation:
         require(run.get("event") == "workflow_dispatch", "validation candidate must be manually dispatched")
     name = f"{'validation' if validation else 'nightly'}-candidate-{run_id}-{attempt}"