From 86fc31f2313c0bfd292a9cdfaad54cdcd2d34546 Mon Sep 17 00:00:00 2001
From: Kevin <kev.brimm@gmail.com>
Date: Wed, 26 Aug 2026 14:17:03 -0700
Subject: [PATCH] fix: verify GitHub runner registration before reporting ready
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The readiness check previously validated only GCE instance metadata
(gh_ready=1), which is set immediately after ./svc.sh start — before
the runner process has actually connected to GitHub's API. When a
runner silently fails to register, downstream jobs queue for 24 hours
waiting for a runner that never appears.

This adds a second verification stage that polls the GitHub Actions
runners API to confirm runners with the expected label are registered.
If they don't appear within 3 minutes, all VMs are cleaned up and the
step fails fast.

Also fixes the loop counter bug where `i` was never reset between
instances, causing later VMs to get fewer readiness polling attempts.

Also moves the GITHUB_OUTPUT label write to after all verification
passes (defense-in-depth).

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
---
 action.sh | 40 ++++++++++++++++++++++++++++++++++++++--
 1 file changed, 38 insertions(+), 2 deletions(-)

diff --git a/action.sh b/action.sh
index 83fd077..398b927 100755
--- a/action.sh
+++ b/action.sh
@@ -311,8 +311,6 @@ function start_vm {
     ${max_run_duration_flag} \
     --metadata=startup-script="$startup_script"
  
-  echo "label=${VM_ID}" >> $GITHUB_OUTPUT
-
   safety_off
   launched_instances=$(gcloud compute instances list --filter "labels.vm_id=${VM_ID}" --format='get(name)')
   if [ -z "$launched_instances" ]; then
@@ -321,6 +319,7 @@ function start_vm {
   fi
 
   for instance in $launched_instances; do
+    i=0
     while (( i++ < 60 )); do
       GH_READY=$(gcloud compute instances describe ${instance} --zone=${machine_zone} --format='json(labels)' | jq -r .labels.gh_ready)
       if [[ $GH_READY == 1 ]]; then
@@ -347,6 +346,43 @@ function start_vm {
       exit 1
     fi
   done
+
+  # Verify runners are actually registered with GitHub's API.
+  # The GCE metadata check above only confirms the startup script completed,
+  # but ./svc.sh start launches the runner as a background service — it may
+  # not have connected to GitHub yet. Without this check, downstream jobs
+  # can queue for 24 hours waiting for runners that never registered.
+  echo "Verifying runners registered with GitHub API..."
+  EXPECTED_RUNNERS=${num_instances}
+  REGISTERED=0
+  VERIFY_ATTEMPTS=0
+  MAX_VERIFY_ATTEMPTS=36  # 36 * 5s = 3 minutes
+
+  while (( VERIFY_ATTEMPTS++ < MAX_VERIFY_ATTEMPTS )); do
+    REGISTERED=$(curl -s -H "authorization: Bearer ${token}" \
+      "https://api.github.com/repos/${GITHUB_REPOSITORY}/actions/runners" | \
+      jq "[.runners[] | select(.labels[].name == \"${VM_ID}\")] | length")
+
+    if (( REGISTERED >= EXPECTED_RUNNERS )); then
+      echo "✅ All ${EXPECTED_RUNNERS} runners registered with GitHub"
+      break
+    fi
+
+    echo "⏳ ${REGISTERED}/${EXPECTED_RUNNERS} runners registered with GitHub, waiting 5 secs ..."
+    sleep 5
+  done
+
+  if (( REGISTERED < EXPECTED_RUNNERS )); then
+    echo "❌ Only ${REGISTERED}/${EXPECTED_RUNNERS} runners registered with GitHub after 3 minutes"
+    echo "Cleaning up VMs..."
+    for instance in $launched_instances; do
+      echo "Deleting ${instance} ..."
+      gcloud --quiet compute instances delete ${instance} --zone=${machine_zone}
+    done
+    exit 1
+  fi
+
+  echo "label=${VM_ID}" >> $GITHUB_OUTPUT
 }
 
 safety_on