fix(release): synchronize GARM runner wrapper

CHANGELOG.md

@@ -1,5 +1,9 @@
 # Changelog
 
+- Added GARM derivative `v0.2.1-nddev.81`: all embedded bootstrap downloads
+  now use the same three-total-attempt contract as provider `.78`, preventing
+  an incompatible wrapper rollout from blocking every new runner.
+
 - Added provider derivative `v0.1.5-nddev.78`: runner metadata bootstrap uses
   the same three-total-attempt download contract as every other reviewed
   network artifact path.

config/example-runner-1.yaml

@@ -6,7 +6,7 @@ platform:
 
 control_plane:
   manager: garm
-  manager_version: v0.2.1-nddev.80
+  manager_version: v0.2.1-nddev.81
   scheduling_mode: scale-set
   provider: incus
   provider_version: v0.1.5-nddev.78

config/example-runner-2.yaml

@@ -6,7 +6,7 @@ platform:
 
 control_plane:
   manager: garm
-  manager_version: v0.2.1-nddev.80
+  manager_version: v0.2.1-nddev.81
   scheduling_mode: scale-set
   provider: incus
   provider_version: v0.1.5-nddev.78

config/example-runner-3.yaml

@@ -6,7 +6,7 @@ platform:
 
 control_plane:
   manager: garm
-  manager_version: v0.2.1-nddev.80
+  manager_version: v0.2.1-nddev.81
   scheduling_mode: scale-set
   provider: incus
   provider_version: v0.1.5-nddev.78

config/example-runner-4.yaml

@@ -6,7 +6,7 @@ platform:
 
 control_plane:
   manager: garm
-  manager_version: v0.2.1-nddev.80
+  manager_version: v0.2.1-nddev.81
   scheduling_mode: scale-set
   provider: incus
   provider_version: v0.1.5-nddev.78

config/example-services.yaml

@@ -24,7 +24,7 @@ platform:
 
 control_plane:
   manager: garm
-  manager_version: v0.2.1-nddev.80
+  manager_version: v0.2.1-nddev.81
   scheduling_mode: scale-set
   provider: incus
   provider_version: v0.1.5-nddev.78

config/garm-derivative.yaml

@@ -1,6 +1,6 @@
 schema_version: 1
 artifact: garm
-derivative_version: v0.2.1-nddev.80
+derivative_version: v0.2.1-nddev.81
 upstream:
   repository: https://github.com/cloudbase/garm
   release: v0.2.1
@@ -85,6 +85,9 @@ patches:
   - path: third_party/garm/patches/0026-normalize-registration-token-errors.patch
     sha256: e3c09371d7a67072e1185afc09db823b5ccdc3ce6e524e5e504bc101cd4e06f0
     purpose: Normalize GitHub registration-token API failures through the shared typed error mapping so repository 404 responses remain detectable through wrapped scale-set operations.
+  - path: third_party/garm/patches/0027-bound-upstream-download-attempts.patch
+    sha256: 35f257c8016c7999c8780970f673c5b35e04b3b3301565cffea0bd242e168456
+    purpose: Bound every upstream Linux and Gitea bootstrap curl path to two retries after the initial attempt so GARM and the provider enforce the same three-total-attempt contract.
 overlays:
   - path: third_party/garm/overlay/workers/scaleset/queue_intent.go
     sha256: 0793ef4a19052849f265dfe40a06a8f3330bef053ea2fff84e080b269a85f7b7
@@ -112,7 +115,7 @@ build:
     - sqlite_omit_load_extension
   reproducible_rebuilds: 2
   maximum_required_glibc: "2.34"
-  binary_sha256: 5f4e5177044f0a3bff8ece13d5f865a85362a75296ebad07eccf8d3faa9af0f0
+  binary_sha256: 118b1ee61acb7e2737f8e2c8c907d6498881d710251a9431a8878358352d2419
 runtime_contract:
   queue_intent_schema_version: 4
   event_driven_scale_set_wake: true

internal/repositorycontract/external_downloads_test.go

@@ -89,3 +89,25 @@ func TestExternalDownloadFallbackMarkersRemainBounded(t *testing.T) {
 		}
 	}
 }
+
+func TestGARMAndProviderRunnerWrappersShareTheAttemptBudget(t *testing.T) {
+	t.Parallel()
+	root := toolCacheRepositoryRoot(t)
+	paths := []string{
+		"internal/garmproviderincus/provider/specs.go",
+		"third_party/garm/patches/0027-bound-upstream-download-attempts.patch",
+	}
+	for _, relative := range paths {
+		raw, err := os.ReadFile(filepath.Join(root, relative))
+		if err != nil {
+			t.Fatal(err)
+		}
+		text := string(raw)
+		if !strings.Contains(text, `--retry 2 --retry-delay 5 --retry-connrefused`) {
+			t.Errorf("%s does not pin the three-total-attempt runner wrapper", relative)
+		}
+		if relative == paths[0] && strings.Contains(text, `--retry 5 --retry-delay 5 --retry-connrefused`) {
+			t.Errorf("%s retains the incompatible six-total-attempt runner wrapper", relative)
+		}
+	}
+}

scripts/build-garm-nddev.sh

@@ -19,7 +19,7 @@ set -Eeuo pipefail
 # Every value below is the manifest's. Editing one here detaches the build
 # from the provenance it is reviewed against, which is why the region is
 # regenerated and compared rather than maintained.
-readonly derivative_version="v0.2.1-nddev.80"
+readonly derivative_version="v0.2.1-nddev.81"
 readonly upstream_repository="https://github.com/cloudbase/garm"
 readonly upstream_commit="154638445c3949c1958b01812f69d9a1e4d82684"
 readonly build_image="docker.io/library/golang@sha256:116d58cbd88c1297624acc6e967a060012422bacf9930927e23fb719189c6f36"
@@ -32,7 +32,7 @@ readonly build_module_mode="vendor"
 readonly build_tags="osusergo,netgo,sqlite_omit_load_extension"
 readonly build_reproducible_rebuilds="2"
 readonly build_maximum_required_glibc="2.34"
-readonly expected_binary_sha256="5f4e5177044f0a3bff8ece13d5f865a85362a75296ebad07eccf8d3faa9af0f0"
+readonly expected_binary_sha256="118b1ee61acb7e2737f8e2c8c907d6498881d710251a9431a8878358352d2419"
 readonly patch_paths=(
   "third_party/garm/patches/0001-event-driven-reconciliation.patch"
   "third_party/garm/patches/0002-central-queue-admission.patch"
@@ -60,6 +60,7 @@ readonly patch_paths=(
   "third_party/garm/patches/0024-preserve-running-queue-intent.patch"
   "third_party/garm/patches/0025-fix-scaleset-delete-error-handling.patch"
   "third_party/garm/patches/0026-normalize-registration-token-errors.patch"
+  "third_party/garm/patches/0027-bound-upstream-download-attempts.patch"
 )
 readonly patch_sha256s=(
   "2f0571f141e7388d6ea0cb0341549ba5bf5dab26d0006382a71b76655e272d34"
@@ -88,6 +89,7 @@ readonly patch_sha256s=(
   "7af194cfa81d35f9778830c8772d3bf49c465920c4cb7d04f204ea21a25ebe6c"
   "9d3e6a6acccc66424461986905e3343f559c156fd5835adce6b2617b3bee777e"
   "e3c09371d7a67072e1185afc09db823b5ccdc3ce6e524e5e504bc101cd4e06f0"
+  "35f257c8016c7999c8780970f673c5b35e04b3b3301565cffea0bd242e168456"
 )
 readonly overlay_paths=(
   "third_party/garm/overlay/workers/scaleset/queue_intent.go"

third_party/garm/patches/0027-bound-upstream-download-attempts.patch

@@ -0,0 +1,128 @@
+diff --git a/internal/templates/userdata/gitea_linux_userdata.tmpl b/internal/templates/userdata/gitea_linux_userdata.tmpl
+index b64dec99..57ee9cb9 100644
+--- a/internal/templates/userdata/gitea_linux_userdata.tmpl
++++ b/internal/templates/userdata/gitea_linux_userdata.tmpl
+@@ -21,7 +21,7 @@ fi
+ function call() {
+ 	PAYLOAD="$1"
+ 	[[ $CALLBACK_URL =~ ^(.*)/status(/)?$ ]] || CALLBACK_URL="${CALLBACK_URL}/status"
+-	curl --retry 5 --retry-delay 5 --retry-connrefused --fail -s -X POST -d "${PAYLOAD}" -H 'Accept: application/json' -H "Authorization: Bearer ${BEARER_TOKEN}" "${CALLBACK_URL}" || echo "failed to call home: exit code ($?)"
++	curl --retry 2 --retry-delay 5 --retry-connrefused --fail -s -X POST -d "${PAYLOAD}" -H 'Accept: application/json' -H "Authorization: Bearer ${BEARER_TOKEN}" "${CALLBACK_URL}" || echo "failed to call home: exit code ($?)"
+ }
+
+ function systemInfo() {
+@@ -35,7 +35,7 @@ function systemInfo() {
+ 	[[ $CALLBACK_URL =~ ^(.*)/status(/)?$ ]] && CALLBACK_URL="${BASH_REMATCH[1]}" || true
+ 	SYSINFO_URL="${CALLBACK_URL}/system-info/"
+ 	PAYLOAD="{\"os_name\": \"$OS_NAME\", \"os_version\": \"$OS_VERSION\", \"agent_id\": $AGENT_ID}"
+-	curl --retry 5 --retry-delay 5 --retry-connrefused --fail -s -X POST -d "${PAYLOAD}" -H 'Accept: application/json' -H "Authorization: Bearer ${BEARER_TOKEN}" "${SYSINFO_URL}" || true
++	curl --retry 2 --retry-delay 5 --retry-connrefused --fail -s -X POST -d "${PAYLOAD}" -H 'Accept: application/json' -H "Authorization: Bearer ${BEARER_TOKEN}" "${SYSINFO_URL}" || true
+ }
+
+ function sendStatus() {
+@@ -68,7 +68,7 @@ if [ "$AGENT_MODE" == "true" ]; then
+     AGENT_TOKEN="{{ .AgentToken }}"
+     AGENT_SHELL={{ .AgentShell }}
+     sendStatus "Downloading agent from $DOWNLOAD_URL"
+-    sudo curl --retry 5 \
++    sudo curl --retry 2 \
+         --retry-delay 5 \
+         --retry-connrefused \
+         --fail -L \
+@@ -119,7 +119,7 @@ fi
+ function downloadAndExtractRunner() {
+ 	sendStatus "downloading tools from {{ .DownloadURL }}"
+     mkdir -p "$RUN_HOME" || fail "failed to create actions-runner folder"
+-	curl --retry 5 --retry-delay 5 --retry-connrefused --fail -L -o "$RUN_HOME/gitea-runner" "{{ .DownloadURL }}" || fail "failed to download tools"
++	curl --retry 2 --retry-delay 5 --retry-connrefused --fail -L -o "$RUN_HOME/gitea-runner" "{{ .DownloadURL }}" || fail "failed to download tools"
+ 	chown {{ .RunnerUsername }}:{{ .RunnerGroup }} -R "$RUN_HOME"/ || fail "failed to change owner"
+ 	chmod +x "$RUN_HOME/gitea-runner" || fail "failed to set executable flag"
+ }
+@@ -134,14 +134,14 @@ cd "$RUN_HOME"
+
+ sendStatus "configuring runner"
+ function getRunnerFile() {
+-	curl --retry 5 --retry-delay 5 \
++	curl --retry 2 --retry-delay 5 \
+ 		--retry-connrefused --fail -s \
+ 		-X GET -H 'Accept: application/json' \
+ 		-H "Authorization: Bearer ${BEARER_TOKEN}" \
+ 		"${METADATA_URL}/$1" -o "$2"
+ }
+
+-GITHUB_TOKEN=$(curl --retry 5 --retry-delay 5 --retry-connrefused --fail -s -X GET -H 'Accept: application/json' -H "Authorization: Bearer ${BEARER_TOKEN}" "${METADATA_URL}/runner-registration-token/")
++GITHUB_TOKEN=$(curl --retry 2 --retry-delay 5 --retry-connrefused --fail -s -X GET -H 'Accept: application/json' -H "Authorization: Bearer ${BEARER_TOKEN}" "${METADATA_URL}/runner-registration-token/")
+
+ set +e
+ attempt=1
+diff --git a/internal/templates/userdata/github_linux_userdata.tmpl b/internal/templates/userdata/github_linux_userdata.tmpl
+index 609623cf..fbf68938 100644
+--- a/internal/templates/userdata/github_linux_userdata.tmpl
++++ b/internal/templates/userdata/github_linux_userdata.tmpl
+@@ -30,7 +30,7 @@ fi
+ function call() {
+ 	PAYLOAD="$1"
+ 	[[ $CALLBACK_URL =~ ^(.*)/status(/)?$ ]] || CALLBACK_URL="${CALLBACK_URL}/status"
+-	curl --retry 5 --retry-delay 5 --retry-connrefused --fail -s -X POST -d "${PAYLOAD}" -H 'Accept: application/json' -H "Authorization: Bearer ${BEARER_TOKEN}" "${CALLBACK_URL}" || echo "failed to call home: exit code ($?)"
++	curl --retry 2 --retry-delay 5 --retry-connrefused --fail -s -X POST -d "${PAYLOAD}" -H 'Accept: application/json' -H "Authorization: Bearer ${BEARER_TOKEN}" "${CALLBACK_URL}" || echo "failed to call home: exit code ($?)"
+ }
+
+ function systemInfo() {
+@@ -44,7 +44,7 @@ function systemInfo() {
+ 	[[ $CALLBACK_URL =~ ^(.*)/status(/)?$ ]] && CALLBACK_URL="${BASH_REMATCH[1]}" || true
+ 	SYSINFO_URL="${CALLBACK_URL}/system-info/"
+ 	PAYLOAD="{\"os_name\": \"$OS_NAME\", \"os_version\": \"$OS_VERSION\", \"agent_id\": $AGENT_ID}"
+-	curl --retry 5 --retry-delay 5 --retry-connrefused --fail -s -X POST -d "${PAYLOAD}" -H 'Accept: application/json' -H "Authorization: Bearer ${BEARER_TOKEN}" "${SYSINFO_URL}" || true
++	curl --retry 2 --retry-delay 5 --retry-connrefused --fail -s -X POST -d "${PAYLOAD}" -H 'Accept: application/json' -H "Authorization: Bearer ${BEARER_TOKEN}" "${SYSINFO_URL}" || true
+ }
+
+ function sendStatus() {
+@@ -77,7 +77,7 @@ if [ "$AGENT_MODE" == "true" ]; then
+     AGENT_TOKEN="{{ .AgentToken }}"
+     AGENT_SHELL={{ .AgentShell }}
+     sendStatus "Downloading agent from $DOWNLOAD_URL"
+-    sudo curl --retry 5 \
++    sudo curl --retry 2 \
+         --retry-delay 5 \
+         --retry-connrefused \
+         --fail -L \
+@@ -130,7 +130,7 @@ function downloadAndExtractRunner() {
+ 	if [ ! -z "{{ .TempDownloadToken }}" ]; then
+ 	TEMP_TOKEN="Authorization: Bearer {{ .TempDownloadToken }}"
+ 	fi
+-	curl --retry 5 --retry-delay 5 --retry-connrefused --fail -L -H "${TEMP_TOKEN}" -o "/home/{{ .RunnerUsername }}/{{ .FileName }}" "{{ .DownloadURL }}" || fail "failed to download tools"
++	curl --retry 2 --retry-delay 5 --retry-connrefused --fail -L -H "${TEMP_TOKEN}" -o "/home/{{ .RunnerUsername }}/{{ .FileName }}" "{{ .DownloadURL }}" || fail "failed to download tools"
+ 	mkdir -p "$RUN_HOME" || fail "failed to create actions-runner folder"
+ 	sendStatus "extracting runner"
+ 	tar xf "/home/{{ .RunnerUsername }}/{{ .FileName }}" -C "$RUN_HOME"/ || fail "failed to extract runner"
+@@ -160,7 +160,7 @@ fi
+ sendStatus "configuring runner"
+ {{- if .UseJITConfig }}
+ function getRunnerFile() {
+-	curl --retry 5 --retry-delay 5 \
++	curl --retry 2 --retry-delay 5 \
+ 		--retry-connrefused --fail -s \
+ 		-X GET -H 'Accept: application/json' \
+ 		-H "Authorization: Bearer ${BEARER_TOKEN}" \
+@@ -192,7 +192,7 @@ if [ "$AGENT_MODE" != "true" ]; then
+ fi
+ {{- else}}
+
+-GITHUB_TOKEN=$(curl --retry 5 --retry-delay 5 --retry-connrefused --fail -s -X GET -H 'Accept: application/json' -H "Authorization: Bearer ${BEARER_TOKEN}" "${METADATA_URL}/runner-registration-token/")
++GITHUB_TOKEN=$(curl --retry 2 --retry-delay 5 --retry-connrefused --fail -s -X GET -H 'Accept: application/json' -H "Authorization: Bearer ${BEARER_TOKEN}" "${METADATA_URL}/runner-registration-token/")
+
+
+ set +e
+diff --git a/internal/templates/userdata/linux_wrapper.tmpl b/internal/templates/userdata/linux_wrapper.tmpl
+index b672e8a5..4d44139a 100644
+--- a/internal/templates/userdata/linux_wrapper.tmpl
++++ b/internal/templates/userdata/linux_wrapper.tmpl
+@@ -6,7 +6,7 @@ set -o pipefail
+ METADATA_URL="{{ .MetadataURL }}"
+ BEARER_TOKEN="{{ .CallbackToken }}"
+
+-curl -H "Authorization: Bearer $BEARER_TOKEN" --retry 5 --retry-delay 5 --retry-connrefused --fail $METADATA_URL/install-script/ -o /tmp/real-install.sh
++curl -H "Authorization: Bearer $BEARER_TOKEN" --retry 2 --retry-delay 5 --retry-connrefused --fail $METADATA_URL/install-script/ -o /tmp/real-install.sh
+ chmod +x /tmp/real-install.sh
+
+ /tmp/real-install.sh