From 571cdd62bfb28db167ea226e3f75aed6fca794b9 Mon Sep 17 00:00:00 2001 From: rldyourmnd Date: Wed, 26 Aug 2026 04:17:23 +0500 Subject: [PATCH] fix(release): synchronize GARM runner wrapper --- CHANGELOG.md | 4 + config/example-runner-1.yaml | 2 +- config/example-runner-2.yaml | 2 +- config/example-runner-3.yaml | 2 +- config/example-runner-4.yaml | 2 +- config/example-services.yaml | 2 +- config/garm-derivative.yaml | 7 +- .../external_downloads_test.go | 22 +++ scripts/build-garm-nddev.sh | 6 +- ...027-bound-upstream-download-attempts.patch | 128 ++++++++++++++++++ 10 files changed, 168 insertions(+), 9 deletions(-) create mode 100644 third_party/garm/patches/0027-bound-upstream-download-attempts.patch diff --git a/CHANGELOG.md b/CHANGELOG.md index 758eeec..0092fd4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +- Added GARM derivative `v0.2.1-nddev.81`: all embedded bootstrap downloads + now use the same three-total-attempt contract as provider `.78`, preventing + an incompatible wrapper rollout from blocking every new runner. + - Added provider derivative `v0.1.5-nddev.78`: runner metadata bootstrap uses the same three-total-attempt download contract as every other reviewed network artifact path. diff --git a/config/example-runner-1.yaml b/config/example-runner-1.yaml index 1e51d25..4b9976a 100644 --- a/config/example-runner-1.yaml +++ b/config/example-runner-1.yaml @@ -6,7 +6,7 @@ platform: control_plane: manager: garm - manager_version: v0.2.1-nddev.80 + manager_version: v0.2.1-nddev.81 scheduling_mode: scale-set provider: incus provider_version: v0.1.5-nddev.78 diff --git a/config/example-runner-2.yaml b/config/example-runner-2.yaml index 06586bd..5a8f390 100644 --- a/config/example-runner-2.yaml +++ b/config/example-runner-2.yaml @@ -6,7 +6,7 @@ platform: control_plane: manager: garm - manager_version: v0.2.1-nddev.80 + manager_version: v0.2.1-nddev.81 scheduling_mode: scale-set provider: incus provider_version: v0.1.5-nddev.78 diff --git a/config/example-runner-3.yaml b/config/example-runner-3.yaml index 373cca9..ae0971a 100644 --- a/config/example-runner-3.yaml +++ b/config/example-runner-3.yaml @@ -6,7 +6,7 @@ platform: control_plane: manager: garm - manager_version: v0.2.1-nddev.80 + manager_version: v0.2.1-nddev.81 scheduling_mode: scale-set provider: incus provider_version: v0.1.5-nddev.78 diff --git a/config/example-runner-4.yaml b/config/example-runner-4.yaml index dabe025..cab611e 100644 --- a/config/example-runner-4.yaml +++ b/config/example-runner-4.yaml @@ -6,7 +6,7 @@ platform: control_plane: manager: garm - manager_version: v0.2.1-nddev.80 + manager_version: v0.2.1-nddev.81 scheduling_mode: scale-set provider: incus provider_version: v0.1.5-nddev.78 diff --git a/config/example-services.yaml b/config/example-services.yaml index 2bc0297..0019f43 100644 --- a/config/example-services.yaml +++ b/config/example-services.yaml @@ -24,7 +24,7 @@ platform: control_plane: manager: garm - manager_version: v0.2.1-nddev.80 + manager_version: v0.2.1-nddev.81 scheduling_mode: scale-set provider: incus provider_version: v0.1.5-nddev.78 diff --git a/config/garm-derivative.yaml b/config/garm-derivative.yaml index 93e8bc8..e2bb171 100644 --- a/config/garm-derivative.yaml +++ b/config/garm-derivative.yaml @@ -1,6 +1,6 @@ schema_version: 1 artifact: garm -derivative_version: v0.2.1-nddev.80 +derivative_version: v0.2.1-nddev.81 upstream: repository: https://github.com/cloudbase/garm release: v0.2.1 @@ -85,6 +85,9 @@ patches: - path: third_party/garm/patches/0026-normalize-registration-token-errors.patch sha256: e3c09371d7a67072e1185afc09db823b5ccdc3ce6e524e5e504bc101cd4e06f0 purpose: Normalize GitHub registration-token API failures through the shared typed error mapping so repository 404 responses remain detectable through wrapped scale-set operations. + - path: third_party/garm/patches/0027-bound-upstream-download-attempts.patch + sha256: 35f257c8016c7999c8780970f673c5b35e04b3b3301565cffea0bd242e168456 + purpose: Bound every upstream Linux and Gitea bootstrap curl path to two retries after the initial attempt so GARM and the provider enforce the same three-total-attempt contract. overlays: - path: third_party/garm/overlay/workers/scaleset/queue_intent.go sha256: 0793ef4a19052849f265dfe40a06a8f3330bef053ea2fff84e080b269a85f7b7 @@ -112,7 +115,7 @@ build: - sqlite_omit_load_extension reproducible_rebuilds: 2 maximum_required_glibc: "2.34" - binary_sha256: 5f4e5177044f0a3bff8ece13d5f865a85362a75296ebad07eccf8d3faa9af0f0 + binary_sha256: 118b1ee61acb7e2737f8e2c8c907d6498881d710251a9431a8878358352d2419 runtime_contract: queue_intent_schema_version: 4 event_driven_scale_set_wake: true diff --git a/internal/repositorycontract/external_downloads_test.go b/internal/repositorycontract/external_downloads_test.go index d54d342..da72aee 100644 --- a/internal/repositorycontract/external_downloads_test.go +++ b/internal/repositorycontract/external_downloads_test.go @@ -89,3 +89,25 @@ func TestExternalDownloadFallbackMarkersRemainBounded(t *testing.T) { } } } + +func TestGARMAndProviderRunnerWrappersShareTheAttemptBudget(t *testing.T) { + t.Parallel() + root := toolCacheRepositoryRoot(t) + paths := []string{ + "internal/garmproviderincus/provider/specs.go", + "third_party/garm/patches/0027-bound-upstream-download-attempts.patch", + } + for _, relative := range paths { + raw, err := os.ReadFile(filepath.Join(root, relative)) + if err != nil { + t.Fatal(err) + } + text := string(raw) + if !strings.Contains(text, `--retry 2 --retry-delay 5 --retry-connrefused`) { + t.Errorf("%s does not pin the three-total-attempt runner wrapper", relative) + } + if relative == paths[0] && strings.Contains(text, `--retry 5 --retry-delay 5 --retry-connrefused`) { + t.Errorf("%s retains the incompatible six-total-attempt runner wrapper", relative) + } + } +} diff --git a/scripts/build-garm-nddev.sh b/scripts/build-garm-nddev.sh index 11c290a..a883286 100755 --- a/scripts/build-garm-nddev.sh +++ b/scripts/build-garm-nddev.sh @@ -19,7 +19,7 @@ set -Eeuo pipefail # Every value below is the manifest's. Editing one here detaches the build # from the provenance it is reviewed against, which is why the region is # regenerated and compared rather than maintained. -readonly derivative_version="v0.2.1-nddev.80" +readonly derivative_version="v0.2.1-nddev.81" readonly upstream_repository="https://github.com/cloudbase/garm" readonly upstream_commit="154638445c3949c1958b01812f69d9a1e4d82684" readonly build_image="docker.io/library/golang@sha256:116d58cbd88c1297624acc6e967a060012422bacf9930927e23fb719189c6f36" @@ -32,7 +32,7 @@ readonly build_module_mode="vendor" readonly build_tags="osusergo,netgo,sqlite_omit_load_extension" readonly build_reproducible_rebuilds="2" readonly build_maximum_required_glibc="2.34" -readonly expected_binary_sha256="5f4e5177044f0a3bff8ece13d5f865a85362a75296ebad07eccf8d3faa9af0f0" +readonly expected_binary_sha256="118b1ee61acb7e2737f8e2c8c907d6498881d710251a9431a8878358352d2419" readonly patch_paths=( "third_party/garm/patches/0001-event-driven-reconciliation.patch" "third_party/garm/patches/0002-central-queue-admission.patch" @@ -60,6 +60,7 @@ readonly patch_paths=( "third_party/garm/patches/0024-preserve-running-queue-intent.patch" "third_party/garm/patches/0025-fix-scaleset-delete-error-handling.patch" "third_party/garm/patches/0026-normalize-registration-token-errors.patch" + "third_party/garm/patches/0027-bound-upstream-download-attempts.patch" ) readonly patch_sha256s=( "2f0571f141e7388d6ea0cb0341549ba5bf5dab26d0006382a71b76655e272d34" @@ -88,6 +89,7 @@ readonly patch_sha256s=( "7af194cfa81d35f9778830c8772d3bf49c465920c4cb7d04f204ea21a25ebe6c" "9d3e6a6acccc66424461986905e3343f559c156fd5835adce6b2617b3bee777e" "e3c09371d7a67072e1185afc09db823b5ccdc3ce6e524e5e504bc101cd4e06f0" + "35f257c8016c7999c8780970f673c5b35e04b3b3301565cffea0bd242e168456" ) readonly overlay_paths=( "third_party/garm/overlay/workers/scaleset/queue_intent.go" diff --git a/third_party/garm/patches/0027-bound-upstream-download-attempts.patch b/third_party/garm/patches/0027-bound-upstream-download-attempts.patch new file mode 100644 index 0000000..8894db4 --- /dev/null +++ b/third_party/garm/patches/0027-bound-upstream-download-attempts.patch @@ -0,0 +1,128 @@ +diff --git a/internal/templates/userdata/gitea_linux_userdata.tmpl b/internal/templates/userdata/gitea_linux_userdata.tmpl +index b64dec99..57ee9cb9 100644 +--- a/internal/templates/userdata/gitea_linux_userdata.tmpl ++++ b/internal/templates/userdata/gitea_linux_userdata.tmpl +@@ -21,7 +21,7 @@ fi + function call() { + PAYLOAD="$1" + [[ $CALLBACK_URL =~ ^(.*)/status(/)?$ ]] || CALLBACK_URL="${CALLBACK_URL}/status" +- curl --retry 5 --retry-delay 5 --retry-connrefused --fail -s -X POST -d "${PAYLOAD}" -H 'Accept: application/json' -H "Authorization: Bearer ${BEARER_TOKEN}" "${CALLBACK_URL}" || echo "failed to call home: exit code ($?)" ++ curl --retry 2 --retry-delay 5 --retry-connrefused --fail -s -X POST -d "${PAYLOAD}" -H 'Accept: application/json' -H "Authorization: Bearer ${BEARER_TOKEN}" "${CALLBACK_URL}" || echo "failed to call home: exit code ($?)" + } + + function systemInfo() { +@@ -35,7 +35,7 @@ function systemInfo() { + [[ $CALLBACK_URL =~ ^(.*)/status(/)?$ ]] && CALLBACK_URL="${BASH_REMATCH[1]}" || true + SYSINFO_URL="${CALLBACK_URL}/system-info/" + PAYLOAD="{\"os_name\": \"$OS_NAME\", \"os_version\": \"$OS_VERSION\", \"agent_id\": $AGENT_ID}" +- curl --retry 5 --retry-delay 5 --retry-connrefused --fail -s -X POST -d "${PAYLOAD}" -H 'Accept: application/json' -H "Authorization: Bearer ${BEARER_TOKEN}" "${SYSINFO_URL}" || true ++ curl --retry 2 --retry-delay 5 --retry-connrefused --fail -s -X POST -d "${PAYLOAD}" -H 'Accept: application/json' -H "Authorization: Bearer ${BEARER_TOKEN}" "${SYSINFO_URL}" || true + } + + function sendStatus() { +@@ -68,7 +68,7 @@ if [ "$AGENT_MODE" == "true" ]; then + AGENT_TOKEN="{{ .AgentToken }}" + AGENT_SHELL={{ .AgentShell }} + sendStatus "Downloading agent from $DOWNLOAD_URL" +- sudo curl --retry 5 \ ++ sudo curl --retry 2 \ + --retry-delay 5 \ + --retry-connrefused \ + --fail -L \ +@@ -119,7 +119,7 @@ fi + function downloadAndExtractRunner() { + sendStatus "downloading tools from {{ .DownloadURL }}" + mkdir -p "$RUN_HOME" || fail "failed to create actions-runner folder" +- curl --retry 5 --retry-delay 5 --retry-connrefused --fail -L -o "$RUN_HOME/gitea-runner" "{{ .DownloadURL }}" || fail "failed to download tools" ++ curl --retry 2 --retry-delay 5 --retry-connrefused --fail -L -o "$RUN_HOME/gitea-runner" "{{ .DownloadURL }}" || fail "failed to download tools" + chown {{ .RunnerUsername }}:{{ .RunnerGroup }} -R "$RUN_HOME"/ || fail "failed to change owner" + chmod +x "$RUN_HOME/gitea-runner" || fail "failed to set executable flag" + } +@@ -134,14 +134,14 @@ cd "$RUN_HOME" + + sendStatus "configuring runner" + function getRunnerFile() { +- curl --retry 5 --retry-delay 5 \ ++ curl --retry 2 --retry-delay 5 \ + --retry-connrefused --fail -s \ + -X GET -H 'Accept: application/json' \ + -H "Authorization: Bearer ${BEARER_TOKEN}" \ + "${METADATA_URL}/$1" -o "$2" + } + +-GITHUB_TOKEN=$(curl --retry 5 --retry-delay 5 --retry-connrefused --fail -s -X GET -H 'Accept: application/json' -H "Authorization: Bearer ${BEARER_TOKEN}" "${METADATA_URL}/runner-registration-token/") ++GITHUB_TOKEN=$(curl --retry 2 --retry-delay 5 --retry-connrefused --fail -s -X GET -H 'Accept: application/json' -H "Authorization: Bearer ${BEARER_TOKEN}" "${METADATA_URL}/runner-registration-token/") + + set +e + attempt=1 +diff --git a/internal/templates/userdata/github_linux_userdata.tmpl b/internal/templates/userdata/github_linux_userdata.tmpl +index 609623cf..fbf68938 100644 +--- a/internal/templates/userdata/github_linux_userdata.tmpl ++++ b/internal/templates/userdata/github_linux_userdata.tmpl +@@ -30,7 +30,7 @@ fi + function call() { + PAYLOAD="$1" + [[ $CALLBACK_URL =~ ^(.*)/status(/)?$ ]] || CALLBACK_URL="${CALLBACK_URL}/status" +- curl --retry 5 --retry-delay 5 --retry-connrefused --fail -s -X POST -d "${PAYLOAD}" -H 'Accept: application/json' -H "Authorization: Bearer ${BEARER_TOKEN}" "${CALLBACK_URL}" || echo "failed to call home: exit code ($?)" ++ curl --retry 2 --retry-delay 5 --retry-connrefused --fail -s -X POST -d "${PAYLOAD}" -H 'Accept: application/json' -H "Authorization: Bearer ${BEARER_TOKEN}" "${CALLBACK_URL}" || echo "failed to call home: exit code ($?)" + } + + function systemInfo() { +@@ -44,7 +44,7 @@ function systemInfo() { + [[ $CALLBACK_URL =~ ^(.*)/status(/)?$ ]] && CALLBACK_URL="${BASH_REMATCH[1]}" || true + SYSINFO_URL="${CALLBACK_URL}/system-info/" + PAYLOAD="{\"os_name\": \"$OS_NAME\", \"os_version\": \"$OS_VERSION\", \"agent_id\": $AGENT_ID}" +- curl --retry 5 --retry-delay 5 --retry-connrefused --fail -s -X POST -d "${PAYLOAD}" -H 'Accept: application/json' -H "Authorization: Bearer ${BEARER_TOKEN}" "${SYSINFO_URL}" || true ++ curl --retry 2 --retry-delay 5 --retry-connrefused --fail -s -X POST -d "${PAYLOAD}" -H 'Accept: application/json' -H "Authorization: Bearer ${BEARER_TOKEN}" "${SYSINFO_URL}" || true + } + + function sendStatus() { +@@ -77,7 +77,7 @@ if [ "$AGENT_MODE" == "true" ]; then + AGENT_TOKEN="{{ .AgentToken }}" + AGENT_SHELL={{ .AgentShell }} + sendStatus "Downloading agent from $DOWNLOAD_URL" +- sudo curl --retry 5 \ ++ sudo curl --retry 2 \ + --retry-delay 5 \ + --retry-connrefused \ + --fail -L \ +@@ -130,7 +130,7 @@ function downloadAndExtractRunner() { + if [ ! -z "{{ .TempDownloadToken }}" ]; then + TEMP_TOKEN="Authorization: Bearer {{ .TempDownloadToken }}" + fi +- curl --retry 5 --retry-delay 5 --retry-connrefused --fail -L -H "${TEMP_TOKEN}" -o "/home/{{ .RunnerUsername }}/{{ .FileName }}" "{{ .DownloadURL }}" || fail "failed to download tools" ++ curl --retry 2 --retry-delay 5 --retry-connrefused --fail -L -H "${TEMP_TOKEN}" -o "/home/{{ .RunnerUsername }}/{{ .FileName }}" "{{ .DownloadURL }}" || fail "failed to download tools" + mkdir -p "$RUN_HOME" || fail "failed to create actions-runner folder" + sendStatus "extracting runner" + tar xf "/home/{{ .RunnerUsername }}/{{ .FileName }}" -C "$RUN_HOME"/ || fail "failed to extract runner" +@@ -160,7 +160,7 @@ fi + sendStatus "configuring runner" + {{- if .UseJITConfig }} + function getRunnerFile() { +- curl --retry 5 --retry-delay 5 \ ++ curl --retry 2 --retry-delay 5 \ + --retry-connrefused --fail -s \ + -X GET -H 'Accept: application/json' \ + -H "Authorization: Bearer ${BEARER_TOKEN}" \ +@@ -192,7 +192,7 @@ if [ "$AGENT_MODE" != "true" ]; then + fi + {{- else}} + +-GITHUB_TOKEN=$(curl --retry 5 --retry-delay 5 --retry-connrefused --fail -s -X GET -H 'Accept: application/json' -H "Authorization: Bearer ${BEARER_TOKEN}" "${METADATA_URL}/runner-registration-token/") ++GITHUB_TOKEN=$(curl --retry 2 --retry-delay 5 --retry-connrefused --fail -s -X GET -H 'Accept: application/json' -H "Authorization: Bearer ${BEARER_TOKEN}" "${METADATA_URL}/runner-registration-token/") + + + set +e +diff --git a/internal/templates/userdata/linux_wrapper.tmpl b/internal/templates/userdata/linux_wrapper.tmpl +index b672e8a5..4d44139a 100644 +--- a/internal/templates/userdata/linux_wrapper.tmpl ++++ b/internal/templates/userdata/linux_wrapper.tmpl +@@ -6,7 +6,7 @@ set -o pipefail + METADATA_URL="{{ .MetadataURL }}" + BEARER_TOKEN="{{ .CallbackToken }}" + +-curl -H "Authorization: Bearer $BEARER_TOKEN" --retry 5 --retry-delay 5 --retry-connrefused --fail $METADATA_URL/install-script/ -o /tmp/real-install.sh ++curl -H "Authorization: Bearer $BEARER_TOKEN" --retry 2 --retry-delay 5 --retry-connrefused --fail $METADATA_URL/install-script/ -o /tmp/real-install.sh + chmod +x /tmp/real-install.sh + + /tmp/real-install.sh