From 574b0d19d3c3e836fe4be3c4dd3e7b05a0d06270 Mon Sep 17 00:00:00 2001 From: rldyourmnd Date: Wed, 26 Aug 2026 06:48:12 +0500 Subject: [PATCH] fix(ci): converge workflow and retry contracts --- .github/workflows/codeql.yml | 2 +- docs/external-downloads.md | 6 +++--- internal/schedulerrecovery/command.go | 26 ++++++++++++++++++++------ 3 files changed, 24 insertions(+), 10 deletions(-) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index faaa556..6d57b65 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -31,7 +31,7 @@ jobs: actions: read contents: read security-events: write - uses: NDDev-OpenNetwork/ci-workflows/.github/workflows/public-codeql.yml@b50364e2a415267688c1d845cea6866cdb5e53d6 # 0.1.4 + uses: NDDev-OpenNetwork/ci-workflows/.github/workflows/public-codeql.yml@409817cf743e76383c84e30c72edf781d73b71a1 # 0.1.11 with: # Public repository: `pull_request` runs untrusted fork code. Name the # hosted runner explicitly — the reusable's default belongs to the pinned diff --git a/docs/external-downloads.md b/docs/external-downloads.md index 5a76c67..c8aa971 100644 --- a/docs/external-downloads.md +++ b/docs/external-downloads.md @@ -7,11 +7,11 @@ appears outside this inventory. | Surface | Classification | Resilience and authority | | --- | --- | --- | | `internal/imagebuild/artifacts.go`, `config/golden-image*.yaml` | image materialization | Exact HTTPS host/path and SHA identities; size bounds; transient transport/read and HTTP 408/429/5xx receive at most three attempts. Verified bytes are baked once into the immutable worker image. | -| `actions/tool-cache/tool-cache.sh` | per-job standalone tool | Trust-scoped immutable RustFS object first; size and SHA reverified; unavailable, missing, incomplete or corrupt cache falls back to exact upstream with five bounded attempts and emits `nddev_tool_cache_event`. | -| `scripts/install-benchmark-toolchain.sh` | representative benchmark fallback | Exact preinstalled version exits without download. Hosted/cold fallback uses fixed URLs, SHA-256 and five bounded attempts; this script is benchmark evidence, not a normal fleet start hook. | +| `actions/tool-cache/tool-cache.sh` | per-job standalone tool | Trust-scoped immutable RustFS object first; size and SHA reverified; unavailable, missing, incomplete or corrupt cache falls back to exact upstream with three total attempts and emits `nddev_tool_cache_event`. | +| `scripts/install-benchmark-toolchain.sh` | representative benchmark fallback | Exact preinstalled version exits without download. Hosted/cold fallback uses fixed URLs, SHA-256 and three total attempts; this script is benchmark evidence, not a normal fleet start hook. | | `.github/workflows/ci.yml` | public self-CI | GitHub-hosted `setup-go`; private fleet consumers instead use the baked toolchain paths published by `ci-workflows`. | | `actions/package-cache/package-cache.sh`, `scripts/configure-sccache.sh` | private package/cache data | Authenticated repository-scoped S3-compatible cache traffic; never an executable download authority. Cache failure degrades to the package manager or upstream path. | -| `internal/garmproviderincus/provider/{incus.go,specs.go,admission.go,cache_delivery.go}` | VPC-local runner bootstrap | One-use instance identity and pinned runner metadata from the fleet gateway; install-script fetches use five bounded attempts. No arbitrary external tool origin is accepted. | +| `internal/garmproviderincus/provider/{incus.go,specs.go,admission.go,cache_delivery.go}` | VPC-local runner bootstrap | One-use instance identity and pinned runner metadata from the fleet gateway; install-script fetches use three total attempts. No arbitrary external tool origin is accepted. | | `scripts/build-garm-nddev.sh` | reviewed source build | Fetches the exact reviewed upstream commit and applies the checked-in derivative patch set; not executed in a job start hook. | | `internal/imagebuild/assets/{provision.sh,container-provision.sh}` | local image provisioning | Talks only to the guest-local Incus socket or installs already-downloaded, verified image inputs. | | `internal/imagebuild/assets/{smoke.sh,smoke-integration.sh}` | reachability smoke | HTTP requests assert egress, service-container and metadata isolation. Returned bytes are never executed or promoted. | diff --git a/internal/schedulerrecovery/command.go b/internal/schedulerrecovery/command.go index 0a35a20..c5b309a 100644 --- a/internal/schedulerrecovery/command.go +++ b/internal/schedulerrecovery/command.go @@ -4,10 +4,12 @@ import ( "bytes" "context" "encoding/json" + "errors" "fmt" "os/exec" "path/filepath" "strings" + "syscall" "time" ) @@ -85,12 +87,24 @@ func (executor CommandExecutor) run(ctx context.Context, argv []string, attempt } bounded, cancel := context.WithTimeout(ctx, executor.Config.Timeout) defer cancel() - command := exec.CommandContext(bounded, argv[0], argv[1:]...) - command.Env = append(command.Environ(), - "GHA_SCHEDULER_RECOVERY_ATTEMPT="+attempt.ID, - "GHA_SCHEDULER_RECOVERY_STUCK="+strings.Join(attempt.Stuck, ","), - ) - output, err := command.Output() + var output []byte + var err error + for executionAttempt := 0; executionAttempt < 3; executionAttempt++ { + command := exec.CommandContext(bounded, argv[0], argv[1:]...) + command.Env = append(command.Environ(), + "GHA_SCHEDULER_RECOVERY_ATTEMPT="+attempt.ID, + "GHA_SCHEDULER_RECOVERY_STUCK="+strings.Join(attempt.Stuck, ","), + ) + output, err = command.Output() + if !errors.Is(err, syscall.ETXTBSY) { + break + } + select { + case <-bounded.Done(): + break + case <-time.After(5 * time.Millisecond): + } + } if bounded.Err() != nil { return nil, fmt.Errorf("command timed out: %w", bounded.Err()) }