Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,10 @@
# Changelog

- Added provider derivative `v0.1.5-nddev.77`: undeclared pool reservations
now use hard CPU and memory limits, removing implicit overcommit from queue
accounting and per-member placement while retaining explicitly reviewed
measured envelopes.

- Added GARM derivative `v0.2.1-nddev.80`: pre-job reservation skips exact
intents whose own retry is terminal or still deferred, preserving that state
while allowing the oldest eligible sibling job to proceed instead of
Expand Down Expand Up @@ -71,6 +76,9 @@ Versioning.

### Fixed

- Default undeclared queue reservations to each pool's hard CPU and memory
limits instead of silently applying a global historical overcommit table;
smaller measured envelopes now require an explicit reviewed pool field.
- Convert second-based alert evaluation contracts to OpenObserve's supported
minute cadence, recompute consecutive-evaluation thresholds from the rendered
schedule, and keep the lookback period no shorter than that cadence so short
Expand Down
20 changes: 10 additions & 10 deletions config/example-queue-admission.json
Original file line number Diff line number Diff line change
Expand Up @@ -12,16 +12,16 @@
"max_repository_share_percent": 75,
"capacity": {"cpu_units": 40, "memory_mib": 57344},
"scale_sets": {
"nddev-priority-integration": {"cpu_units": 4, "memory_mib": 6144, "reservation_cpu_units": 1, "reservation_memory_mib": 2048, "priority": 0},
"nddev-priority-standard": {"cpu_units": 2, "memory_mib": 4096, "reservation_cpu_units": 1, "reservation_memory_mib": 2560, "priority": 0},
"nddev-priority-untrusted": {"cpu_units": 4, "memory_mib": 6144, "reservation_cpu_units": 1, "reservation_memory_mib": 2048, "priority": 0},
"nddev-linux-container-canary": {"cpu_units": 2, "memory_mib": 2048, "reservation_cpu_units": 1, "reservation_memory_mib": 512, "priority": 1},
"nddev-linux-docker-container-canary": {"cpu_units": 2, "memory_mib": 4096, "reservation_cpu_units": 1, "reservation_memory_mib": 2560, "priority": 1},
"nddev-linux-fast": {"cpu_units": 2, "memory_mib": 3072, "reservation_cpu_units": 1, "reservation_memory_mib": 512, "priority": 1},
"nddev-linux-integration": {"cpu_units": 4, "memory_mib": 6144, "reservation_cpu_units": 1, "reservation_memory_mib": 2048, "priority": 1},
"nddev-linux-release": {"cpu_units": 4, "memory_mib": 6144, "reservation_cpu_units": 1, "reservation_memory_mib": 2048, "priority": 1},
"nddev-linux-standard": {"cpu_units": 2, "memory_mib": 4096, "reservation_cpu_units": 1, "reservation_memory_mib": 2560, "priority": 1},
"nddev-linux-untrusted": {"cpu_units": 4, "memory_mib": 6144, "reservation_cpu_units": 1, "reservation_memory_mib": 2048, "priority": 1}
"nddev-priority-integration": {"cpu_units": 4, "memory_mib": 6144, "reservation_cpu_units": 4, "reservation_memory_mib": 6144, "priority": 0},
"nddev-priority-standard": {"cpu_units": 2, "memory_mib": 4096, "reservation_cpu_units": 2, "reservation_memory_mib": 4096, "priority": 0},
"nddev-priority-untrusted": {"cpu_units": 4, "memory_mib": 6144, "reservation_cpu_units": 4, "reservation_memory_mib": 6144, "priority": 0},
"nddev-linux-container-canary": {"cpu_units": 2, "memory_mib": 2048, "reservation_cpu_units": 2, "reservation_memory_mib": 2048, "priority": 1},
"nddev-linux-docker-container-canary": {"cpu_units": 2, "memory_mib": 4096, "reservation_cpu_units": 2, "reservation_memory_mib": 4096, "priority": 1},
"nddev-linux-fast": {"cpu_units": 2, "memory_mib": 3072, "reservation_cpu_units": 2, "reservation_memory_mib": 3072, "priority": 1},
"nddev-linux-integration": {"cpu_units": 4, "memory_mib": 6144, "reservation_cpu_units": 4, "reservation_memory_mib": 6144, "priority": 1},
"nddev-linux-release": {"cpu_units": 4, "memory_mib": 6144, "reservation_cpu_units": 4, "reservation_memory_mib": 6144, "priority": 1},
"nddev-linux-standard": {"cpu_units": 2, "memory_mib": 4096, "reservation_cpu_units": 2, "reservation_memory_mib": 4096, "priority": 1},
"nddev-linux-untrusted": {"cpu_units": 4, "memory_mib": 6144, "reservation_cpu_units": 4, "reservation_memory_mib": 6144, "priority": 1}
},
"repositories": {
"example-org/example-actions": {"weight": 1, "max_in_flight": 6}
Expand Down
2 changes: 1 addition & 1 deletion config/example-runner-1.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ control_plane:
manager_version: v0.2.1-nddev.80
scheduling_mode: scale-set
provider: incus
provider_version: v0.1.5-nddev.76
provider_version: v0.1.5-nddev.77
provider_interface: v0.1.0
worker_kind: incus-container
runner: actions/runner
Expand Down
2 changes: 1 addition & 1 deletion config/example-runner-2.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ control_plane:
manager_version: v0.2.1-nddev.80
scheduling_mode: scale-set
provider: incus
provider_version: v0.1.5-nddev.76
provider_version: v0.1.5-nddev.77
provider_interface: v0.1.0
worker_kind: incus-container
runner: actions/runner
Expand Down
2 changes: 1 addition & 1 deletion config/example-runner-3.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ control_plane:
manager_version: v0.2.1-nddev.80
scheduling_mode: scale-set
provider: incus
provider_version: v0.1.5-nddev.76
provider_version: v0.1.5-nddev.77
provider_interface: v0.1.0
worker_kind: incus-container
runner: actions/runner
Expand Down
2 changes: 1 addition & 1 deletion config/example-runner-4.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ control_plane:
manager_version: v0.2.1-nddev.80
scheduling_mode: scale-set
provider: incus
provider_version: v0.1.5-nddev.76
provider_version: v0.1.5-nddev.77
provider_interface: v0.1.0
worker_kind: incus-container
runner: actions/runner
Expand Down
2 changes: 1 addition & 1 deletion config/example-services.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ control_plane:
manager_version: v0.2.1-nddev.80
scheduling_mode: scale-set
provider: incus
provider_version: v0.1.5-nddev.76
provider_version: v0.1.5-nddev.77
provider_interface: v0.1.0
worker_kind: incus-container
runner: actions/runner
Expand Down
6 changes: 3 additions & 3 deletions config/provider-derivative.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ artifact: garm-provider-incus
# state all move together, because all three derive from here. A provider change
# that does not bump it ships under the previous version, which is exactly how
# runner-1 and runner-2 diverged.
derivative_version: v0.1.5-nddev.76
derivative_version: v0.1.5-nddev.77

# The external-provider protocol GARM speaks to this binary. It moves on its own
# schedule -- a provider release does not imply an interface release -- so it is
Expand All @@ -37,8 +37,8 @@ runtime:
queue_intent_schema_version: 4

build:
source_commit: 7886cc72f041934587eff945a63c7666fcd78faf
binary_sha256: 94612caff1cf461b741719a9fd27e193dbe255a7131b984abb22de63281fb6c6
source_commit: b9937e043eb0f9e4ce62b531cc0ee421ca382f21
binary_sha256: 965d39a69654180c3d91b9c92d2e8a6b41ca96a056416aed9656409aed209990
go_version: go1.26.6
cgo_enabled: false
target_os: linux
Expand Down
20 changes: 20 additions & 0 deletions internal/config/reservation_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
package config

import "testing"

func TestEffectiveReservationDefaultsToHardLimit(t *testing.T) {
pool := Pool{Resources: Resources{VCPU: 4, MemoryMiB: 6144}}
if got, want := pool.EffectiveReservation(), (Reservation{CPUUnits: 4, MemoryMiB: 6144}); got != want {
t.Fatalf("implicit reservation = %#v, want hard limit %#v", got, want)
}
}

func TestEffectiveReservationPreservesExplicitMeasuredEnvelope(t *testing.T) {
pool := Pool{
Resources: Resources{VCPU: 4, MemoryMiB: 6144},
Reservation: Reservation{CPUUnits: 2, MemoryMiB: 4096},
}
if got, want := pool.EffectiveReservation(), pool.Reservation; got != want {
t.Fatalf("explicit reservation = %#v, want %#v", got, want)
}
}
15 changes: 5 additions & 10 deletions internal/config/types.go
Original file line number Diff line number Diff line change
Expand Up @@ -192,20 +192,15 @@ type Reservation struct {
MemoryMiB int `json:"memory_mib" yaml:"memory_mib"`
}

// EffectiveReservation returns the measured fleet admission envelope. The
// current immutable classes use seven-day OpenObserve p95 memory rounded up to
// 256 MiB; one CPU unit lets host PSI, rather than worst-case cgroup ceilings,
// close admission under real contention. Unknown development fixtures retain
// their hard limits until they have measured evidence.
// EffectiveReservation returns an explicitly reviewed measured envelope or the
// hard resource limit. Implicit overcommit is unsafe: a generic memory size
// cannot encode the workload class, peak concurrency, or host reserve that made
// one historical measurement valid.
func (p Pool) EffectiveReservation() Reservation {
if p.Reservation.CPUUnits > 0 && p.Reservation.MemoryMiB > 0 {
return p.Reservation
}
memory := map[int]int{2048: 512, 3072: 512, 4096: 2560, 6144: 2048}[p.Resources.MemoryMiB]
if memory == 0 {
return Reservation{CPUUnits: p.Resources.VCPU, MemoryMiB: p.Resources.MemoryMiB}
}
return Reservation{CPUUnits: 1, MemoryMiB: memory}
return Reservation{CPUUnits: p.Resources.VCPU, MemoryMiB: p.Resources.MemoryMiB}
}

type WarmPool struct {
Expand Down
2 changes: 1 addition & 1 deletion internal/incusplacement/scriptlet.go
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,7 @@ def committed_memory_bytes(member_name, pending_count):
fail("fleet instance has non-MiB memory limit: " + instance.name)
limit_mib = int(memory[:-3])
if limit_mib not in RESERVATION_BY_LIMIT_MIB:
fail("fleet instance has no measured memory reservation: " + instance.name)
fail("fleet instance has no memory reservation: " + instance.name)
committed += RESERVATION_BY_LIMIT_MIB[limit_mib] * 1024 * 1024
pending_without_record = pending_count - len(instances)
if pending_without_record < 0:
Expand Down
4 changes: 2 additions & 2 deletions internal/incusplacement/scriptlet_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,8 @@ func TestRenderCPUAwarePlacementFromPortablePolicy(t *testing.T) {
for _, required := range []string{
`PROJECT = "gha-fleet"`, `POOL = "gha-lvm"`,
"MINIMUM_MEMORY_BYTES = 1024 * 1024 * 1024",
"MINIMUM_MEMORY_PERCENT = 10", "MAX_WORKER_MEMORY_BYTES = 2560 * 1024 * 1024",
"2048: 512", "3072: 512", "4096: 2560", "6144: 2048",
"MINIMUM_MEMORY_PERCENT = 10", "MAX_WORKER_MEMORY_BYTES = 6144 * 1024 * 1024",
"2048: 2048", "3072: 3072", "4096: 4096", "6144: 6144",
"LOAD_TIE_EPSILON = 0.05", "def load_per_core(state, resources)",
`getattr(sysinfo, "load_averages", None)`, `getattr(cpu, "total", 0)`,
"member_load < chosen_load - LOAD_TIE_EPSILON",
Expand Down