diff --git a/CHANGELOG.md b/CHANGELOG.md index d85eeea..67e830f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # Changelog +- Added provider derivative `v0.1.5-nddev.77`: undeclared pool reservations + now use hard CPU and memory limits, removing implicit overcommit from queue + accounting and per-member placement while retaining explicitly reviewed + measured envelopes. + - Added GARM derivative `v0.2.1-nddev.80`: pre-job reservation skips exact intents whose own retry is terminal or still deferred, preserving that state while allowing the oldest eligible sibling job to proceed instead of @@ -71,6 +76,9 @@ Versioning. ### Fixed +- Default undeclared queue reservations to each pool's hard CPU and memory + limits instead of silently applying a global historical overcommit table; + smaller measured envelopes now require an explicit reviewed pool field. - Convert second-based alert evaluation contracts to OpenObserve's supported minute cadence, recompute consecutive-evaluation thresholds from the rendered schedule, and keep the lookback period no shorter than that cadence so short diff --git a/config/example-queue-admission.json b/config/example-queue-admission.json index 9ab2138..e1be92d 100644 --- a/config/example-queue-admission.json +++ b/config/example-queue-admission.json @@ -12,16 +12,16 @@ "max_repository_share_percent": 75, "capacity": {"cpu_units": 40, "memory_mib": 57344}, "scale_sets": { - "nddev-priority-integration": {"cpu_units": 4, "memory_mib": 6144, "reservation_cpu_units": 1, "reservation_memory_mib": 2048, "priority": 0}, - "nddev-priority-standard": {"cpu_units": 2, "memory_mib": 4096, "reservation_cpu_units": 1, "reservation_memory_mib": 2560, "priority": 0}, - "nddev-priority-untrusted": {"cpu_units": 4, "memory_mib": 6144, "reservation_cpu_units": 1, "reservation_memory_mib": 2048, "priority": 0}, - "nddev-linux-container-canary": {"cpu_units": 2, "memory_mib": 2048, "reservation_cpu_units": 1, "reservation_memory_mib": 512, "priority": 1}, - "nddev-linux-docker-container-canary": {"cpu_units": 2, "memory_mib": 4096, "reservation_cpu_units": 1, "reservation_memory_mib": 2560, "priority": 1}, - "nddev-linux-fast": {"cpu_units": 2, "memory_mib": 3072, "reservation_cpu_units": 1, "reservation_memory_mib": 512, "priority": 1}, - "nddev-linux-integration": {"cpu_units": 4, "memory_mib": 6144, "reservation_cpu_units": 1, "reservation_memory_mib": 2048, "priority": 1}, - "nddev-linux-release": {"cpu_units": 4, "memory_mib": 6144, "reservation_cpu_units": 1, "reservation_memory_mib": 2048, "priority": 1}, - "nddev-linux-standard": {"cpu_units": 2, "memory_mib": 4096, "reservation_cpu_units": 1, "reservation_memory_mib": 2560, "priority": 1}, - "nddev-linux-untrusted": {"cpu_units": 4, "memory_mib": 6144, "reservation_cpu_units": 1, "reservation_memory_mib": 2048, "priority": 1} + "nddev-priority-integration": {"cpu_units": 4, "memory_mib": 6144, "reservation_cpu_units": 4, "reservation_memory_mib": 6144, "priority": 0}, + "nddev-priority-standard": {"cpu_units": 2, "memory_mib": 4096, "reservation_cpu_units": 2, "reservation_memory_mib": 4096, "priority": 0}, + "nddev-priority-untrusted": {"cpu_units": 4, "memory_mib": 6144, "reservation_cpu_units": 4, "reservation_memory_mib": 6144, "priority": 0}, + "nddev-linux-container-canary": {"cpu_units": 2, "memory_mib": 2048, "reservation_cpu_units": 2, "reservation_memory_mib": 2048, "priority": 1}, + "nddev-linux-docker-container-canary": {"cpu_units": 2, "memory_mib": 4096, "reservation_cpu_units": 2, "reservation_memory_mib": 4096, "priority": 1}, + "nddev-linux-fast": {"cpu_units": 2, "memory_mib": 3072, "reservation_cpu_units": 2, "reservation_memory_mib": 3072, "priority": 1}, + "nddev-linux-integration": {"cpu_units": 4, "memory_mib": 6144, "reservation_cpu_units": 4, "reservation_memory_mib": 6144, "priority": 1}, + "nddev-linux-release": {"cpu_units": 4, "memory_mib": 6144, "reservation_cpu_units": 4, "reservation_memory_mib": 6144, "priority": 1}, + "nddev-linux-standard": {"cpu_units": 2, "memory_mib": 4096, "reservation_cpu_units": 2, "reservation_memory_mib": 4096, "priority": 1}, + "nddev-linux-untrusted": {"cpu_units": 4, "memory_mib": 6144, "reservation_cpu_units": 4, "reservation_memory_mib": 6144, "priority": 1} }, "repositories": { "example-org/example-actions": {"weight": 1, "max_in_flight": 6} diff --git a/config/example-runner-1.yaml b/config/example-runner-1.yaml index 5c43bab..bc81009 100644 --- a/config/example-runner-1.yaml +++ b/config/example-runner-1.yaml @@ -9,7 +9,7 @@ control_plane: manager_version: v0.2.1-nddev.80 scheduling_mode: scale-set provider: incus - provider_version: v0.1.5-nddev.76 + provider_version: v0.1.5-nddev.77 provider_interface: v0.1.0 worker_kind: incus-container runner: actions/runner diff --git a/config/example-runner-2.yaml b/config/example-runner-2.yaml index d7e1d39..222a980 100644 --- a/config/example-runner-2.yaml +++ b/config/example-runner-2.yaml @@ -9,7 +9,7 @@ control_plane: manager_version: v0.2.1-nddev.80 scheduling_mode: scale-set provider: incus - provider_version: v0.1.5-nddev.76 + provider_version: v0.1.5-nddev.77 provider_interface: v0.1.0 worker_kind: incus-container runner: actions/runner diff --git a/config/example-runner-3.yaml b/config/example-runner-3.yaml index d66b2bc..ea45787 100644 --- a/config/example-runner-3.yaml +++ b/config/example-runner-3.yaml @@ -9,7 +9,7 @@ control_plane: manager_version: v0.2.1-nddev.80 scheduling_mode: scale-set provider: incus - provider_version: v0.1.5-nddev.76 + provider_version: v0.1.5-nddev.77 provider_interface: v0.1.0 worker_kind: incus-container runner: actions/runner diff --git a/config/example-runner-4.yaml b/config/example-runner-4.yaml index eb1a311..b4940a0 100644 --- a/config/example-runner-4.yaml +++ b/config/example-runner-4.yaml @@ -9,7 +9,7 @@ control_plane: manager_version: v0.2.1-nddev.80 scheduling_mode: scale-set provider: incus - provider_version: v0.1.5-nddev.76 + provider_version: v0.1.5-nddev.77 provider_interface: v0.1.0 worker_kind: incus-container runner: actions/runner diff --git a/config/example-services.yaml b/config/example-services.yaml index 2ceb439..4ed47d5 100644 --- a/config/example-services.yaml +++ b/config/example-services.yaml @@ -27,7 +27,7 @@ control_plane: manager_version: v0.2.1-nddev.80 scheduling_mode: scale-set provider: incus - provider_version: v0.1.5-nddev.76 + provider_version: v0.1.5-nddev.77 provider_interface: v0.1.0 worker_kind: incus-container runner: actions/runner diff --git a/config/provider-derivative.yaml b/config/provider-derivative.yaml index 961c2ee..9e1db4e 100644 --- a/config/provider-derivative.yaml +++ b/config/provider-derivative.yaml @@ -16,7 +16,7 @@ artifact: garm-provider-incus # state all move together, because all three derive from here. A provider change # that does not bump it ships under the previous version, which is exactly how # runner-1 and runner-2 diverged. -derivative_version: v0.1.5-nddev.76 +derivative_version: v0.1.5-nddev.77 # The external-provider protocol GARM speaks to this binary. It moves on its own # schedule -- a provider release does not imply an interface release -- so it is @@ -37,8 +37,8 @@ runtime: queue_intent_schema_version: 4 build: - source_commit: 7886cc72f041934587eff945a63c7666fcd78faf - binary_sha256: 94612caff1cf461b741719a9fd27e193dbe255a7131b984abb22de63281fb6c6 + source_commit: b9937e043eb0f9e4ce62b531cc0ee421ca382f21 + binary_sha256: 965d39a69654180c3d91b9c92d2e8a6b41ca96a056416aed9656409aed209990 go_version: go1.26.6 cgo_enabled: false target_os: linux diff --git a/internal/config/reservation_test.go b/internal/config/reservation_test.go new file mode 100644 index 0000000..9b09904 --- /dev/null +++ b/internal/config/reservation_test.go @@ -0,0 +1,20 @@ +package config + +import "testing" + +func TestEffectiveReservationDefaultsToHardLimit(t *testing.T) { + pool := Pool{Resources: Resources{VCPU: 4, MemoryMiB: 6144}} + if got, want := pool.EffectiveReservation(), (Reservation{CPUUnits: 4, MemoryMiB: 6144}); got != want { + t.Fatalf("implicit reservation = %#v, want hard limit %#v", got, want) + } +} + +func TestEffectiveReservationPreservesExplicitMeasuredEnvelope(t *testing.T) { + pool := Pool{ + Resources: Resources{VCPU: 4, MemoryMiB: 6144}, + Reservation: Reservation{CPUUnits: 2, MemoryMiB: 4096}, + } + if got, want := pool.EffectiveReservation(), pool.Reservation; got != want { + t.Fatalf("explicit reservation = %#v, want %#v", got, want) + } +} diff --git a/internal/config/types.go b/internal/config/types.go index 351d299..8bcefbf 100644 --- a/internal/config/types.go +++ b/internal/config/types.go @@ -192,20 +192,15 @@ type Reservation struct { MemoryMiB int `json:"memory_mib" yaml:"memory_mib"` } -// EffectiveReservation returns the measured fleet admission envelope. The -// current immutable classes use seven-day OpenObserve p95 memory rounded up to -// 256 MiB; one CPU unit lets host PSI, rather than worst-case cgroup ceilings, -// close admission under real contention. Unknown development fixtures retain -// their hard limits until they have measured evidence. +// EffectiveReservation returns an explicitly reviewed measured envelope or the +// hard resource limit. Implicit overcommit is unsafe: a generic memory size +// cannot encode the workload class, peak concurrency, or host reserve that made +// one historical measurement valid. func (p Pool) EffectiveReservation() Reservation { if p.Reservation.CPUUnits > 0 && p.Reservation.MemoryMiB > 0 { return p.Reservation } - memory := map[int]int{2048: 512, 3072: 512, 4096: 2560, 6144: 2048}[p.Resources.MemoryMiB] - if memory == 0 { - return Reservation{CPUUnits: p.Resources.VCPU, MemoryMiB: p.Resources.MemoryMiB} - } - return Reservation{CPUUnits: 1, MemoryMiB: memory} + return Reservation{CPUUnits: p.Resources.VCPU, MemoryMiB: p.Resources.MemoryMiB} } type WarmPool struct { diff --git a/internal/incusplacement/scriptlet.go b/internal/incusplacement/scriptlet.go index ab0d546..70cd026 100644 --- a/internal/incusplacement/scriptlet.go +++ b/internal/incusplacement/scriptlet.go @@ -98,7 +98,7 @@ def committed_memory_bytes(member_name, pending_count): fail("fleet instance has non-MiB memory limit: " + instance.name) limit_mib = int(memory[:-3]) if limit_mib not in RESERVATION_BY_LIMIT_MIB: - fail("fleet instance has no measured memory reservation: " + instance.name) + fail("fleet instance has no memory reservation: " + instance.name) committed += RESERVATION_BY_LIMIT_MIB[limit_mib] * 1024 * 1024 pending_without_record = pending_count - len(instances) if pending_without_record < 0: diff --git a/internal/incusplacement/scriptlet_test.go b/internal/incusplacement/scriptlet_test.go index e9eefa2..827c966 100644 --- a/internal/incusplacement/scriptlet_test.go +++ b/internal/incusplacement/scriptlet_test.go @@ -19,8 +19,8 @@ func TestRenderCPUAwarePlacementFromPortablePolicy(t *testing.T) { for _, required := range []string{ `PROJECT = "gha-fleet"`, `POOL = "gha-lvm"`, "MINIMUM_MEMORY_BYTES = 1024 * 1024 * 1024", - "MINIMUM_MEMORY_PERCENT = 10", "MAX_WORKER_MEMORY_BYTES = 2560 * 1024 * 1024", - "2048: 512", "3072: 512", "4096: 2560", "6144: 2048", + "MINIMUM_MEMORY_PERCENT = 10", "MAX_WORKER_MEMORY_BYTES = 6144 * 1024 * 1024", + "2048: 2048", "3072: 3072", "4096: 4096", "6144: 6144", "LOAD_TIE_EPSILON = 0.05", "def load_per_core(state, resources)", `getattr(sysinfo, "load_averages", None)`, `getattr(cpu, "total", 0)`, "member_load < chosen_load - LOAD_TIE_EPSILON",