fix(queue): prevent terminal job resurrection

config/example-runner-1.yaml

@@ -6,10 +6,10 @@ platform:
 
 control_plane:
   manager: garm
-  manager_version: v0.2.1-nddev.81
+  manager_version: v0.2.1-nddev.82
   scheduling_mode: scale-set
   provider: incus
-  provider_version: v0.1.5-nddev.78
+  provider_version: v0.1.5-nddev.79
   provider_interface: v0.1.0
   worker_kind: incus-container
   runner: actions/runner

config/example-runner-2.yaml

@@ -6,10 +6,10 @@ platform:
 
 control_plane:
   manager: garm
-  manager_version: v0.2.1-nddev.81
+  manager_version: v0.2.1-nddev.82
   scheduling_mode: scale-set
   provider: incus
-  provider_version: v0.1.5-nddev.78
+  provider_version: v0.1.5-nddev.79
   provider_interface: v0.1.0
   worker_kind: incus-container
   runner: actions/runner

config/example-runner-3.yaml

@@ -6,10 +6,10 @@ platform:
 
 control_plane:
   manager: garm
-  manager_version: v0.2.1-nddev.81
+  manager_version: v0.2.1-nddev.82
   scheduling_mode: scale-set
   provider: incus
-  provider_version: v0.1.5-nddev.78
+  provider_version: v0.1.5-nddev.79
   provider_interface: v0.1.0
   worker_kind: incus-container
   runner: actions/runner

config/example-runner-4.yaml

@@ -6,10 +6,10 @@ platform:
 
 control_plane:
   manager: garm
-  manager_version: v0.2.1-nddev.81
+  manager_version: v0.2.1-nddev.82
   scheduling_mode: scale-set
   provider: incus
-  provider_version: v0.1.5-nddev.78
+  provider_version: v0.1.5-nddev.79
   provider_interface: v0.1.0
   worker_kind: incus-container
   runner: actions/runner

config/example-services.yaml

@@ -24,10 +24,10 @@ platform:
 
 control_plane:
   manager: garm
-  manager_version: v0.2.1-nddev.81
+  manager_version: v0.2.1-nddev.82
   scheduling_mode: scale-set
   provider: incus
-  provider_version: v0.1.5-nddev.78
+  provider_version: v0.1.5-nddev.79
   provider_interface: v0.1.0
   worker_kind: incus-container
   runner: actions/runner

config/garm-derivative.yaml

@@ -1,6 +1,6 @@
 schema_version: 1
 artifact: garm
-derivative_version: v0.2.1-nddev.81
+derivative_version: v0.2.1-nddev.82
 upstream:
   repository: https://github.com/cloudbase/garm
   release: v0.2.1
@@ -90,11 +90,11 @@ patches:
     purpose: Bound every upstream Linux and Gitea bootstrap curl path to two retries after the initial attempt so GARM and the provider enforce the same three-total-attempt contract.
 overlays:
   - path: third_party/garm/overlay/workers/scaleset/queue_intent.go
-    sha256: 0793ef4a19052849f265dfe40a06a8f3330bef053ea2fff84e080b269a85f7b7
-    purpose: Fsync-backed central queue admission where disabled scale sets cannot record or acquire work, sparse JobAssigned owns a bounded provisional token without refreshing its TTL, JobAvailable or the authoritative queued DB row binds repository identity, authoritative JobStarted rehydrates missing running ownership, and state_entered_at remains stable across same-phase lease refreshes.
+    sha256: 180e60554d264358d5a71c3b16b90d897a82cdb51a7b3d70b587029f97e5e3fe
+    purpose: Fsync-backed central queue admission where bounded terminal tombstones prevent delayed JobAssigned redelivery from resurrecting completed work, while provisional capacity, repository binding, running rehydration and stable phase-entry semantics remain intact.
   - path: third_party/garm/overlay/workers/scaleset/queue_intent_test.go
-    sha256: 02449e9feaf09de10f225323053b1efbc75e810a0f7bba5113b9b27fb1856c65
-    purpose: Deterministic disabled-scale-set isolation, provisional bootstrap, authoritative repository enrichment and rebinding refusal, one-time legacy horizon migration, non-refreshing assigned expiry, orphan-start rehydration, phase-entry and expiry, priority, fairness, width, idempotency, acknowledgement, crash expiry and concurrent-selection tests.
+    sha256: 2f7d98f63033d73d972a9e91ca7cca0934e09abbc62fa971b6ed99e328f531b9
+    purpose: Deterministic terminal-redelivery suppression, disabled-scale-set isolation, provisional bootstrap, authoritative repository enrichment, orphan-start rehydration, phase-entry and expiry, priority, fairness, width, idempotency, acknowledgement and concurrent-selection tests.
   - path: third_party/garm/overlay/workers/provider/nddev_create_retry.go
     sha256: 611d67a74ff626f04cc0240409a22848525a06c2ae4dc7180c4b75001fb4b599
     purpose: Fsync-backed schema-v2 instance-to-intent reservations and per-job attempt leases, pruning inactive terminal jobs while retaining active state, and selecting the oldest eligible exact intent so one terminal or deferred job cannot block newer work in the same scale set.
@@ -115,9 +115,9 @@ build:
     - sqlite_omit_load_extension
   reproducible_rebuilds: 2
   maximum_required_glibc: "2.34"
-  binary_sha256: 118b1ee61acb7e2737f8e2c8c907d6498881d710251a9431a8878358352d2419
+  binary_sha256: 18089d33be778705d0a5b20a87f5b1ddea06b0a7bc9daab883a8014b0507d741
 runtime_contract:
-  queue_intent_schema_version: 4
+  queue_intent_schema_version: 5
   event_driven_scale_set_wake: true
   event_driven_instance_wake: true
   startup_states_protected_from_scale_down: true

config/provider-derivative.yaml

@@ -16,7 +16,7 @@ artifact: garm-provider-incus
 # state all move together, because all three derive from here. A provider change
 # that does not bump it ships under the previous version, which is exactly how
 # runner-1 and runner-2 diverged.
-derivative_version: v0.1.5-nddev.78
+derivative_version: v0.1.5-nddev.79
 
 # The external-provider protocol GARM speaks to this binary. It moves on its own
 # schedule -- a provider release does not imply an interface release -- so it is
@@ -34,11 +34,11 @@ runtime:
   # Incus API the fleet can speak, so it belongs to the release identity rather
   # than to the code that happens to import it.
   incus_sdk_version: v7.3.0
-  queue_intent_schema_version: 4
+  queue_intent_schema_version: 5
 
 build:
-  source_commit: f8a91e798f58ca71bcf0204854d387a228262a5c
-  binary_sha256: 204e3e1a4a4e9b9beeb3516fb8921e58fe9b2f928ed70404a8bdd4497ee0ade2
+  source_commit: 2ce03cd8ca00e6f1f08a4d32ca08e291f5bc1f79
+  binary_sha256: 9a7f38b3685e3e73fb5d8bcc7a3af1e3070827204ced4de469f573d22005740c
   go_version: go1.26.6
   cgo_enabled: false
   target_os: linux

internal/queueintent/absent_journal_test.go

@@ -53,7 +53,7 @@ func TestPresentButUnusableJournalStillFails(t *testing.T) {
 	}{
 		{name: "group readable", contents: `{"schema_version":1,"intents":{},"repositories":{}}`, mode: 0o640, wantError: "private regular file"},
 		{name: "malformed", contents: `{`, mode: 0o600, wantError: "decode queue-intent journal"},
-		{name: "wrong schema", contents: `{"schema_version":5,"intents":{},"repositories":{}}`, mode: 0o600, wantError: "schema_version"},
+		{name: "wrong schema", contents: `{"schema_version":99,"intents":{},"repositories":{}}`, mode: 0o600, wantError: "schema_version"},
 		{name: "null maps", contents: `{"schema_version":1,"intents":null,"repositories":null}`, mode: 0o600, wantError: "must not be null"},
 	} {
 		t.Run(testCase.name, func(t *testing.T) {

internal/queueintent/queueintent.go

@@ -21,8 +21,8 @@ import (
 
 const (
 	LegacySchemaVersion   = 1
-	PreviousSchemaVersion = 3
-	SchemaVersion         = 4
+	PreviousSchemaVersion = 4
+	SchemaVersion         = 5
 	maxJournalBytes       = 4 * 1024 * 1024
 )
 
@@ -84,6 +84,7 @@ type Journal struct {
 	UpdatedAt     time.Time                  `json:"updated_at"`
 	Intents       map[string]Intent          `json:"intents"`
 	Repositories  map[string]RepositoryState `json:"repositories"`
+	TerminalJobs  map[string]time.Time       `json:"terminal_jobs"`
 }
 
 type Snapshot struct {
@@ -235,6 +236,7 @@ func readJournal(path string) (Journal, error) {
 				SchemaVersion: SchemaVersion,
 				Intents:       map[string]Intent{},
 				Repositories:  map[string]RepositoryState{},
+				TerminalJobs:  map[string]time.Time{},
 			}, nil
 		}
 		return Journal{}, fmt.Errorf("open queue-intent journal: %w", err)
@@ -270,12 +272,17 @@ func readJournal(path string) (Journal, error) {
 	}
 	switch journal.SchemaVersion {
 	case LegacySchemaVersion, 2:
+		journal.TerminalJobs = make(map[string]time.Time)
 		journal.SchemaVersion = SchemaVersion
 		for key, intent := range journal.Intents {
 			intent.StateEnteredAt = intent.UpdatedAt
 			journal.Intents[key] = intent
 		}
+	case 3:
+		journal.TerminalJobs = make(map[string]time.Time)
+		journal.SchemaVersion = SchemaVersion
 	case PreviousSchemaVersion:
+		journal.TerminalJobs = make(map[string]time.Time)
 		journal.SchemaVersion = SchemaVersion
 	case SchemaVersion:
 	default:
@@ -291,7 +298,7 @@ func (j Journal) Validate() error {
 	if j.SchemaVersion != SchemaVersion {
 		return fmt.Errorf("queue-intent journal schema_version must be %d", SchemaVersion)
 	}
-	if j.Intents == nil || j.Repositories == nil {
+	if j.Intents == nil || j.Repositories == nil || j.TerminalJobs == nil {
 		return fmt.Errorf("queue-intent journal maps must not be null")
 	}
 	for key, intent := range j.Intents {
@@ -322,6 +329,11 @@ func (j Journal) Validate() error {
 			return fmt.Errorf("repository scheduler state %q is invalid", key)
 		}
 	}
+	for jobID, expiry := range j.TerminalJobs {
+		if !validText(jobID) || expiry.IsZero() {
+			return fmt.Errorf("terminal queue job %q is invalid", jobID)
+		}
+	}
 	return nil
 }
 

internal/queueintent/queueintent_test.go

@@ -43,6 +43,7 @@ func TestReaderReturnsOnlyActiveIntentsInDeterministicOrder(t *testing.T) {
       "updated_at": "2026-08-09T13:50:00Z", "expires_at": "2026-08-09T13:59:59Z"
     }
   },
+  "terminal_jobs": {},
   "repositories": {
     "owner/standard": {"repository": "owner/standard", "weight": 1, "pass": 10},
     "owner/release": {"repository": "owner/release", "weight": 2, "pass": 5}
@@ -105,6 +106,7 @@ func TestReaderAuthorizesAssignedUUIDWithoutRunnerRequestID(t *testing.T) {
       "expires_at": "2026-08-10T15:20:46Z"
     }
   },
+  "terminal_jobs": {},
   "repositories": {
     "example-user/github-actions": {"repository": "example-user/github-actions", "weight": 1, "pass": 1000000}
   }

internal/queueintent/recovery_linux_test.go

@@ -28,6 +28,7 @@ func TestRecoverUnboundRunningUsesExactCASAndGeneration(t *testing.T) {
 			StateEnteredAt: updatedAt, UpdatedAt: updatedAt, ExpiresAt: updatedAt.Add(24 * time.Hour),
 		}},
 		Repositories: map[string]RepositoryState{"example-org": {Repository: "example-org", Weight: 1}},
+		TerminalJobs: map[string]time.Time{},
 	}
 	content, err := json.Marshal(journal)
 	if err != nil {
@@ -69,7 +70,7 @@ func TestRecoverUnboundRunningRejectsChangedPrecondition(t *testing.T) {
 	lockPath := filepath.Join(directory, "queue-intents.lock")
 	updatedAt := time.Now().UTC().Add(-time.Hour)
 	key := intentKey(1, "bound-job")
-	journal := Journal{SchemaVersion: SchemaVersion, Intents: map[string]Intent{key: {
+	journal := Journal{SchemaVersion: SchemaVersion, TerminalJobs: map[string]time.Time{}, Intents: map[string]Intent{key: {
 		Key: key, ScaleSetID: 1, JobID: "bound-job", RunnerRequestID: 42,
 		ScaleSetName: "nddev-linux-standard", Owner: "example-org", Repository: "example-org/repository",
 		WorkflowRef: "workflow.yml@refs/heads/main", EventName: "push", QueueTime: updatedAt,
@@ -96,7 +97,7 @@ func TestRecoverCanceledUnboundAcceptsOnlySparseQueuedOrAssigned(t *testing.T) {
 			lockPath := filepath.Join(directory, "queue-intents.lock")
 			updatedAt := time.Now().UTC().Add(-time.Hour)
 			key := intentKey(6, "canceled-job")
-			journal := Journal{SchemaVersion: SchemaVersion, Intents: map[string]Intent{key: {
+			journal := Journal{SchemaVersion: SchemaVersion, TerminalJobs: map[string]time.Time{}, Intents: map[string]Intent{key: {
 				Key: key, ScaleSetID: 6, JobID: "canceled-job", ScaleSetName: "nddev-linux-untrusted",
 				Owner: "example-org", Repository: "example-org",
 				WorkflowRef: "unavailable-before-job-available", EventName: "unavailable-before-job-available",

internal/queueintent/schema_evolution_test.go

@@ -43,7 +43,7 @@ func jsonFieldNames(t *testing.T, value any) []string {
 // that decision -- internal/providerjournal already carries a worked example of
 // a version ladder that upgrades in memory.
 func TestQueueIntentWireShapeIsPinnedToItsSchemaVersion(t *testing.T) {
-	if SchemaVersion != 4 {
+	if SchemaVersion != 5 {
 		t.Fatalf("SchemaVersion = %d; update the golden field sets below with it", SchemaVersion)
 	}
 	for _, testCase := range []struct {
@@ -55,7 +55,7 @@ func TestQueueIntentWireShapeIsPinnedToItsSchemaVersion(t *testing.T) {
 			name:  "Journal",
 			value: Journal{},
 			wanted: []string{
-				"schema_version", "generation", "updated_at", "intents", "repositories",
+				"schema_version", "generation", "updated_at", "intents", "repositories", "terminal_jobs",
 			},
 		},
 		{
@@ -119,6 +119,7 @@ func TestUnknownFieldRejectsTheWholeJournalRatherThanOneIntent(t *testing.T) {
 				"repository": "example-org/example-actions", "weight": 1, "pass": 0,
 			},
 		},
+		"terminal_jobs": map[string]any{},
 	}
 	raw, err := json.Marshal(journal)
 	if err != nil {

internal/queueintent/tenant_scope_test.go

@@ -29,6 +29,7 @@ func TestActiveForScaleSetSeparatesAccountsSharingAClassName(t *testing.T) {
       "expires_at": "2026-08-12T14:10:00Z"
     }
   },
+  "terminal_jobs": {},
   "repositories": {
     "owner/attention": {"repository": "owner/attention", "weight": 1, "pass": 1}
   }
@@ -56,7 +57,8 @@ func TestRepositoryForScaleSetNarrowsOrganizationIntent(t *testing.T) {
     "repository":"example-org/example-actions","workflow_ref":"x","event_name":"workflow_dispatch",
     "queue_time":"2026-01-01T00:00:00Z","state":"acquired","priority":1,
     "updated_at":"2026-01-01T00:00:00Z","expires_at":"2099-01-01T00:00:00Z"}},
-  "repositories":{"example-org":{"repository":"example-org","weight":1,"pass":1}}
+  "repositories":{"example-org":{"repository":"example-org","weight":1,"pass":1}},
+  "terminal_jobs":{}
 }`)}
 	repository, err := reader.RepositoryForScaleSet(context.Background(), "example-org", "nddev-linux-standard")
 	if err != nil {

scripts/build-garm-nddev.sh

@@ -19,7 +19,7 @@ set -Eeuo pipefail
 # Every value below is the manifest's. Editing one here detaches the build
 # from the provenance it is reviewed against, which is why the region is
 # regenerated and compared rather than maintained.
-readonly derivative_version="v0.2.1-nddev.81"
+readonly derivative_version="v0.2.1-nddev.82"
 readonly upstream_repository="https://github.com/cloudbase/garm"
 readonly upstream_commit="154638445c3949c1958b01812f69d9a1e4d82684"
 readonly build_image="docker.io/library/golang@sha256:116d58cbd88c1297624acc6e967a060012422bacf9930927e23fb719189c6f36"
@@ -32,7 +32,7 @@ readonly build_module_mode="vendor"
 readonly build_tags="osusergo,netgo,sqlite_omit_load_extension"
 readonly build_reproducible_rebuilds="2"
 readonly build_maximum_required_glibc="2.34"
-readonly expected_binary_sha256="118b1ee61acb7e2737f8e2c8c907d6498881d710251a9431a8878358352d2419"
+readonly expected_binary_sha256="18089d33be778705d0a5b20a87f5b1ddea06b0a7bc9daab883a8014b0507d741"
 readonly patch_paths=(
   "third_party/garm/patches/0001-event-driven-reconciliation.patch"
   "third_party/garm/patches/0002-central-queue-admission.patch"
@@ -98,8 +98,8 @@ readonly overlay_paths=(
   "third_party/garm/overlay/workers/provider/nddev_create_retry_test.go"
 )
 readonly overlay_sha256s=(
-  "0793ef4a19052849f265dfe40a06a8f3330bef053ea2fff84e080b269a85f7b7"
-  "02449e9feaf09de10f225323053b1efbc75e810a0f7bba5113b9b27fb1856c65"
+  "180e60554d264358d5a71c3b16b90d897a82cdb51a7b3d70b587029f97e5e3fe"
+  "2f7d98f63033d73d972a9e91ca7cca0934e09abbc62fa971b6ed99e328f531b9"
   "611d67a74ff626f04cc0240409a22848525a06c2ae4dc7180c4b75001fb4b599"
   "9379d69b8a40295e176fdfb0587b299fabda69de9708e63672c41546fd9a3508"
 )