diff --git a/config/observability-rules.yaml b/config/observability-rules.yaml index a4d7513..e7f9f01 100644 --- a/config/observability-rules.yaml +++ b/config/observability-rules.yaml @@ -130,6 +130,22 @@ rules: summary: Queue, provider journal and Incus inventory disagree. action: Preserve all three states and recover only from exact authoritative identity. recovery: Orphan, missing and uncovered-running metrics remain zero. + - id: lifecycle_queued_delivery_stall + severity: page + query_language: promql + stream_name: gha_fleet_queue_intent_oldest_state_age_seconds + expression: max(gha_fleet_queue_intent_oldest_state_age_seconds{state="queued"}) + operator: ">" + threshold: 300 + evaluation_seconds: 60 + hold_seconds: 300 + destination_ref: fleet_oncall + enabled: false + owner: fleet-operations + runbook: https://github.com/NDDev-OpenNetwork/github-actions/blob/main/docs/runbooks/fleet-alerts.md + summary: An eligible queued job has not produced runner delivery progress within five minutes. + action: Preserve the job and correlate queue intent, provider retry, admission, worker creation and GitHub registration before recovery. + recovery: Queued age remains below five minutes or the exact job advances without operator cancellation. - id: memory_psi_slow_burn severity: ticket query_language: promql diff --git a/internal/observabilityrules/rules_test.go b/internal/observabilityrules/rules_test.go index 08ee8d3..aac9a5d 100644 --- a/internal/observabilityrules/rules_test.go +++ b/internal/observabilityrules/rules_test.go @@ -10,8 +10,8 @@ func TestRepositoryBundleIsValid(t *testing.T) { if err != nil { t.Fatal(err) } - if len(bundle.Rules) != 13 { - t.Fatalf("rules = %d, want 13", len(bundle.Rules)) + if len(bundle.Rules) != 14 { + t.Fatalf("rules = %d, want 14", len(bundle.Rules)) } } @@ -22,6 +22,7 @@ func TestRepositoryRulesUseCurrentMetricSemantics(t *testing.T) { } wanted := map[string]string{ "github_correlation_persistent": "gha_fleet_queue_missing_workflow_run_id_beyond_grace", + "lifecycle_queued_delivery_stall": `gha_fleet_queue_intent_oldest_state_age_seconds{state="queued"}`, "memory_psi_slow_burn": `window_seconds="10"`, "queue_wait_slow_burn": `gha_fleet_queue_intent_oldest_state_age_seconds{state="queued"}`, "provider_retry_error_persistent": `gha_fleet_provider_retry_deferred_records_by_error_class{error_class=~"identity|intent|provider|timeout|unknown"}`,