From db23cc0e4f81be997412ec28fbf62474a175562b Mon Sep 17 00:00:00 2001 From: rldyourmnd Date: Wed, 26 Aug 2026 01:36:30 +0500 Subject: [PATCH] fix(observe): render alert cadence in minutes --- CHANGELOG.md | 4 +++ internal/observabilityrules/openobserve.go | 5 ++-- internal/observabilityrules/rules.go | 3 +- internal/observabilityrules/rules_test.go | 35 ++++++++++++++++++++++ 4 files changed, 44 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ee1f206..d85eeea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -71,6 +71,10 @@ Versioning. ### Fixed +- Convert second-based alert evaluation contracts to OpenObserve's supported + minute cadence, recompute consecutive-evaluation thresholds from the rendered + schedule, and keep the lookback period no shorter than that cadence so short + OOM and lifecycle incidents cannot disappear between evaluations. - Provision the standard runner cache directory with runner ownership before each job, preventing setup actions from exporting an unwritable home cache. - Refresh the grouped provider dependency set to smithy-go 1.27.9 and diff --git a/internal/observabilityrules/openobserve.go b/internal/observabilityrules/openobserve.go index 9387064..02aaf45 100644 --- a/internal/observabilityrules/openobserve.go +++ b/internal/observabilityrules/openobserve.go @@ -76,7 +76,8 @@ func RenderOpenObserve(bundle Bundle, destination string, enable bool) (OpenObse priority = 1 silence = 10 } - periodMinutes := (rule.HoldSecs + 59) / 60 + frequencyMinutes := (rule.EvaluationSecs + 59) / 60 + periodMinutes := max((rule.HoldSecs+59)/60, frequencyMinutes) result.Alerts = append(result.Alerts, OpenObserveAlert{ Name: rule.ID, OrgID: bundle.Organization, @@ -91,7 +92,7 @@ func RenderOpenObserve(bundle Bundle, destination string, enable bool) (OpenObse }, TriggerCondition: OpenObserveTrigger{ Period: periodMinutes, Operator: ">=", Threshold: rule.RequiredEvaluations(), - Frequency: rule.EvaluationSecs, FrequencyType: "minutes", Silence: silence, + Frequency: frequencyMinutes, FrequencyType: "minutes", Silence: silence, Timezone: "UTC", AlignTime: true, }, Destinations: []string{destination}, diff --git a/internal/observabilityrules/rules.go b/internal/observabilityrules/rules.go index 0c657d7..7721ce7 100644 --- a/internal/observabilityrules/rules.go +++ b/internal/observabilityrules/rules.go @@ -119,5 +119,6 @@ func (r Rule) Validate() error { } func (r Rule) RequiredEvaluations() int { - return (r.HoldSecs + r.EvaluationSecs - 1) / r.EvaluationSecs + frequencySecs := ((r.EvaluationSecs + 59) / 60) * 60 + return (r.HoldSecs + frequencySecs - 1) / frequencySecs } diff --git a/internal/observabilityrules/rules_test.go b/internal/observabilityrules/rules_test.go index f5f4f95..08ee8d3 100644 --- a/internal/observabilityrules/rules_test.go +++ b/internal/observabilityrules/rules_test.go @@ -88,3 +88,38 @@ func TestRenderOpenObserveSeparatesExpressionAndThreshold(t *testing.T) { } } } + +func TestRenderOpenObserveConvertsSecondsToMinuteSchedule(t *testing.T) { + for _, test := range []struct { + name string + severity string + evaluationSecs int + holdSecs int + frequencyMinutes int + periodMinutes int + evaluations int + }{ + {name: "thirty seconds", severity: "page", evaluationSecs: 30, holdSecs: 120, frequencyMinutes: 1, periodMinutes: 2, evaluations: 2}, + {name: "sixty seconds", severity: "page", evaluationSecs: 60, holdSecs: 60, frequencyMinutes: 1, periodMinutes: 1, evaluations: 1}, + {name: "five minutes", severity: "ticket", evaluationSecs: 300, holdSecs: 900, frequencyMinutes: 5, periodMinutes: 15, evaluations: 3}, + } { + t.Run(test.name, func(t *testing.T) { + bundle := Bundle{SchemaVersion: SchemaVersion, Backend: "openobserve", Organization: "default", Rules: []Rule{{ + ID: "example_rule", Severity: test.severity, QueryLanguage: "promql", StreamName: "example_metric", + Expression: "max(example_metric)", Operator: ">", Threshold: 0, + EvaluationSecs: test.evaluationSecs, HoldSecs: test.holdSecs, DestinationRef: "fleet_oncall", + Owner: "fleet-operations", Runbook: "https://github.com/NDDev-OpenNetwork/github-actions/blob/main/docs/runbooks/fleet-alerts.md", + Summary: "Example summary.", Action: "Example action.", Recovery: "Example recovery.", + }}} + rendered, err := RenderOpenObserve(bundle, "fleet_oncall", true) + if err != nil { + t.Fatal(err) + } + trigger := rendered.Alerts[0].TriggerCondition + if trigger.FrequencyType != "minutes" || trigger.Frequency != test.frequencyMinutes || + trigger.Period != test.periodMinutes || trigger.Threshold != test.evaluations { + t.Fatalf("trigger = %#v", trigger) + } + }) + } +}