Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,10 @@ Versioning.

### Fixed

- Convert second-based alert evaluation contracts to OpenObserve's supported
minute cadence, recompute consecutive-evaluation thresholds from the rendered
schedule, and keep the lookback period no shorter than that cadence so short
OOM and lifecycle incidents cannot disappear between evaluations.
- Provision the standard runner cache directory with runner ownership before
each job, preventing setup actions from exporting an unwritable home cache.
- Refresh the grouped provider dependency set to smithy-go 1.27.9 and
Expand Down
5 changes: 3 additions & 2 deletions internal/observabilityrules/openobserve.go
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,8 @@ func RenderOpenObserve(bundle Bundle, destination string, enable bool) (OpenObse
priority = 1
silence = 10
}
periodMinutes := (rule.HoldSecs + 59) / 60
frequencyMinutes := (rule.EvaluationSecs + 59) / 60
periodMinutes := max((rule.HoldSecs+59)/60, frequencyMinutes)
result.Alerts = append(result.Alerts, OpenObserveAlert{
Name: rule.ID,
OrgID: bundle.Organization,
Expand All @@ -91,7 +92,7 @@ func RenderOpenObserve(bundle Bundle, destination string, enable bool) (OpenObse
},
TriggerCondition: OpenObserveTrigger{
Period: periodMinutes, Operator: ">=", Threshold: rule.RequiredEvaluations(),
Frequency: rule.EvaluationSecs, FrequencyType: "minutes", Silence: silence,
Frequency: frequencyMinutes, FrequencyType: "minutes", Silence: silence,
Timezone: "UTC", AlignTime: true,
},
Destinations: []string{destination},
Expand Down
3 changes: 2 additions & 1 deletion internal/observabilityrules/rules.go
Original file line number Diff line number Diff line change
Expand Up @@ -119,5 +119,6 @@ func (r Rule) Validate() error {
}

func (r Rule) RequiredEvaluations() int {
return (r.HoldSecs + r.EvaluationSecs - 1) / r.EvaluationSecs
frequencySecs := ((r.EvaluationSecs + 59) / 60) * 60
return (r.HoldSecs + frequencySecs - 1) / frequencySecs
}
35 changes: 35 additions & 0 deletions internal/observabilityrules/rules_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -88,3 +88,38 @@ func TestRenderOpenObserveSeparatesExpressionAndThreshold(t *testing.T) {
}
}
}

func TestRenderOpenObserveConvertsSecondsToMinuteSchedule(t *testing.T) {
for _, test := range []struct {
name string
severity string
evaluationSecs int
holdSecs int
frequencyMinutes int
periodMinutes int
evaluations int
}{
{name: "thirty seconds", severity: "page", evaluationSecs: 30, holdSecs: 120, frequencyMinutes: 1, periodMinutes: 2, evaluations: 2},
{name: "sixty seconds", severity: "page", evaluationSecs: 60, holdSecs: 60, frequencyMinutes: 1, periodMinutes: 1, evaluations: 1},
{name: "five minutes", severity: "ticket", evaluationSecs: 300, holdSecs: 900, frequencyMinutes: 5, periodMinutes: 15, evaluations: 3},
} {
t.Run(test.name, func(t *testing.T) {
bundle := Bundle{SchemaVersion: SchemaVersion, Backend: "openobserve", Organization: "default", Rules: []Rule{{
ID: "example_rule", Severity: test.severity, QueryLanguage: "promql", StreamName: "example_metric",
Expression: "max(example_metric)", Operator: ">", Threshold: 0,
EvaluationSecs: test.evaluationSecs, HoldSecs: test.holdSecs, DestinationRef: "fleet_oncall",
Owner: "fleet-operations", Runbook: "https://github.com/NDDev-OpenNetwork/github-actions/blob/main/docs/runbooks/fleet-alerts.md",
Summary: "Example summary.", Action: "Example action.", Recovery: "Example recovery.",
}}}
rendered, err := RenderOpenObserve(bundle, "fleet_oncall", true)
if err != nil {
t.Fatal(err)
}
trigger := rendered.Alerts[0].TriggerCondition
if trigger.FrequencyType != "minutes" || trigger.Frequency != test.frequencyMinutes ||
trigger.Period != test.periodMinutes || trigger.Threshold != test.evaluations {
t.Fatalf("trigger = %#v", trigger)
}
})
}
}