diff --git a/config/components/alerts/shell-sessions.yaml b/config/components/alerts/shell-sessions.yaml index ab0753c4..6467b9df 100644 --- a/config/components/alerts/shell-sessions.yaml +++ b/config/components/alerts/shell-sessions.yaml @@ -32,9 +32,9 @@ spec: for: 10m labels: severity: critical - service: compute-shell-sessions + service: compute namespace: compute-system - team: compute + team: build annotations: summary: Shell sessions in {{ $labels.location }} are ending with {{ $labels.reason }} description: | @@ -59,9 +59,9 @@ spec: for: 5m labels: severity: warning - service: compute-shell-sessions + service: compute namespace: compute-system - team: compute + team: build annotations: summary: Shell agent {{ $labels.agent }} on cell {{ $labels.cell }} cannot reach its endpoint description: | @@ -77,9 +77,9 @@ spec: for: 5m labels: severity: critical - service: compute-shell-sessions + service: compute namespace: compute-system - team: compute + team: build annotations: summary: No shell agent on cell {{ $labels.cell }} can reach its endpoint description: | @@ -102,9 +102,9 @@ spec: for: 15m labels: severity: warning - service: compute-shell-sessions + service: compute namespace: compute-system - team: compute + team: build annotations: summary: Shell sessions in {{ $labels.location }} take {{ $value | humanizeDuration }} to connect (p95) description: | @@ -125,9 +125,9 @@ spec: sum by (cell) (increase(compute_shell_agent_cleanup_unconfirmed_total[1h])) > 0 labels: severity: warning - service: compute-shell-sessions + service: compute namespace: compute-system - team: compute + team: build annotations: summary: Cell {{ $labels.cell }} did not confirm cleanup for {{ $value | humanize }} shell sessions description: | @@ -155,9 +155,9 @@ spec: for: 10m labels: severity: warning - service: compute-shell-sessions + service: compute namespace: compute-system - team: compute + team: build annotations: summary: Shell session creation in {{ $labels.location }} is far above its weekly baseline description: | diff --git a/test/alerts/shell-sessions.test.yaml b/test/alerts/shell-sessions.test.yaml new file mode 100644 index 00000000..04d11b1b --- /dev/null +++ b/test/alerts/shell-sessions.test.yaml @@ -0,0 +1,16 @@ +# Run with test/alerts/test-shell-sessions.sh (requires yq and promtool). +rule_files: +- shell-sessions.rules.yaml +evaluation_interval: 1m +tests: +- name: A session ending Unavailable pages compute and its team, build, as the platform's catalog names them + interval: 1m + input_series: + - series: compute_shell_sessions_ended_total{location="us-east-1",reason="Unavailable"} + values: 0+1x30 + promql_expr_test: + - expr: ALERTS{alertname="ComputeShellSessionsUnavailable",alertstate="firing"} + eval_time: 25m + exp_samples: + - labels: ALERTS{alertname="ComputeShellSessionsUnavailable",alertstate="firing",location="us-east-1",reason="Unavailable",severity="critical",service="compute",namespace="compute-system",team="build"} + value: 1 diff --git a/test/alerts/test-shell-sessions.sh b/test/alerts/test-shell-sessions.sh new file mode 100755 index 00000000..5309ee6d --- /dev/null +++ b/test/alerts/test-shell-sessions.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Extract the actual VMRule rather than maintaining a second copy of its expr. +root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +rules="$root/config/components/alerts/shell-sessions.yaml" +work="$(mktemp -d)" +trap 'rm -rf "$work"' EXIT + +# Every alert names the service and team the platform's catalog gives compute. +# An alert whose service the catalog does not list reaches no team, and a team +# the catalog does not list is counted against nobody. +owners="$(yq '[.spec.groups[].rules[] | select(.alert) | .labels.service + "/" + .labels.team] | unique | join(",")' "$rules")" +if [ "$owners" != "compute/build" ]; then + echo "shell-session alerts name service/team '$owners'; want 'compute/build', the catalog's row" >&2 + exit 1 +fi + +yq '.spec' "$rules" > "$work/shell-sessions.rules.yaml" +cp "$root/test/alerts/shell-sessions.test.yaml" "$work/test.yaml" +cd "$work" +promtool check rules shell-sessions.rules.yaml +promtool test rules test.yaml