From 5e0452d5ad5c2336ab1386632dfafe7c4e0f54a2 Mon Sep 17 00:00:00 2001 From: Buck Doyle Date: Thu, 8 Oct 2026 03:25:05 -0500 Subject: [PATCH 1/2] Alert in Discord when the Actions collector goes silent in Loki Add a staging alert rule that fires when Loki receives no lines from the GitHub Actions queue collector for ten minutes. It posts to the alarms-staging-internal Discord channel through a new contact point. apply-alerting.sh now also pushes provisioning/alerting-hosted//: contact points first, then rule groups. The local stack does not mount that directory. The contact point gets its webhook URL from SSM, through the staging apply workflow, so the URL is not in this repository. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01QZtQde6na3jLiKVjAavLPz --- .../workflows/observability-apply-staging.yml | 2 + packages/observability/README.md | 15 ++- .../contact-points/discord-alarms.json | 20 ++++ .../rules/actions-collector-group.json | 100 ++++++++++++++++++ .../observability/scripts/apply-alerting.sh | 89 ++++++++++++++-- 5 files changed, 217 insertions(+), 9 deletions(-) create mode 100644 packages/observability/provisioning/alerting-hosted/staging/contact-points/discord-alarms.json create mode 100644 packages/observability/provisioning/alerting-hosted/staging/rules/actions-collector-group.json diff --git a/.github/workflows/observability-apply-staging.yml b/.github/workflows/observability-apply-staging.yml index ee81a6bc9ea..6462bf58f10 100644 --- a/.github/workflows/observability-apply-staging.yml +++ b/.github/workflows/observability-apply-staging.yml @@ -121,6 +121,8 @@ jobs: fetch BOXEL_DB_USER /staging/boxel/GRAFANA_DB_USER fetch BOXEL_DB_PASSWORD /staging/boxel/GRAFANA_DB_PASSWORD --with-decryption fetch GRAFANA_SECRET /staging/boxel/GRAFANA_SECRET --with-decryption + # provisioning/alerting-hosted/staging/contact-points/discord-alarms.json + fetch DISCORD_ALARMS_WEBHOOK /staging/boxel-grafana/discord_alarms_webhook --with-decryption - name: Lint # Belt-and-suspenders: ci-lint.yaml runs the same lint at PR time diff --git a/packages/observability/README.md b/packages/observability/README.md index 2dabdbbae4c..f5df7f36580 100644 --- a/packages/observability/README.md +++ b/packages/observability/README.md @@ -38,8 +38,13 @@ grafanactl/ dashboards/ # grafanactl push: dashboard JSON, organized by folder provisioning/ # mounted into Grafana at /etc/grafana/provisioning/ datasources/ # data sources (Loki, Postgres, CloudWatch, Prometheus) - alerting/ # alert rule groups (no contact points / notification - # policies live here — routing uses Grafana's defaults) + alerting/ # alert rule groups for every env; routing uses + # Grafana's default notification policy + alerting-hosted/ # NOT mounted locally. /contact-points/ and + # /rules/ exist in that hosted env only; + # apply-alerting.sh pushes contact points first, with + # ${VAR} secrets from SSM. A rule here may route to + # one through `notification_settings.receiver`. local-only/ # local-dev overrides — bind-mounted file-by-file over # `datasources/`. apply-datasources.sh ignores this dir. collectors/ @@ -147,6 +152,12 @@ public — a fine-grained token scoped to _Public repositories (read-only)_, or even a classic token with no scopes ticked, is enough. Were the repository ever made private, it would need Actions: Read-only on it. +The staging alert "Actions Collector Silent in Loki" +(`provisioning/alerting-hosted/staging/rules/actions-collector-group.json`) +posts to the `alarms-staging-internal` Discord channel when Loki receives no +collector lines for ten minutes. Its summary says how to tell a stalled log +router from a stopped collector. + Build and deploy with the **Manual Deploy [actions-collector]** workflow. It is `workflow_dispatch` only — putting the deploy of a queue sampler on a CI trigger would make it queue behind the backlog it reports on. diff --git a/packages/observability/provisioning/alerting-hosted/staging/contact-points/discord-alarms.json b/packages/observability/provisioning/alerting-hosted/staging/contact-points/discord-alarms.json new file mode 100644 index 00000000000..8f84dd78eca --- /dev/null +++ b/packages/observability/provisioning/alerting-hosted/staging/contact-points/discord-alarms.json @@ -0,0 +1,20 @@ +{ + "apiVersion": 1, + "contactPoints": [ + { + "orgId": 1, + "name": "Discord alarms-staging-internal", + "receivers": [ + { + "uid": "discord-alarms-staging-internal", + "type": "discord", + "disableResolveMessage": false, + "settings": { + "url": "${DISCORD_ALARMS_WEBHOOK}", + "use_discord_username": false + } + } + ] + } + ] +} diff --git a/packages/observability/provisioning/alerting-hosted/staging/rules/actions-collector-group.json b/packages/observability/provisioning/alerting-hosted/staging/rules/actions-collector-group.json new file mode 100644 index 00000000000..59b645ac349 --- /dev/null +++ b/packages/observability/provisioning/alerting-hosted/staging/rules/actions-collector-group.json @@ -0,0 +1,100 @@ +{ + "apiVersion": 1, + "groups": [ + { + "orgId": 1, + "name": "actions-collector-group", + "folder": "defd2d156sav4d", + "interval": "60s", + "rules": [ + { + "uid": "actions-collector-silent", + "title": "Actions Collector Silent in Loki", + "condition": "C", + "data": [ + { + "refId": "A", + "queryType": "instant", + "relativeTimeRange": { + "from": 300, + "to": 0 + }, + "datasourceUid": "loki", + "model": { + "datasource": { + "type": "loki", + "uid": "loki" + }, + "editorMode": "code", + "expr": "sum(count_over_time({service=\"actions-collector\", env=\"staging\"}[5m]))", + "intervalMs": 1000, + "maxDataPoints": 43200, + "queryType": "instant", + "refId": "A" + } + }, + { + "refId": "B", + "queryType": "", + "relativeTimeRange": { + "from": 0, + "to": 0 + }, + "datasourceUid": "__expr__", + "model": { + "datasource": { + "type": "__expr__", + "uid": "__expr__" + }, + "expression": "A", + "intervalMs": 1000, + "maxDataPoints": 43200, + "reducer": "last", + "refId": "B", + "type": "reduce" + } + }, + { + "refId": "C", + "queryType": "", + "relativeTimeRange": { + "from": 0, + "to": 0 + }, + "datasourceUid": "__expr__", + "model": { + "conditions": [ + { + "evaluator": { + "params": [1], + "type": "lt" + } + } + ], + "datasource": { + "type": "__expr__", + "uid": "__expr__" + }, + "expression": "B", + "intervalMs": 1000, + "maxDataPoints": 43200, + "refId": "C", + "type": "threshold" + } + } + ], + "noDataState": "Alerting", + "execErrState": "Alerting", + "for": "5m", + "annotations": { + "summary": "Loki has received no lines from the GitHub Actions queue collector for at least 10 minutes, so the GitHub Actions Queue dashboard shows no data. The collector writes at least one line a minute. A stalled query returns no series rather than zero, so noDataState is Alerting. Compare with CloudWatch group ecs-boxel-actions-collector-staging: if lines still arrive there, the FireLens log router has stopped sending to Loki, and `aws ecs update-service --cluster staging --service boxel-actions-collector-staging --force-new-deployment` restores it. If CloudWatch is silent too, the collector itself has stopped." + }, + "isPaused": false, + "notification_settings": { + "receiver": "Discord alarms-staging-internal" + } + } + ] + } + ] +} diff --git a/packages/observability/scripts/apply-alerting.sh b/packages/observability/scripts/apply-alerting.sh index 788cf58e554..8f7991f330a 100755 --- a/packages/observability/scripts/apply-alerting.sh +++ b/packages/observability/scripts/apply-alerting.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash -# Push alert rule groups in `provisioning/alerting/` to a hosted Grafana -# via the provisioning HTTP API. grafanactl doesn't manage alert rules +# Push alert rule groups in `provisioning/alerting/`, plus the contact +# points and rule groups in `provisioning/alerting-hosted//`, to a hosted +# Grafana via the provisioning HTTP API. grafanactl doesn't manage alert rules # (its `resources list` covers App Platform kinds only — dashboards, # folders, playlists, etc.), and locally docker-compose mounts # `provisioning/alerting/` into the container so Grafana provisions from @@ -15,6 +16,8 @@ # run, source ./scripts/grafanactl-env.sh first. # # GRAFANA_TOKEN — service-account token (SecureString) +# Plus every ${VAR} the env's files reference, e.g. DISCORD_ALARMS_WEBHOOK +# for provisioning/alerting-hosted/staging/contact-points/. # # What it pushes: # - Each `.json` in `provisioning/alerting/` is read as the standard @@ -162,13 +165,49 @@ upsert_group() { esac } -shopt -s nullglob -files=(provisioning/alerting/*.json) -[[ "${#files[@]}" -gt 0 ]] || { echo "no alert-rule-group json files found" >&2; exit 0; } +# A contact point is created when its uid is new and replaced otherwise. +# The provisioning API has no single upsert call for contact points, so look +# the uid up first. +upsert_contact_point() { + local uid="$1" body="$2" + local method url http_status response existing + existing="$(curl -sS --fail-with-body \ + -H "Authorization: Bearer ${GRAFANA_TOKEN}" \ + "${grafana_server}/api/v1/provisioning/contact-points")" \ + || fail "listing contact points failed: ${existing}" + if jq -e --arg uid "$uid" 'any(.[]; .uid == $uid)' <<<"$existing" >/dev/null; then + method=PUT + url="${grafana_server}/api/v1/provisioning/contact-points/${uid}" + else + method=POST + url="${grafana_server}/api/v1/provisioning/contact-points" + fi -echo "apply-alerting: env=${env_name} server=${grafana_server} files=${#files[@]}" >&2 + response="$(mktemp -t alerting-response.XXXXXX)" + http_status="$(curl -sS -o "$response" -w '%{http_code}' -X "$method" \ + -H "Authorization: Bearer ${GRAFANA_TOKEN}" \ + -H "Content-Type: application/json" \ + -H "X-Disable-Provenance: true" \ + --data-binary "$body" \ + "$url")" -for f in "${files[@]}"; do + case "$http_status" in + 2??) + echo " ↻ contact point ${uid} (${method})" >&2 + rm -f "$response" + ;; + *) + # The body carries the webhook URL, so print only the server's message. + echo " ✗ contact point ${uid} → HTTP ${http_status}" >&2 + jq -r '.message // empty' "$response" 2>/dev/null | sed 's/^/ /' >&2 || true + rm -f "$response" + fail "contact point push failed" + ;; + esac +} + +push_rule_groups() { + local f="$1" raw resolved folder_uid group_name interval_raw interval_secs body echo "→ ${f}" >&2 # Each file is `apiVersion: 1` + `groups: [...]`. Pull groups out as # JSON one per line, envsubst them so ${VAR} placeholders resolve, then @@ -195,6 +234,42 @@ for f in "${files[@]}"; do upsert_group "$folder_uid" "$group_name" "$body" done +} + +push_contact_points() { + local f="$1" raw resolved uid body + echo "→ ${f}" >&2 + # Same file shape as Grafana file provisioning: `contactPoints[].receivers[]`. + # The API takes one receiver at a time, named after its contact point. + jq -c '.contactPoints[] | .name as $name | .receivers[] | . + {name: $name}' "$f" \ + | while IFS= read -r raw; do + resolved="$(resolve_placeholders "$raw" "$f")" + uid="$(jq -r '.uid' <<<"$resolved")" + [[ -n "$uid" && "$uid" != "null" ]] || fail "${f}: receiver missing 'uid'" + body="$(jq -c '{uid, name, type, settings, disableResolveMessage}' <<<"$resolved")" + upsert_contact_point "$uid" "$body" + done +} + +shopt -s nullglob +# provisioning/alerting/ applies to every environment, and docker-compose +# mounts it for local Grafana too. provisioning/alerting-hosted// holds +# what exists in one hosted environment only: rules for a service that runs +# there alone, and contact points whose secrets come from that +# environment's SSM. Contact points go first, so a rule that routes to one +# can find it. +hosted_dir="provisioning/alerting-hosted/${env_name}" +contact_point_files=("${hosted_dir}"/contact-points/*.json) +rule_files=(provisioning/alerting/*.json "${hosted_dir}"/rules/*.json) + +echo "apply-alerting: env=${env_name} server=${grafana_server} contact-point files=${#contact_point_files[@]} rule files=${#rule_files[@]}" >&2 + +for f in "${contact_point_files[@]}"; do + push_contact_points "$f" +done + +for f in "${rule_files[@]}"; do + push_rule_groups "$f" done echo "apply-alerting: done" >&2 From dcd2a423104759adc2f170353cd1fb12602e571b Mon Sep 17 00:00:00 2001 From: Buck Doyle Date: Thu, 8 Oct 2026 03:25:05 -0500 Subject: [PATCH 2/2] Check alert placeholders before apply.sh pushes anything apply-alerting.sh found an unset ${VAR} only when it reached the group that used it. A missing DISCORD_ALARMS_WEBHOOK therefore failed after apply.sh had already pushed dashboards and data sources, and after any earlier alert files. apply-alerting.sh now reads every ${VAR} in the files it will push for the env, and names all unset ones before it pushes anything. With --preflight it only makes that check. apply.sh runs the preflight with its other prerequisite checks. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01QZtQde6na3jLiKVjAavLPz --- .../observability/scripts/apply-alerting.sh | 63 ++++++++++++++----- packages/observability/scripts/apply.sh | 3 + 2 files changed, 52 insertions(+), 14 deletions(-) diff --git a/packages/observability/scripts/apply-alerting.sh b/packages/observability/scripts/apply-alerting.sh index 8f7991f330a..ab89abe205f 100755 --- a/packages/observability/scripts/apply-alerting.sh +++ b/packages/observability/scripts/apply-alerting.sh @@ -8,7 +8,12 @@ # the file directly. This script only runs against staging / production. # # Usage: -# ./scripts/apply-alerting.sh --env +# ./scripts/apply-alerting.sh --env [--preflight] +# +# --preflight only check that every ${VAR} the env's files reference is +# set, then exit without contacting Grafana. apply.sh runs this +# before it pushes anything, so a missing secret cannot leave +# a partial apply. A normal run makes the same check first too. # # Required env vars (staging / production only). The CI apply workflow # (.github/workflows/observability-apply-{staging,production}.yml) fetches @@ -45,6 +50,7 @@ usage_error() { echo "error: $1" >&2; exit 2; } fail() { echo "error: $1" >&2; exit 1; } env_name="" +preflight="" while [[ $# -gt 0 ]]; do case "$1" in --env) @@ -57,6 +63,10 @@ while [[ $# -gt 0 ]]; do [[ -n "$env_name" ]] || usage_error "--env requires a value" shift ;; + --preflight) + preflight=1 + shift + ;; *) usage_error "unknown option: $1";; esac done @@ -75,6 +85,44 @@ case "$env_name" in ;; esac +cd "$(dirname "$0")/.." + +shopt -s nullglob +# provisioning/alerting/ applies to every environment, and docker-compose +# mounts it for local Grafana too. provisioning/alerting-hosted// holds +# what exists in one hosted environment only: rules for a service that runs +# there alone, and contact points whose secrets come from that +# environment's SSM. Contact points go first, so a rule that routes to one +# can find it. +hosted_dir="provisioning/alerting-hosted/${env_name}" +contact_point_files=("${hosted_dir}"/contact-points/*.json) +rule_files=(provisioning/alerting/*.json "${hosted_dir}"/rules/*.json) + +# Fail before any push when a file this run would push references a ${VAR} +# that is not set, naming every missing one at once. resolve_placeholders +# below makes the same check per group, but only when it reaches that group, +# after earlier files have been pushed. +check_placeholders() { + local f name missing=() + for f in "${contact_point_files[@]}" "${rule_files[@]}"; do + while IFS= read -r name; do + [[ -z "$name" ]] && continue + [[ -n "${!name:-}" ]] || missing+=("\${${name}} (${f})") + done < <(grep -oE '\$\{[A-Z_][A-Z0-9_]*\}' "$f" | sed -E 's/^\$\{(.*)\}$/\1/' | sort -u) + done + if [[ "${#missing[@]}" -gt 0 ]]; then + printf 'error: not set (or empty) in environment; CI fetches these from SSM in observability-apply-%s.yml:\n' "$env_name" >&2 + printf ' %s\n' "${missing[@]}" >&2 + exit 1 + fi +} + +check_placeholders +if [[ -n "$preflight" ]]; then + echo "apply-alerting: preflight ok (env=${env_name})" >&2 + exit 0 +fi + for cmd in yq jq curl envsubst; do command -v "$cmd" >/dev/null \ || fail "missing dependency: ${cmd}. Install via brew (yq, jq, gettext for envsubst) or apt (yq, jq, gettext-base)." @@ -82,8 +130,6 @@ done [[ -n "${GRAFANA_TOKEN:-}" ]] || fail "GRAFANA_TOKEN not set; run \`source ./scripts/grafanactl-env.sh ${env_name}\` first" -cd "$(dirname "$0")/.." - # Pull the per-env Grafana server URL from grafanactl's committed config so # this script and grafanactl always agree on the target host. grafana_server="$(yq -r ".contexts.${env_name}.grafana.server" grafanactl/config.yaml)" @@ -251,17 +297,6 @@ push_contact_points() { done } -shopt -s nullglob -# provisioning/alerting/ applies to every environment, and docker-compose -# mounts it for local Grafana too. provisioning/alerting-hosted// holds -# what exists in one hosted environment only: rules for a service that runs -# there alone, and contact points whose secrets come from that -# environment's SSM. Contact points go first, so a rule that routes to one -# can find it. -hosted_dir="provisioning/alerting-hosted/${env_name}" -contact_point_files=("${hosted_dir}"/contact-points/*.json) -rule_files=(provisioning/alerting/*.json "${hosted_dir}"/rules/*.json) - echo "apply-alerting: env=${env_name} server=${grafana_server} contact-point files=${#contact_point_files[@]} rule files=${#rule_files[@]}" >&2 for f in "${contact_point_files[@]}"; do diff --git a/packages/observability/scripts/apply.sh b/packages/observability/scripts/apply.sh index 8b5bf5ac7af..ac82b57d2f3 100755 --- a/packages/observability/scripts/apply.sh +++ b/packages/observability/scripts/apply.sh @@ -87,6 +87,9 @@ if [[ "$env_name" != "local" ]]; then [[ -n "${!v:-}" ]] \ || { echo "error: ${v} not set; CI fetches it from SSM in observability-apply-${env_name}.yml — for a local hosted run, export it manually first (see apply-datasources.sh header for the SSM path)" >&2; exit 1; } done + # Every ${VAR} in the alert files this env pushes, including + # provisioning/alerting-hosted//, read from the files themselves. + ./scripts/apply-alerting.sh --env "$env_name" --preflight fi cfg="$(./scripts/render-config.sh "$env_name")"