Alert in Discord when the Actions collector goes silent in Loki

.github/workflows/observability-apply-staging.yml

@@ -121,6 +121,8 @@ jobs:
           fetch BOXEL_DB_USER          /staging/boxel/GRAFANA_DB_USER
           fetch BOXEL_DB_PASSWORD      /staging/boxel/GRAFANA_DB_PASSWORD  --with-decryption
           fetch GRAFANA_SECRET         /staging/boxel/GRAFANA_SECRET       --with-decryption
+          # provisioning/alerting-hosted/staging/contact-points/discord-alarms.json
+          fetch DISCORD_ALARMS_WEBHOOK /staging/boxel-grafana/discord_alarms_webhook --with-decryption
 
       - name: Lint
         # Belt-and-suspenders: ci-lint.yaml runs the same lint at PR time

packages/observability/README.md

@@ -38,8 +38,13 @@ grafanactl/
     dashboards/        # grafanactl push: dashboard JSON, organized by folder
 provisioning/          # mounted into Grafana at /etc/grafana/provisioning/
   datasources/         # data sources (Loki, Postgres, CloudWatch, Prometheus)
-  alerting/            # alert rule groups (no contact points / notification
-                       # policies live here — routing uses Grafana's defaults)
+  alerting/            # alert rule groups for every env; routing uses
+                       # Grafana's default notification policy
+  alerting-hosted/     # NOT mounted locally. <env>/contact-points/ and
+                       # <env>/rules/ exist in that hosted env only;
+                       # apply-alerting.sh pushes contact points first, with
+                       # ${VAR} secrets from SSM. A rule here may route to
+                       # one through `notification_settings.receiver`.
   local-only/          # local-dev overrides — bind-mounted file-by-file over
                        # `datasources/`. apply-datasources.sh ignores this dir.
 collectors/
@@ -147,6 +152,12 @@ public — a fine-grained token scoped to _Public repositories (read-only)_, or
 even a classic token with no scopes ticked, is enough. Were the repository ever
 made private, it would need Actions: Read-only on it.
 
+The staging alert "Actions Collector Silent in Loki"
+(`provisioning/alerting-hosted/staging/rules/actions-collector-group.json`)
+posts to the `alarms-staging-internal` Discord channel when Loki receives no
+collector lines for ten minutes. Its summary says how to tell a stalled log
+router from a stopped collector.
+
 Build and deploy with the **Manual Deploy [actions-collector]** workflow. It is
 `workflow_dispatch` only — putting the deploy of a queue sampler on a CI
 trigger would make it queue behind the backlog it reports on.

packages/observability/provisioning/alerting-hosted/staging/contact-points/discord-alarms.json

@@ -0,0 +1,20 @@
+{
+  "apiVersion": 1,
+  "contactPoints": [
+    {
+      "orgId": 1,
+      "name": "Discord alarms-staging-internal",
+      "receivers": [
+        {
+          "uid": "discord-alarms-staging-internal",
+          "type": "discord",
+          "disableResolveMessage": false,
+          "settings": {
+            "url": "${DISCORD_ALARMS_WEBHOOK}",
+            "use_discord_username": false
+          }
+        }
+      ]
+    }
+  ]
+}

packages/observability/provisioning/alerting-hosted/staging/rules/actions-collector-group.json

@@ -0,0 +1,100 @@
+{
+  "apiVersion": 1,
+  "groups": [
+    {
+      "orgId": 1,
+      "name": "actions-collector-group",
+      "folder": "defd2d156sav4d",
+      "interval": "60s",
+      "rules": [
+        {
+          "uid": "actions-collector-silent",
+          "title": "Actions Collector Silent in Loki",
+          "condition": "C",
+          "data": [
+            {
+              "refId": "A",
+              "queryType": "instant",
+              "relativeTimeRange": {
+                "from": 300,
+                "to": 0
+              },
+              "datasourceUid": "loki",
+              "model": {
+                "datasource": {
+                  "type": "loki",
+                  "uid": "loki"
+                },
+                "editorMode": "code",
+                "expr": "sum(count_over_time({service=\"actions-collector\", env=\"staging\"}[5m]))",
+                "intervalMs": 1000,
+                "maxDataPoints": 43200,
+                "queryType": "instant",
+                "refId": "A"
+              }
+            },
+            {
+              "refId": "B",
+              "queryType": "",
+              "relativeTimeRange": {
+                "from": 0,
+                "to": 0
+              },
+              "datasourceUid": "__expr__",
+              "model": {
+                "datasource": {
+                  "type": "__expr__",
+                  "uid": "__expr__"
+                },
+                "expression": "A",
+                "intervalMs": 1000,
+                "maxDataPoints": 43200,
+                "reducer": "last",
+                "refId": "B",
+                "type": "reduce"
+              }
+            },
+            {
+              "refId": "C",
+              "queryType": "",
+              "relativeTimeRange": {
+                "from": 0,
+                "to": 0
+              },
+              "datasourceUid": "__expr__",
+              "model": {
+                "conditions": [
+                  {
+                    "evaluator": {
+                      "params": [1],
+                      "type": "lt"
+                    }
+                  }
+                ],
+                "datasource": {
+                  "type": "__expr__",
+                  "uid": "__expr__"
+                },
+                "expression": "B",
+                "intervalMs": 1000,
+                "maxDataPoints": 43200,
+                "refId": "C",
+                "type": "threshold"
+              }
+            }
+          ],
+          "noDataState": "Alerting",
+          "execErrState": "Alerting",
+          "for": "5m",
+          "annotations": {
+            "summary": "Loki has received no lines from the GitHub Actions queue collector for at least 10 minutes, so the GitHub Actions Queue dashboard shows no data. The collector writes at least one line a minute. A stalled query returns no series rather than zero, so noDataState is Alerting. Compare with CloudWatch group ecs-boxel-actions-collector-staging: if lines still arrive there, the FireLens log router has stopped sending to Loki, and `aws ecs update-service --cluster staging --service boxel-actions-collector-staging --force-new-deployment` restores it. If CloudWatch is silent too, the collector itself has stopped."
+          },
+          "isPaused": false,
+          "notification_settings": {
+            "receiver": "Discord alarms-staging-internal"
+          }
+        }
+      ]
+    }
+  ]
+}

packages/observability/scripts/apply-alerting.sh

@@ -1,20 +1,28 @@
 #!/usr/bin/env bash
-# Push alert rule groups in `provisioning/alerting/` to a hosted Grafana
-# via the provisioning HTTP API. grafanactl doesn't manage alert rules
+# Push alert rule groups in `provisioning/alerting/`, plus the contact
+# points and rule groups in `provisioning/alerting-hosted/<env>/`, to a hosted
+# Grafana via the provisioning HTTP API. grafanactl doesn't manage alert rules
 # (its `resources list` covers App Platform kinds only — dashboards,
 # folders, playlists, etc.), and locally docker-compose mounts
 # `provisioning/alerting/` into the container so Grafana provisions from
 # the file directly. This script only runs against staging / production.
 #
 # Usage:
-#   ./scripts/apply-alerting.sh --env <local|staging|production>
+#   ./scripts/apply-alerting.sh --env <local|staging|production> [--preflight]
+#
+#   --preflight  only check that every ${VAR} the env's files reference is
+#                set, then exit without contacting Grafana. apply.sh runs this
+#                before it pushes anything, so a missing secret cannot leave
+#                a partial apply. A normal run makes the same check first too.
 #
 # Required env vars (staging / production only). The CI apply workflow
 # (.github/workflows/observability-apply-{staging,production}.yml) fetches
 # GRAFANA_TOKEN from SSM and exports it to $GITHUB_ENV. For a local hosted
 # run, source ./scripts/grafanactl-env.sh first.
 #
 #   GRAFANA_TOKEN — service-account token (SecureString)
+#   Plus every ${VAR} the env's files reference, e.g. DISCORD_ALARMS_WEBHOOK
+#   for provisioning/alerting-hosted/staging/contact-points/.
 #
 # What it pushes:
 #   - Each `.json` in `provisioning/alerting/` is read as the standard
@@ -42,6 +50,7 @@ usage_error() { echo "error: $1" >&2; exit 2; }
 fail() { echo "error: $1" >&2; exit 1; }
 
 env_name=""
+preflight=""
 while [[ $# -gt 0 ]]; do
   case "$1" in
     --env)
@@ -54,6 +63,10 @@ while [[ $# -gt 0 ]]; do
       [[ -n "$env_name" ]] || usage_error "--env requires a value"
       shift
       ;;
+    --preflight)
+      preflight=1
+      shift
+      ;;
     *) usage_error "unknown option: $1";;
   esac
 done
@@ -72,15 +85,51 @@ case "$env_name" in
     ;;
 esac
 
+cd "$(dirname "$0")/.."
+
+shopt -s nullglob
+# provisioning/alerting/ applies to every environment, and docker-compose
+# mounts it for local Grafana too. provisioning/alerting-hosted/<env>/ holds
+# what exists in one hosted environment only: rules for a service that runs
+# there alone, and contact points whose secrets come from that
+# environment's SSM. Contact points go first, so a rule that routes to one
+# can find it.
+hosted_dir="provisioning/alerting-hosted/${env_name}"
+contact_point_files=("${hosted_dir}"/contact-points/*.json)
+rule_files=(provisioning/alerting/*.json "${hosted_dir}"/rules/*.json)
+
+# Fail before any push when a file this run would push references a ${VAR}
+# that is not set, naming every missing one at once. resolve_placeholders
+# below makes the same check per group, but only when it reaches that group,
+# after earlier files have been pushed.
+check_placeholders() {
+  local f name missing=()
+  for f in "${contact_point_files[@]}" "${rule_files[@]}"; do
+    while IFS= read -r name; do
+      [[ -z "$name" ]] && continue
+      [[ -n "${!name:-}" ]] || missing+=("\${${name}} (${f})")
+    done < <(grep -oE '\$\{[A-Z_][A-Z0-9_]*\}' "$f" | sed -E 's/^\$\{(.*)\}$/\1/' | sort -u)
+  done
+  if [[ "${#missing[@]}" -gt 0 ]]; then
+    printf 'error: not set (or empty) in environment; CI fetches these from SSM in observability-apply-%s.yml:\n' "$env_name" >&2
+    printf '  %s\n' "${missing[@]}" >&2
+    exit 1
+  fi
+}
+
+check_placeholders
+if [[ -n "$preflight" ]]; then
+  echo "apply-alerting: preflight ok (env=${env_name})" >&2
+  exit 0
+fi
+
 for cmd in yq jq curl envsubst; do
   command -v "$cmd" >/dev/null \
     || fail "missing dependency: ${cmd}. Install via brew (yq, jq, gettext for envsubst) or apt (yq, jq, gettext-base)."
 done
 
 [[ -n "${GRAFANA_TOKEN:-}" ]] || fail "GRAFANA_TOKEN not set; run \`source ./scripts/grafanactl-env.sh ${env_name}\` first"
 
-cd "$(dirname "$0")/.."
-
 # Pull the per-env Grafana server URL from grafanactl's committed config so
 # this script and grafanactl always agree on the target host.
 grafana_server="$(yq -r ".contexts.${env_name}.grafana.server" grafanactl/config.yaml)"
@@ -162,13 +211,49 @@ upsert_group() {
   esac
 }
 
-shopt -s nullglob
-files=(provisioning/alerting/*.json)
-[[ "${#files[@]}" -gt 0 ]] || { echo "no alert-rule-group json files found" >&2; exit 0; }
+# A contact point is created when its uid is new and replaced otherwise.
+# The provisioning API has no single upsert call for contact points, so look
+# the uid up first.
+upsert_contact_point() {
+  local uid="$1" body="$2"
+  local method url http_status response existing
+  existing="$(curl -sS --fail-with-body \
+    -H "Authorization: Bearer ${GRAFANA_TOKEN}" \
+    "${grafana_server}/api/v1/provisioning/contact-points")" \
+    || fail "listing contact points failed: ${existing}"
+  if jq -e --arg uid "$uid" 'any(.[]; .uid == $uid)' <<<"$existing" >/dev/null; then
+    method=PUT
+    url="${grafana_server}/api/v1/provisioning/contact-points/${uid}"
+  else
+    method=POST
+    url="${grafana_server}/api/v1/provisioning/contact-points"
+  fi
 
-echo "apply-alerting: env=${env_name} server=${grafana_server} files=${#files[@]}" >&2
+  response="$(mktemp -t alerting-response.XXXXXX)"
+  http_status="$(curl -sS -o "$response" -w '%{http_code}' -X "$method" \
+    -H "Authorization: Bearer ${GRAFANA_TOKEN}" \
+    -H "Content-Type: application/json" \
+    -H "X-Disable-Provenance: true" \
+    --data-binary "$body" \
+    "$url")"
+
+  case "$http_status" in
+    2??)
+      echo "  ↻ contact point ${uid} (${method})" >&2
+      rm -f "$response"
+      ;;
+    *)
+      # The body carries the webhook URL, so print only the server's message.
+      echo "  ✗ contact point ${uid} → HTTP ${http_status}" >&2
+      jq -r '.message // empty' "$response" 2>/dev/null | sed 's/^/    /' >&2 || true
+      rm -f "$response"
+      fail "contact point push failed"
+      ;;
+  esac
+}
 
-for f in "${files[@]}"; do
+push_rule_groups() {
+  local f="$1" raw resolved folder_uid group_name interval_raw interval_secs body
   echo "→ ${f}" >&2
   # Each file is `apiVersion: 1` + `groups: [...]`. Pull groups out as
   # JSON one per line, envsubst them so ${VAR} placeholders resolve, then
@@ -195,6 +280,31 @@ for f in "${files[@]}"; do
 
     upsert_group "$folder_uid" "$group_name" "$body"
   done
+}
+
+push_contact_points() {
+  local f="$1" raw resolved uid body
+  echo "→ ${f}" >&2
+  # Same file shape as Grafana file provisioning: `contactPoints[].receivers[]`.
+  # The API takes one receiver at a time, named after its contact point.
+  jq -c '.contactPoints[] | .name as $name | .receivers[] | . + {name: $name}' "$f" \
+    | while IFS= read -r raw; do
+      resolved="$(resolve_placeholders "$raw" "$f")"
+      uid="$(jq -r '.uid' <<<"$resolved")"
+      [[ -n "$uid" && "$uid" != "null" ]] || fail "${f}: receiver missing 'uid'"
+      body="$(jq -c '{uid, name, type, settings, disableResolveMessage}' <<<"$resolved")"
+      upsert_contact_point "$uid" "$body"
+    done
+}
+
+echo "apply-alerting: env=${env_name} server=${grafana_server} contact-point files=${#contact_point_files[@]} rule files=${#rule_files[@]}" >&2
+
+for f in "${contact_point_files[@]}"; do
+  push_contact_points "$f"
+done
+
+for f in "${rule_files[@]}"; do
+  push_rule_groups "$f"
 done
 
 echo "apply-alerting: done" >&2

packages/observability/scripts/apply.sh

@@ -87,6 +87,9 @@ if [[ "$env_name" != "local" ]]; then
     [[ -n "${!v:-}" ]] \
       || { echo "error: ${v} not set; CI fetches it from SSM in observability-apply-${env_name}.yml — for a local hosted run, export it manually first (see apply-datasources.sh header for the SSM path)" >&2; exit 1; }
   done
+  # Every ${VAR} in the alert files this env pushes, including
+  # provisioning/alerting-hosted/<env>/, read from the files themselves.
+  ./scripts/apply-alerting.sh --env "$env_name" --preflight
 fi
 
 cfg="$(./scripts/render-config.sh "$env_name")"