Skip to content

Commit c9ecd21

Browse files
ralyodioclaude
andcommitted
alert: say loudly when smshub rejects the key, add --check + tests
A 401/403 from smshub now emails with the subject "smshub alert key REJECTED", names where the key came from (env override vs vault) and how to replace it, and logs KEY REJECTED. alert --check asks smshub about the key without alerting anyone (daily cron), emailing once per 6h while it is rejected. SMSHUB_URL is overridable so test/alert.sh can drive a fake smshub; scripts-check runs test/*.sh. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
1 parent d692367 commit c9ecd21

4 files changed

Lines changed: 141 additions & 19 deletions

File tree

‎README.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -258,12 +258,17 @@ alert "fleet-db-backup nightly failed"
258258
alert --severity critical "coinpay webhooks are down" # always rings
259259
alert --key qc-sweep --every 6h "riotcoder-qc sweep is erroring"
260260
alert --clear qc-sweep # recovered
261+
alert --check # is the key accepted?
261262
```
262263

263264
smshub sends the same text once per 10 minutes; `--key`/`--every` (default:
264265
the message, 6h) keeps a job that fails every minute to one alert per window.
265266
If smshub cannot be reached it emails `ALERT_EMAIL` (set in
266267
`~/.config/alert/env`) instead. Log: `~/.local/state/alert/alert.log`.
268+
If smshub rejects the key (401/403) the email subject says "smshub alert key
269+
REJECTED", names where the key came from and how to replace it; every real alert
270+
still gets its own email. `alert --check` (daily cron) asks smshub about the key
271+
without alerting anyone and sends that email once per 6h while it is rejected.
267272

268273
### `anthropic-credit-check`
269274

‎bin/alert‎

Lines changed: 67 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
# alert --severity critical "coinpay webhooks are down" always rings the phone
66
# alert --key qc-sweep --every 6h "riotcoder-qc sweep is erroring"
77
# alert --clear qc-sweep it recovered: the next failure alerts at once
8+
# alert --check is the smshub key accepted? (daily cron)
89
#
910
# Posts to https://smshub.dev/api/v1/alerts with the API key in vault
1011
# smshub--prod (SMSHUB_ALERT_API_KEY, Anthony's account), which texts his
@@ -13,13 +14,17 @@
1314
# --key/--every (default: the message itself, 6h) keeps a cron job that fails
1415
# every minute down to one alert per window. If smshub cannot be reached the
1516
# alert is emailed instead (ALERT_EMAIL, default anthony@profullstack.com; set
16-
# it in ~/.config/alert/env), so a failure is never silent.
17+
# it in ~/.config/alert/env), so a failure is never silent. If smshub REJECTS
18+
# the key (401/403) that email says so in its subject: the phone did not ring
19+
# and will not for any alert until the key is replaced. `--check` asks smshub
20+
# about the key without alerting anyone and sends that same email when it is
21+
# rejected, so a revoked key is found by cron, not by the next real outage.
1722
#
1823
# Exit 0 when sent or deliberately skipped, 1 when nothing went out.
1924
# Log: ~/.local/state/alert/alert.log
2025
set -uo pipefail
2126

22-
severity=warning key= every=6h to= dry=0
27+
severity=warning key= every=6h to= dry=0 check=0
2328
while [ $# -gt 0 ]; do
2429
case "$1" in
2530
--severity) severity=$2; shift 2 ;;
@@ -31,14 +36,16 @@ while [ $# -gt 0 ]; do
3136
--to) to=$2; shift 2 ;;
3237
--to=*) to=${1#*=}; shift ;;
3338
--dry-run) dry=1; shift ;;
39+
--check) check=1; shift ;;
3440
--clear) rm -f "$HOME/.local/state/alert/$(printf '%s' "$2" | tr -c 'A-Za-z0-9._-' '_').last"; exit 0 ;;
35-
-h|--help) sed -n '2,20p' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;;
41+
-h|--help) sed -n '2,25p' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;;
3642
--) shift; break ;;
3743
-*) echo "alert: unknown option $1" >&2; exit 64 ;;
3844
*) break ;;
3945
esac
4046
done
4147
msg=$(printf '%s ' "$@" | tr -s '[:space:]' ' ' | sed 's/^ //; s/ $//')
48+
[ "$check" = 1 ] && msg="smshub alert key check"
4249
[ -n "$msg" ] || { echo 'usage: alert [--severity info|warning|critical] [--key k --every 6h] "message"' >&2; exit 64; }
4350
host=$(hostname -s 2>/dev/null || hostname)
4451
case "$msg" in "$host:"*) ;; *) msg="$host: $msg" ;; esac
@@ -50,13 +57,60 @@ export PATH="$HOME/.local/bin:$HOME/.local/share/mise/shims:/usr/local/bin:/usr/
5057
STATE="$HOME/.local/state/alert"
5158
mkdir -p "$STATE"
5259
log() { printf '%s %s\n' "$(date -u +%FT%TZ)" "$*" >>"$STATE/alert.log"; }
60+
SMSHUB_URL=${SMSHUB_URL:-https://smshub.dev}
5361

5462
seconds() {
5563
case "$1" in
5664
*d) echo $(( ${1%d} * 86400 )) ;; *h) echo $(( ${1%h} * 3600 )) ;;
5765
*m) echo $(( ${1%m} * 60 )) ;; *s) echo "${1%s}" ;; *) echo "$1" ;;
5866
esac
5967
}
68+
apikey=${SMSHUB_ALERT_API_KEY:-} keysrc='the SMSHUB_ALERT_API_KEY environment variable (not the vault)'
69+
if [ -z "$apikey" ]; then
70+
keysrc='vault smshub--prod SMSHUB_ALERT_API_KEY'
71+
tmp=$(mktemp -d "$STATE/v.XXXXXX")
72+
if logicsrc teams export profullstack smshub prod -o "$tmp/v.csv" --yes >/dev/null 2>&1; then
73+
apikey=$(python3 -I -c 'import csv,sys
74+
for r in csv.DictReader(open(sys.argv[1])):
75+
if r["key"]=="SMSHUB_ALERT_API_KEY": print(r["value"])' "$tmp/v.csv")
76+
fi
77+
rm -rf "$tmp"
78+
fi
79+
80+
email() { # subject body
81+
mail send --to "${ALERT_EMAIL:-anthony@profullstack.com}" --subject "$1" --via resend --body "$2" >/dev/null 2>&1
82+
}
83+
84+
# The key is broken, not smshub: say that loudly, once per 6h however many
85+
# alerts hit it, and name where the key came from and how to replace it.
86+
key_broken() { # http-code response [the alert that did not go out]
87+
log "KEY REJECTED by smshub HTTP $1 (key from $keysrc): ${2:0:200}"
88+
local ks="$STATE/smshub-key-broken.last"
89+
if [ -f "$ks" ] && [ $(( $(date +%s) - $(stat -c %Y "$ks") )) -lt 21600 ] && [ -z "${3:-}" ]; then return 0; fi
90+
email "[alert] $host: smshub alert key REJECTED (HTTP $1): alerts cannot call or text you${3:+ | $3}" "${3:+$3
91+
92+
}SMSHUB REJECTED THE ALERT API KEY (HTTP $1: ${2:0:200}).
93+
Until it is replaced no alert from $host can call or text you; each one
94+
falls back to an email like this.
95+
96+
The key came from $keysrc.
97+
Fix: create a key at $SMSHUB_URL (Settings > API keys, Anthony's account),
98+
then put it in the vault as SMSHUB_ALERT_API_KEY: logicsrc teams pull profullstack
99+
smshub prod --env f.env, edit f.env, logicsrc teams push ... --env f.env, shred f.env.
100+
Check: alert --check" && touch "$ks"
101+
}
102+
103+
if [ "$check" = 1 ]; then
104+
[ -n "$apikey" ] || { log "check: no key ($keysrc)"; email "[alert] $host: no smshub alert key: alerts cannot call or text you" "alert --check on $host found no key in $keysrc (vault unreachable or key missing)."; exit 1; }
105+
out=$(curl -sS -m 30 -w '\n%{http_code}' "$SMSHUB_URL/api/v1/alerts?limit=1" -H "X-API-Key: $apikey" 2>&1)
106+
code=${out##*$'\n'}; out=${out%$'\n'*}
107+
case "$code" in
108+
200) rm -f "$STATE/smshub-key-broken.last"; log "check: key ok"; echo "smshub alert key ok"; exit 0 ;;
109+
401|403) key_broken "$code" "$out"; echo "smshub REJECTED the alert key (HTTP $code)" >&2; exit 1 ;;
110+
*) log "check: smshub HTTP $code (not a key answer): ${out:0:200}"; echo "smshub unreachable or erroring (HTTP $code); key not judged" >&2; exit 1 ;;
111+
esac
112+
fi
113+
60114
[ -n "$key" ] || key=$(printf '%s' "$msg" | sha256sum | cut -c1-16)
61115
key=$(printf '%s' "$key" | tr -c 'A-Za-z0-9._-' '_')
62116
stamp="$STATE/$key.last"
@@ -68,21 +122,10 @@ fi
68122

69123
if [ "$dry" = 1 ]; then echo "would alert ($severity): $msg"; exit 0; fi
70124

71-
apikey=${SMSHUB_ALERT_API_KEY:-}
72-
if [ -z "$apikey" ]; then
73-
tmp=$(mktemp -d "$STATE/v.XXXXXX")
74-
if logicsrc teams export profullstack smshub prod -o "$tmp/v.csv" --yes >/dev/null 2>&1; then
75-
apikey=$(python3 -I -c 'import csv,sys
76-
for r in csv.DictReader(open(sys.argv[1])):
77-
if r["key"]=="SMSHUB_ALERT_API_KEY": print(r["value"])' "$tmp/v.csv")
78-
fi
79-
rm -rf "$tmp"
80-
fi
81-
82125
body=$(jq -nc --arg m "$msg" --arg s "$severity" --arg t "$to" '{message:$m, severity:$s} + (if $t=="" then {} else {to:$t} end)')
83126
out= code=000
84127
if [ -n "$apikey" ]; then
85-
out=$(curl -sS -m 60 -w '\n%{http_code}' -X POST https://smshub.dev/api/v1/alerts \
128+
out=$(curl -sS -m 60 -w '\n%{http_code}' -X POST "$SMSHUB_URL/api/v1/alerts" \
86129
-H "X-API-Key: $apikey" -H 'Content-Type: application/json' --data "$body" 2>&1)
87130
code=${out##*$'\n'}
88131
out=${out%$'\n'*}
@@ -95,12 +138,17 @@ if [ "$code" = 200 ] || [ "$code" = 201 ]; then
95138
fi
96139

97140
# smshub said no (or is down): email, so the failure still reaches a person.
98-
reason=${out:-no API key (vault smshub--prod SMSHUB_ALERT_API_KEY)}
141+
if [ "$code" = 401 ] || [ "$code" = 403 ]; then
142+
if key_broken "$code" "$out" "$msg"; then
143+
touch "$stamp"; log "emailed [$key] instead (key rejected): $msg"; exit 0
144+
fi
145+
log "FAILED [$key] nothing went out: $msg"; exit 1
146+
fi
147+
reason=${out:-no API key ($keysrc)}
99148
log "smshub failed [$key] HTTP $code: $reason"
100-
if mail send --to "${ALERT_EMAIL:-anthony@profullstack.com}" --subject "[alert] $msg" --via resend \
101-
--body "$msg
149+
if email "[alert] $msg" "$msg
102150
103-
(smshub alert failed, HTTP $code: ${reason:0:300})" >/dev/null 2>&1; then
151+
(smshub alert failed, HTTP $code: ${reason:0:300})"; then
104152
touch "$stamp"
105153
log "emailed [$key] instead: $msg"
106154
exit 0

‎bin/scripts-check‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -96,6 +96,16 @@ else
9696
note "shellcheck not installed, skipped (apt-get install shellcheck)"
9797
fi
9898

99+
# ---------------------------------------------------------------- tests
100+
101+
# test/*.sh drive a script against fakes (no network, no real email). Each
102+
# exits non-zero on a failed case and skips itself when a tool is missing.
103+
for t in test/*.sh; do
104+
[ -f "$t" ] || continue
105+
note "test $t"
106+
bash "$t" || fail "$t"
107+
done
108+
99109
# ---------------------------------------------------------------- verdict
100110

101111
if [ "$rc" = 0 ]; then

‎test/alert.sh‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
#!/usr/bin/env bash
2+
# test/alert.sh: bin/alert against a fake smshub. No network, no real email:
3+
# HOME is a temp dir whose ~/.local/bin/mail records what it was asked to send.
4+
# Run by bin/scripts-check.
5+
set -uo pipefail
6+
root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)
7+
for t in python3 jq curl; do command -v "$t" >/dev/null || { echo " alert test skipped: no $t"; exit 0; }; done
8+
9+
tmp=$(mktemp -d); trap 'kill "$srv" 2>/dev/null; rm -rf "$tmp"' EXIT
10+
mkdir -p "$tmp/home/.local/bin"
11+
cat >"$tmp/home/.local/bin/mail" <<'M'
12+
#!/usr/bin/env bash
13+
printf '%s\n' "$@" >>"$HOME/mail.out"
14+
M
15+
chmod +x "$tmp/home/.local/bin/mail"
16+
17+
# Fake smshub: key "good" is accepted, anything else gets smshub's real 401 body.
18+
cat >"$tmp/srv.py" <<'P'
19+
import http.server, json, sys
20+
class H(http.server.BaseHTTPRequestHandler):
21+
def reply(self):
22+
ok = self.headers.get("X-API-Key") == "good"
23+
n = int(self.headers.get("Content-Length") or 0)
24+
if n: self.rfile.read(n)
25+
body = {"alert": {"channels": ["call"]}, "note": "calling", "alerts": []} if ok else {"error": "Invalid API key"}
26+
self.send_response(200 if ok else 401); self.end_headers(); self.wfile.write(json.dumps(body).encode())
27+
do_GET = do_POST = reply
28+
def log_message(self, *a): pass
29+
s = http.server.HTTPServer(("127.0.0.1", 0), H); print(s.server_port, flush=True); s.serve_forever()
30+
P
31+
python3 -I "$tmp/srv.py" >"$tmp/port" & srv=$!
32+
for _ in $(seq 50); do [ -s "$tmp/port" ] && break; sleep 0.1; done
33+
export SMSHUB_URL="http://127.0.0.1:$(cat "$tmp/port")"
34+
35+
fails=0
36+
ok() { echo " ok: $1"; }
37+
bad() { echo " FAIL: $1" >&2; fails=1; }
38+
run() { HOME="$tmp/home" SMSHUB_ALERT_API_KEY=$1 "$root/bin/alert" "${@:2}" >/dev/null 2>&1; }
39+
mailed() { cat "$tmp/home/mail.out" 2>/dev/null; }
40+
41+
run good --key t1 "disk full" && [ ! -s "$tmp/home/mail.out" ] && ok "accepted key: sent, no email" || bad "accepted key"
42+
43+
run bad --key t2 "disk full"; rc=$?
44+
[ $rc = 0 ] && mailed | grep -q 'smshub alert key REJECTED (HTTP 401)' && mailed | grep -q 'disk full' \
45+
&& mailed | grep -q 'environment variable (not the vault)' \
46+
&& ok "401: emailed, subject says key REJECTED, keeps the alert, names the key source" || bad "401 email: $(mailed)"
47+
grep -q 'KEY REJECTED' "$tmp/home/.local/state/alert/alert.log" && ok "401: logged KEY REJECTED" || bad "401 log"
48+
49+
: >"$tmp/home/mail.out"
50+
run bad --key t3 "second outage" && mailed | grep -q 'second outage' && ok "401 again: every real alert still emailed" || bad "401 second alert"
51+
52+
: >"$tmp/home/mail.out"; rm -f "$tmp/home/.local/state/alert/smshub-key-broken.last"
53+
run bad --check; rc=$?
54+
[ $rc = 1 ] && mailed | grep -q 'key REJECTED' && ok "--check on rejected key: exit 1 + email" || bad "--check rejected (rc $rc)"
55+
: >"$tmp/home/mail.out"
56+
run bad --check; [ ! -s "$tmp/home/mail.out" ] && ok "--check: one key email per 6h" || bad "--check repeated email"
57+
run good --check && [ ! -f "$tmp/home/.local/state/alert/smshub-key-broken.last" ] && ok "--check on good key: exit 0, clears" || bad "--check good"
58+
59+
exit $fails

0 commit comments

Comments
 (0)