55# alert --severity critical "coinpay webhooks are down" always rings the phone
66# alert --key qc-sweep --every 6h "riotcoder-qc sweep is erroring"
77# alert --clear qc-sweep it recovered: the next failure alerts at once
8+ # alert --check is the smshub key accepted? (daily cron)
89#
910# Posts to https://smshub.dev/api/v1/alerts with the API key in vault
1011# smshub--prod (SMSHUB_ALERT_API_KEY, Anthony's account), which texts his
1314# --key/--every (default: the message itself, 6h) keeps a cron job that fails
1415# every minute down to one alert per window. If smshub cannot be reached the
1516# alert is emailed instead (ALERT_EMAIL, default anthony@profullstack.com; set
16- # it in ~/.config/alert/env), so a failure is never silent.
17+ # it in ~/.config/alert/env), so a failure is never silent. If smshub REJECTS
18+ # the key (401/403) that email says so in its subject: the phone did not ring
19+ # and will not for any alert until the key is replaced. `--check` asks smshub
20+ # about the key without alerting anyone and sends that same email when it is
21+ # rejected, so a revoked key is found by cron, not by the next real outage.
1722#
1823# Exit 0 when sent or deliberately skipped, 1 when nothing went out.
1924# Log: ~/.local/state/alert/alert.log
2025set -uo pipefail
2126
22- severity=warning key= every=6h to= dry=0
27+ severity=warning key= every=6h to= dry=0 check=0
2328while [ $# -gt 0 ]; do
2429 case " $1 " in
2530 --severity) severity=$2 ; shift 2 ;;
@@ -31,14 +36,16 @@ while [ $# -gt 0 ]; do
3136 --to) to=$2 ; shift 2 ;;
3237 --to=* ) to=${1#* =} ; shift ;;
3338 --dry-run) dry=1; shift ;;
39+ --check) check=1; shift ;;
3440 --clear) rm -f " $HOME /.local/state/alert/$( printf ' %s' " $2 " | tr -c ' A-Za-z0-9._-' ' _' ) .last" ; exit 0 ;;
35- -h|--help) sed -n ' 2,20p ' " $0 " | sed ' s/^# \{0,1\}//' ; exit 0 ;;
41+ -h|--help) sed -n ' 2,25p ' " $0 " | sed ' s/^# \{0,1\}//' ; exit 0 ;;
3642 --) shift ; break ;;
3743 -* ) echo " alert: unknown option $1 " >&2 ; exit 64 ;;
3844 * ) break ;;
3945 esac
4046done
4147msg=$( printf ' %s ' " $@ " | tr -s ' [:space:]' ' ' | sed ' s/^ //; s/ $//' )
48+ [ " $check " = 1 ] && msg=" smshub alert key check"
4249[ -n " $msg " ] || { echo ' usage: alert [--severity info|warning|critical] [--key k --every 6h] "message"' >&2 ; exit 64; }
4350host=$( hostname -s 2> /dev/null || hostname)
4451case " $msg " in " $host :" * ) ;; * ) msg=" $host : $msg " ;; esac
@@ -50,13 +57,60 @@ export PATH="$HOME/.local/bin:$HOME/.local/share/mise/shims:/usr/local/bin:/usr/
5057STATE=" $HOME /.local/state/alert"
5158mkdir -p " $STATE "
5259log () { printf ' %s %s\n' " $( date -u +%FT%TZ) " " $* " >> " $STATE /alert.log" ; }
60+ SMSHUB_URL=${SMSHUB_URL:- https:// smshub.dev}
5361
5462seconds () {
5563 case " $1 " in
5664 * d) echo $(( ${1% d} * 86400 )) ;; * h) echo $(( ${1% h} * 3600 )) ;;
5765 * m) echo $(( ${1% m} * 60 )) ;; * s) echo " ${1% s} " ;; * ) echo " $1 " ;;
5866 esac
5967}
68+ apikey=${SMSHUB_ALERT_API_KEY:- } keysrc=' the SMSHUB_ALERT_API_KEY environment variable (not the vault)'
69+ if [ -z " $apikey " ]; then
70+ keysrc=' vault smshub--prod SMSHUB_ALERT_API_KEY'
71+ tmp=$( mktemp -d " $STATE /v.XXXXXX" )
72+ if logicsrc teams export profullstack smshub prod -o " $tmp /v.csv" --yes > /dev/null 2>&1 ; then
73+ apikey=$( python3 -I -c ' import csv,sys
74+ for r in csv.DictReader(open(sys.argv[1])):
75+ if r["key"]=="SMSHUB_ALERT_API_KEY": print(r["value"])' " $tmp /v.csv" )
76+ fi
77+ rm -rf " $tmp "
78+ fi
79+
80+ email () { # subject body
81+ mail send --to " ${ALERT_EMAIL:- anthony@ profullstack.com} " --subject " $1 " --via resend --body " $2 " > /dev/null 2>&1
82+ }
83+
84+ # The key is broken, not smshub: say that loudly, once per 6h however many
85+ # alerts hit it, and name where the key came from and how to replace it.
86+ key_broken () { # http-code response [the alert that did not go out]
87+ log " KEY REJECTED by smshub HTTP $1 (key from $keysrc ): ${2: 0: 200} "
88+ local ks=" $STATE /smshub-key-broken.last"
89+ if [ -f " $ks " ] && [ $(( $(date +% s) - $(stat - c % Y "$ks ") )) -lt 21600 ] && [ -z " ${3:- } " ]; then return 0; fi
90+ email " [alert] $host : smshub alert key REJECTED (HTTP $1 ): alerts cannot call or text you${3: + | $3 } " " ${3: +$3
91+
92+ } SMSHUB REJECTED THE ALERT API KEY (HTTP $1 : ${2: 0: 200} ).
93+ Until it is replaced no alert from $host can call or text you; each one
94+ falls back to an email like this.
95+
96+ The key came from $keysrc .
97+ Fix: create a key at $SMSHUB_URL (Settings > API keys, Anthony's account),
98+ then put it in the vault as SMSHUB_ALERT_API_KEY: logicsrc teams pull profullstack
99+ smshub prod --env f.env, edit f.env, logicsrc teams push ... --env f.env, shred f.env.
100+ Check: alert --check" && touch " $ks "
101+ }
102+
103+ if [ " $check " = 1 ]; then
104+ [ -n " $apikey " ] || { log " check: no key ($keysrc )" ; email " [alert] $host : no smshub alert key: alerts cannot call or text you" " alert --check on $host found no key in $keysrc (vault unreachable or key missing)." ; exit 1; }
105+ out=$( curl -sS -m 30 -w ' \n%{http_code}' " $SMSHUB_URL /api/v1/alerts?limit=1" -H " X-API-Key: $apikey " 2>&1 )
106+ code=${out##* $' \n ' } ; out=${out% $' \n ' * }
107+ case " $code " in
108+ 200) rm -f " $STATE /smshub-key-broken.last" ; log " check: key ok" ; echo " smshub alert key ok" ; exit 0 ;;
109+ 401|403) key_broken " $code " " $out " ; echo " smshub REJECTED the alert key (HTTP $code )" >&2 ; exit 1 ;;
110+ * ) log " check: smshub HTTP $code (not a key answer): ${out: 0: 200} " ; echo " smshub unreachable or erroring (HTTP $code ); key not judged" >&2 ; exit 1 ;;
111+ esac
112+ fi
113+
60114[ -n " $key " ] || key=$( printf ' %s' " $msg " | sha256sum | cut -c1-16)
61115key=$( printf ' %s' " $key " | tr -c ' A-Za-z0-9._-' ' _' )
62116stamp=" $STATE /$key .last"
68122
69123if [ " $dry " = 1 ]; then echo " would alert ($severity ): $msg " ; exit 0; fi
70124
71- apikey=${SMSHUB_ALERT_API_KEY:- }
72- if [ -z " $apikey " ]; then
73- tmp=$( mktemp -d " $STATE /v.XXXXXX" )
74- if logicsrc teams export profullstack smshub prod -o " $tmp /v.csv" --yes > /dev/null 2>&1 ; then
75- apikey=$( python3 -I -c ' import csv,sys
76- for r in csv.DictReader(open(sys.argv[1])):
77- if r["key"]=="SMSHUB_ALERT_API_KEY": print(r["value"])' " $tmp /v.csv" )
78- fi
79- rm -rf " $tmp "
80- fi
81-
82125body=$( jq -nc --arg m " $msg " --arg s " $severity " --arg t " $to " ' {message:$m, severity:$s} + (if $t=="" then {} else {to:$t} end)' )
83126out= code=000
84127if [ -n " $apikey " ]; then
85- out=$( curl -sS -m 60 -w ' \n%{http_code}' -X POST https://smshub.dev/ api/v1/alerts \
128+ out=$( curl -sS -m 60 -w ' \n%{http_code}' -X POST " $SMSHUB_URL / api/v1/alerts" \
86129 -H " X-API-Key: $apikey " -H ' Content-Type: application/json' --data " $body " 2>&1 )
87130 code=${out##* $' \n ' }
88131 out=${out% $' \n ' * }
@@ -95,12 +138,17 @@ if [ "$code" = 200 ] || [ "$code" = 201 ]; then
95138fi
96139
97140# smshub said no (or is down): email, so the failure still reaches a person.
98- reason=${out:- no API key (vault smshub--prod SMSHUB_ALERT_API_KEY)}
141+ if [ " $code " = 401 ] || [ " $code " = 403 ]; then
142+ if key_broken " $code " " $out " " $msg " ; then
143+ touch " $stamp " ; log " emailed [$key ] instead (key rejected): $msg " ; exit 0
144+ fi
145+ log " FAILED [$key ] nothing went out: $msg " ; exit 1
146+ fi
147+ reason=${out:- no API key ($keysrc )}
99148log " smshub failed [$key ] HTTP $code : $reason "
100- if mail send --to " ${ALERT_EMAIL:- anthony@ profullstack.com} " --subject " [alert] $msg " --via resend \
101- --body " $msg
149+ if email " [alert] $msg " " $msg
102150
103- (smshub alert failed, HTTP $code : ${reason: 0: 300} )" > /dev/null 2>&1 ; then
151+ (smshub alert failed, HTTP $code : ${reason: 0: 300} )" ; then
104152 touch " $stamp "
105153 log " emailed [$key ] instead: $msg "
106154 exit 0
0 commit comments