Alert on sustained node DB CPU saturation

.github/workflows/db-saturation-probe.yml

@@ -0,0 +1,84 @@
+# TC-1601: sustained DB saturation alert for PlanetScale databases.
+#
+# WHAT: every 5 minutes, read CPU and connection-pool metrics for each target in
+# test/probe/db-saturation-targets.json and send one Better Stack heartbeat
+# decision per target. Breach -> POST the heartbeat's /fail URL (pages team@).
+# Healthy -> GET the heartbeat (ping). Unknown or transient API error -> no call,
+# so the missed-heartbeat alert covers it.
+#
+# A target is skipped with a notice until its two secrets are set, so this
+# workflow stays green while unconfigured.
+#
+# SECRETS (Settings -> Secrets and variables -> Actions):
+#   PLANETSCALE_METRICS_TOKEN_NODE_DB     PlanetScale service token ID:TOKEN,
+#                                         read_branch on all branches of tinycloud-node-production.
+#   BETTERSTACK_HEARTBEAT_URL_NODE_DB     "Node DB saturation" heartbeat URL.
+#   PLANETSCALE_METRICS_TOKEN_OPENKEY_DB  Added after the 2026-10-22 DB move.
+#   BETTERSTACK_HEARTBEAT_URL_OPENKEY_DB  "OpenKey DB saturation" heartbeat URL.
+#
+# PROOF RUNS: gh workflow run db-saturation-probe.yml -f threshold_pct=1
+#   expect /fail (pages team@). Then with no inputs: expect a ping.
+#   Dispatch only from the default branch, and warn on-call first.
+#
+# Adding a target: add its entry to db-saturation-targets.json and set its two
+# secrets. Secrets must also appear in the env block below, because Actions only
+# passes secrets that are named here.
+
+name: DB Saturation Probe
+run-name: >-
+  ${{ github.event_name == 'workflow_dispatch' && inputs.threshold_pct != '' && format('DB saturation probe (manual CPU threshold {0}%)', inputs.threshold_pct) || 'DB saturation probe' }}
+
+on:
+  schedule:
+    - cron: "*/5 * * * *"
+  workflow_dispatch:
+    inputs:
+      threshold_pct:
+        description: "Proof runs only: manual CPU threshold (0-100). Empty uses the targets file (85)."
+        required: false
+        default: ""
+        type: string
+      target:
+        description: "Target label from db-saturation-targets.json. Empty runs every target."
+        required: false
+        default: ""
+        type: string
+
+permissions:
+  contents: read
+
+concurrency:
+  group: db-saturation-probe
+  cancel-in-progress: false
+
+jobs:
+  check:
+    runs-on: ubuntu-latest
+    timeout-minutes: 3
+    defaults:
+      run:
+        working-directory: test/probe
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version: "20"
+          cache: npm
+          cache-dependency-path: test/probe/package-lock.json
+
+      - name: Install probe dependencies
+        run: npm ci --no-audit --no-fund
+
+      - name: Type-check probe
+        run: npm run typecheck
+
+      - name: Run saturation check
+        env:
+          DB_TARGET: ${{ inputs.target }}
+          THRESHOLD_PCT: ${{ inputs.threshold_pct }}
+          PLANETSCALE_METRICS_TOKEN_NODE_DB: ${{ secrets.PLANETSCALE_METRICS_TOKEN_NODE_DB }}
+          BETTERSTACK_HEARTBEAT_URL_NODE_DB: ${{ secrets.BETTERSTACK_HEARTBEAT_URL_NODE_DB }}
+          PLANETSCALE_METRICS_TOKEN_OPENKEY_DB: ${{ secrets.PLANETSCALE_METRICS_TOKEN_OPENKEY_DB }}
+          BETTERSTACK_HEARTBEAT_URL_OPENKEY_DB: ${{ secrets.BETTERSTACK_HEARTBEAT_URL_OPENKEY_DB }}
+        run: npm run probe:db

test/probe/README.md

@@ -198,3 +198,30 @@ PROBE_PRIVATE_KEY=0x... PROBE_MODE=dense npm run probe # promote watch (~66 min)
 ```
 
 Without `BETTERSTACK_HEARTBEAT_URL` nothing is reported to Better Stack.
+
+## DB saturation probe (TC-1601)
+
+`db-saturation-run.ts` checks each target in `db-saturation-targets.json` once
+per run (`db-saturation-probe.yml`, every 5 min). It finds the database's production
+branch from the PlanetScale branch list each run, reads that branch's metrics, and
+sends one Better Stack heartbeat decision per target.
+
+- **CPU:** breach when at least 13 of the last 15 one-minute points of
+  `planetscale_primary_pods_cpu_util_percentages` are above `cpu_pct` (85).
+  Points newer than 90 s are ignored for ingestion lag.
+- **Pool:** same counting on the target's `pool` rule. The shipped target pins
+  direct Postgres, port 5432: active connections >= 15
+  (`TINYCLOUD_DATABASE__MAX_CONNECTIONS`). `pool: null` is a config fault. A
+  PgBouncer port (6432) would use `planetscale_pgbouncer_pools_client` waiting clients > 0.
+- **Outcomes:** breach or config fault -> `/fail` with numeric counts. Healthy
+  -> ping. Unknown (fewer than max(`min_points`, `min_breach_points`) valid points,
+  no series, or not exactly one production branch) or transient API error
+  (429, 5xx, timeout) -> no call, so the missed-heartbeat alert fires.
+- **Secrets:** each target reads the env names in its `token_secret` and
+  `heartbeat_secret`. A target whose secrets are unset is skipped. The token is
+  a PlanetScale service token (`ID:TOKEN`) with `read_branch` on all branches of that database.
+
+Proof runs (default branch only): `gh workflow run db-saturation-probe.yml -f threshold_pct=1`
+should send `/fail`; a run with no inputs should send a ping.
+
+Local: `npm run probe:db` (skips when secrets are unset). Tests: `npm test`.

test/probe/db-saturation-run.ts

@@ -0,0 +1,117 @@
+/**
+ * TC-1601 runner: one heartbeat decision per target in db-saturation-targets.json.
+ *
+ *   tsx db-saturation-run.ts
+ *
+ * Env:
+ *   DB_TARGET        optional target label; default runs every target
+ *   THRESHOLD_PCT    optional manual CPU threshold (0-100). Proof runs only; the
+ *                    label in the log and the /fail body says so.
+ *   <token_secret>   PlanetScale token, ID:TOKEN, read_branch on all branches of that database
+ *   <heartbeat_secret> Better Stack heartbeat URL for that target
+ *
+ * Heartbeat: ok -> ping; breach or config fault -> /fail; unknown or transient
+ * API error -> nothing (the missed-heartbeat alert handles it). A target whose
+ * token or heartbeat env is unset is skipped, like write-probe. Logs never
+ * contain the token, the heartbeat URL, or query text.
+ */
+import { readFileSync } from "node:fs";
+import { classify, detail, heartbeatUrl, redact, report, type Delivery, type Fetch } from "./probe-core";
+import { ConfigFault, branchesUrl, evaluateTarget, loadTargets, metricsUrl, parseSeries, productionBranch, type Target, type Verdict } from "./db-saturation";
+
+const TIMEOUT_MS = 10_000;
+const TARGETS_FILE = new URL("./db-saturation-targets.json", import.meta.url);
+
+/** GET a PlanetScale endpoint. 429 and 5xx are transient; any other non-2xx is a token, scope or path fault. */
+async function getJson(url: string, token: string, fetchFn: Fetch): Promise<unknown> {
+  const res = await fetchFn(url, {
+    headers: { authorization: token, accept: "application/json" },
+    signal: AbortSignal.timeout(TIMEOUT_MS),
+  });
+  // 429 and 5xx are transient: no heartbeat call, the missed-heartbeat alert covers it.
+  if (res.status === 429 || res.status >= 500) throw new Error(`planetscale HTTP ${res.status}`);
+  // Any other non-2xx is a token, scope or path fault and must page now.
+  if (!res.ok) throw new ConfigFault(`planetscale HTTP ${res.status}`);
+  return res.json();
+}
+
+function parseOverride(raw: string | undefined): number | undefined {
+  if (raw === undefined || raw.trim() === "") return undefined;
+  const n = Number(raw);
+  if (!Number.isFinite(n) || n < 0 || n > 100) throw new ConfigFault(`THRESHOLD_PCT must be 0-100, got ${raw}`);
+  return n;
+}
+
+function failBody(label: string, verdict: Verdict, now: Date): string {
+  const at = now.toISOString().replace(/\.\d{3}Z$/, "Z");
+  return `TC-1601 db saturation ${verdict.status}: target=${label} ${verdict.detail} at=${at}`;
+}
+
+let targetSecretNames = new Set<string>();
+
+async function main(): Promise<number> {
+  const only = process.env.DB_TARGET?.trim() || undefined;
+  const override = parseOverride(process.env.THRESHOLD_PCT);
+  const targets = loadTargets(JSON.parse(readFileSync(TARGETS_FILE, "utf8")));
+  targetSecretNames = new Set(targets.flatMap((t) => [t.token_secret, t.heartbeat_secret]));
+  const selected = only ? targets.filter((t) => t.label === only) : targets;
+  if (selected.length === 0) throw new ConfigFault(`no target named ${only}`);
+
+  let deliveryFailed = false;
+  for (const target of selected) {
+    const token = process.env[target.token_secret]?.trim();
+    const heartbeatRaw = process.env[target.heartbeat_secret]?.trim();
+    if (!token || !heartbeatRaw) {
+      const missing = !token ? target.token_secret : target.heartbeat_secret;
+      console.log(`${target.label}: skipped (${missing} not set)`);
+      continue;
+    }
+    let heartbeat: URL | undefined;
+    try {
+      heartbeat = heartbeatUrl(heartbeatRaw);
+    } catch {
+      console.log(`${target.label}: config fault, ${target.heartbeat_secret} is not a Better Stack heartbeat URL`);
+      deliveryFailed = true;
+      continue;
+    }
+    const t = override === undefined ? target : { ...target, cpu_pct: override };
+    const label = override === undefined ? target.label : `${target.label} (manual cpu threshold ${override}%)`;
+    const now = new Date();
+
+    let verdict: Verdict;
+    try {
+      const branch = productionBranch(await getJson(branchesUrl(t), token, fetch));
+      verdict = branch === null
+        ? { status: "unknown", detail: "no single production branch in the branch list" }
+        : evaluateTarget(t, parseSeries(await getJson(metricsUrl(t, branch), token, fetch)), Math.floor(now.getTime() / 1000));
+    } catch (error) {
+      if (!(error instanceof ConfigFault)) {
+        console.log(`${label}: check skipped, no heartbeat: ${redact(classify(error), [token])}`);
+        continue;
+      }
+      verdict = { status: "config", detail: error.message };
+    }
+
+    console.log(`${label}: ${verdict.status} ${redact(verdict.detail, [token], 600)}`);
+    if (verdict.status === "unknown") continue;
+    const action = verdict.status === "ok" ? "ping" : "fail";
+    const body = action === "fail" ? redact(failBody(label, verdict, now), [token], 400) : undefined;
+    const delivery: Delivery = await report(action, heartbeat, body, fetch);
+    console.log(delivery.line);
+    if (!delivery.ok) deliveryFailed = true;
+  }
+  return deliveryFailed ? 1 : 0;
+}
+
+main().then(
+  (code) => {
+    process.exitCode = code;
+  },
+  (err: unknown) => {
+    const secrets = Object.entries(process.env)
+      .filter(([k, v]) => v && (targetSecretNames.has(k) || /^(PLANETSCALE_METRICS_TOKEN|BETTERSTACK_HEARTBEAT_URL)/.test(k)))
+      .map(([, v]) => v as string);
+    console.error(`[db-saturation] crashed: ${detail(err, secrets)}`);
+    process.exitCode = 1;
+  },
+);

test/probe/db-saturation-targets.json

@@ -0,0 +1,19 @@
+[
+  {
+    "label": "node-db",
+    "org": "tinycloudlabs",
+    "database": "tinycloud-node-production",
+    "token_secret": "PLANETSCALE_METRICS_TOKEN_NODE_DB",
+    "heartbeat_secret": "BETTERSTACK_HEARTBEAT_URL_NODE_DB",
+    "cpu_pct": 85,
+    "window_min": 15,
+    "min_points": 13,
+    "min_breach_points": 13,
+    "pool": {
+      "metric": "planetscale_primary_postgres_connection_state",
+      "series_labels": { "state": "active" },
+      "op": "gte",
+      "threshold": 15
+    }
+  }
+]